Administrator/Twitch-VOD-Manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 212 Wiki Activity

feat(db): JSON to SQLite migrator (idempotent, fail-soft, 8 tests)

Browse Source 
Migrates config.json (31 whitelisted config_kv keys + downloaded_vod_ids +
streamers) and download_queue.json (queue_items). Per-source try/catch:
malformed JSON logs into result.errors and continues. .v4-backup copies
written on success. migrations_applied marker prevents double-runs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
xRangerDE 2026-05-11 22:04:25 +02:00
parent 93481999bd
commit 6480bc2586
2 changed files with 323 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
src/main/domain/migrator.test.ts Normal file
View File

@ -0,0 +1,122 @@
import { test, expect, describe, beforeEach, afterEach } from 'vitest';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { openDatabase, type DbHandle } from '../infra/db';
import { migrateJsonToSqlite } from './migrator';
let tmpDir: string;
let appDataDir: string;
let db: DbHandle;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'migrator-'));
appDataDir = path.join(tmpDir, 'appdata');
fs.mkdirSync(appDataDir, { recursive: true });
db = openDatabase(path.join(tmpDir, 'app.db'));
});
afterEach(() => {
db.close();
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
});
function writeJson(name: string, payload: unknown): string {
const target = path.join(appDataDir, name);
fs.writeFileSync(target, JSON.stringify(payload, null, 2), 'utf-8');
return target;
}
describe('migrateJsonToSqlite', () => {
test('no JSON files: writes migrations_applied marker', () => {
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.configMigrated).toBe(false);
expect(result.queueMigrated).toBe(false);
expect(result.downloadedVodsCount).toBe(0);
expect(result.streamersCount).toBe(0);
const marker = db.get<{ name: string }>('SELECT name FROM migrations_applied WHERE name = ?', ['v4-to-v5-jsons']);
expect(marker?.name).toBe('v4-to-v5-jsons');
});
test('migrates config.json keys into config_kv', () => {
writeJson('config.json', {
language: 'de',
performance_mode: 'speed',
metadata_cache_minutes: 30,
downloaded_vod_ids: ['1', '2', '3'],
auto_record_streamers: ['foo', 'bar'],
});
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.configMigrated).toBe(true);
const lang = db.get<{ value: string }>('SELECT value FROM config_kv WHERE key = ?', ['language']);
expect(JSON.parse(lang!.value)).toBe('de');
const perf = db.get<{ value: string }>('SELECT value FROM config_kv WHERE key = ?', ['performance_mode']);
expect(JSON.parse(perf!.value)).toBe('speed');
});
test('migrates downloaded_vod_ids', () => {
writeJson('config.json', { downloaded_vod_ids: ['100', '200', '300'] });
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.downloadedVodsCount).toBe(3);
const rows = db.all<{ vod_id: string }>('SELECT vod_id FROM downloaded_vods ORDER BY vod_id');
expect(rows.map(r => r.vod_id)).toEqual(['100', '200', '300']);
});
test('migrates streamers from both auto-record and auto-vod-download lists', () => {
writeJson('config.json', {
auto_record_streamers: ['Alice', '@bob'],
auto_vod_download_streamers: ['bob', 'carol'],
});
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.streamersCount).toBeGreaterThanOrEqual(3);
const alice = db.get<{ login: string; auto_record: number }>('SELECT login, auto_record FROM streamers WHERE login = ?', ['alice']);
expect(alice?.auto_record).toBe(1);
const bob = db.get<{ login: string; auto_record: number; auto_vod_download: number }>('SELECT login, auto_record, auto_vod_download FROM streamers WHERE login = ?', ['bob']);
expect(bob?.auto_record).toBe(1);
expect(bob?.auto_vod_download).toBe(1);
const carol = db.get<{ login: string; auto_vod_download: number }>('SELECT login, auto_vod_download FROM streamers WHERE login = ?', ['carol']);
expect(carol?.auto_vod_download).toBe(1);
});
test('migrates download_queue.json items', () => {
writeJson('download_queue.json', [
{ id: 'q1', status: 'pending', streamer: 'foo', vod_id: 'v1', created_at: 1000, updated_at: 1000 },
{ id: 'q2', status: 'completed', streamer: 'bar', vod_id: 'v2', created_at: 2000, updated_at: 3000, completed_at: 3000 },
]);
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.queueMigrated).toBe(true);
const all = db.all<{ id: string; status: string }>('SELECT id, status FROM queue_items ORDER BY id');
expect(all).toHaveLength(2);
expect(all[0].status).toBe('pending');
expect(all[1].status).toBe('completed');
});
test('idempotent second run', () => {
writeJson('config.json', { downloaded_vod_ids: ['1', '2'] });
migrateJsonToSqlite({ db, appDataDir });
const result2 = migrateJsonToSqlite({ db, appDataDir });
expect(result2.alreadyApplied).toBe(true);
const count = db.get<{ c: number }>('SELECT COUNT(*) AS c FROM downloaded_vods');
expect(count?.c).toBe(2);
});
test('writes .v4-backup of source JSONs', () => {
const configPath = writeJson('config.json', { language: 'en' });
migrateJsonToSqlite({ db, appDataDir });
expect(fs.existsSync(configPath + '.v4-backup')).toBe(true);
expect(fs.readFileSync(configPath + '.v4-backup', 'utf-8')).toContain('"language": "en"');
});
test('malformed JSON is logged + skipped', () => {
fs.writeFileSync(path.join(appDataDir, 'config.json'), '{ not valid json', 'utf-8');
const result = migrateJsonToSqlite({ db, appDataDir });
expect(result.configMigrated).toBe(false);
expect(result.errors.length).toBeGreaterThan(0);
expect(result.errors[0].source).toBe('config.json');
});
});

201
src/main/domain/migrator.ts Normal file
View File

@ -0,0 +1,201 @@
import * as fs from 'fs';
import * as path from 'path';
import type { DbHandle } from '../infra/db';
import { normalizeLogin } from './config-normalize';
export interface MigratorOptions {
db: DbHandle;
appDataDir: string;
}
export interface MigrationError {
source: string;
message: string;
}
export interface MigrationResult {
alreadyApplied: boolean;
configMigrated: boolean;
queueMigrated: boolean;
downloadedVodsCount: number;
streamersCount: number;
errors: MigrationError[];
}
const MIGRATION_NAME = 'v4-to-v5-jsons';
const CONFIG_KV_KEYS = [
'language', 'performance_mode', 'metadata_cache_minutes', 'streamlink_quality',
'streamlink_disable_ads', 'download_chat_replay', 'capture_live_chat',
'discord_webhook_url', 'discord_notify_live_start', 'discord_notify_live_end',
'discord_notify_vod_complete', 'discord_notify_vod_auto_queued',
'auto_cleanup_enabled', 'auto_cleanup_days', 'auto_cleanup_target',
'auto_cleanup_action', 'log_stream_events', 'auto_vod_download_poll_minutes',
'auto_vod_max_age_hours', 'auto_resume_live_recording',
'auto_merge_resumed_parts', 'delete_parts_after_merge',
'auto_record_poll_seconds', 'filename_template_vod', 'filename_template_parts',
'filename_template_clip', 'smart_queue_scheduler', 'prevent_duplicate_downloads',
'persist_queue_on_restart', 'auto_resume_queue_on_startup',
'notify_on_each_completion',
] as const;
function backupOnce(srcPath: string): void {
const backupPath = srcPath + '.v4-backup';
if (!fs.existsSync(backupPath)) {
fs.copyFileSync(srcPath, backupPath);
}
}
function migrateConfig(db: DbHandle, configPath: string, errors: MigrationError[]): { ok: boolean; vodCount: number } {
try {
const raw = fs.readFileSync(configPath, 'utf-8');
const config = JSON.parse(raw) as Record<string, unknown>;
let vodCount = 0;
db.transaction(() => {
for (const key of CONFIG_KV_KEYS) {
if (key in config) {
db.run(
"INSERT OR REPLACE INTO config_kv(key, value, updated_at) VALUES (?, ?, strftime('%s','now'))",
[key, JSON.stringify(config[key])]
);
}
}
const vodIds = Array.isArray(config.downloaded_vod_ids) ? config.downloaded_vod_ids : [];
for (const id of vodIds) {
if (typeof id !== 'string' || !id) continue;
db.run('INSERT OR IGNORE INTO downloaded_vods(vod_id) VALUES (?)', [id]);
vodCount += 1;
}
const autoRec = Array.isArray(config.auto_record_streamers) ? config.auto_record_streamers : [];
for (const s of autoRec) {
if (typeof s !== 'string' || !s) continue;
const login = normalizeLogin(s);
if (!login) continue;
db.run(
'INSERT INTO streamers(login, auto_record) VALUES (?, 1) ON CONFLICT(login) DO UPDATE SET auto_record = 1',
[login]
);
}
const autoDl = Array.isArray(config.auto_vod_download_streamers) ? config.auto_vod_download_streamers : [];
for (const s of autoDl) {
if (typeof s !== 'string' || !s) continue;
const login = normalizeLogin(s);
if (!login) continue;
db.run(
'INSERT INTO streamers(login, auto_vod_download) VALUES (?, 1) ON CONFLICT(login) DO UPDATE SET auto_vod_download = 1',
[login]
);
}
});
backupOnce(configPath);
return { ok: true, vodCount };
} catch (e) {
errors.push({ source: 'config.json', message: e instanceof Error ? e.message : String(e) });
return { ok: false, vodCount: 0 };
}
}
function migrateQueue(db: DbHandle, queuePath: string, errors: MigrationError[]): boolean {
try {
const raw = fs.readFileSync(queuePath, 'utf-8');
const queue = JSON.parse(raw);
if (!Array.isArray(queue)) return false;
const now = Math.floor(Date.now() / 1000);
db.transaction(() => {
for (const rawItem of queue) {
if (!rawItem || typeof rawItem !== 'object') continue;
const item = rawItem as Record<string, unknown>;
const id = typeof item.id === 'string' ? item.id : null;
if (!id) continue;
db.run(
`INSERT OR REPLACE INTO queue_items
(id, streamer_login, vod_id, clip_id, title, output_path, status,
progress_pct, error_message, created_at, updated_at, completed_at, payload_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
id,
typeof item.streamer === 'string' ? normalizeLogin(item.streamer) : null,
typeof item.vod_id === 'string' ? item.vod_id : null,
typeof item.clip_id === 'string' ? item.clip_id : null,
typeof item.title === 'string' ? item.title : null,
typeof item.output_path === 'string' ? item.output_path : null,
typeof item.status === 'string' ? item.status : 'pending',
typeof item.progress_pct === 'number' ? item.progress_pct : null,
typeof item.error_message === 'string' ? item.error_message : null,
typeof item.created_at === 'number' ? item.created_at : now,
typeof item.updated_at === 'number' ? item.updated_at : now,
typeof item.completed_at === 'number' ? item.completed_at : null,
JSON.stringify(item),
]
);
}
});
backupOnce(queuePath);
return true;
} catch (e) {
errors.push({ source: 'download_queue.json', message: e instanceof Error ? e.message : String(e) });
return false;
}
}
export function migrateJsonToSqlite(opts: MigratorOptions): MigrationResult {
const { db, appDataDir } = opts;
const errors: MigrationError[] = [];
const existing = db.get<{ name: string }>(
'SELECT name FROM migrations_applied WHERE name = ?',
[MIGRATION_NAME]
);
if (existing) {
return {
alreadyApplied: true,
configMigrated: false,
queueMigrated: false,
downloadedVodsCount: 0,
streamersCount: 0,
errors: [],
};
}
let configMigrated = false;
let queueMigrated = false;
let downloadedVodsCount = 0;
const configPath = path.join(appDataDir, 'config.json');
if (fs.existsSync(configPath)) {
const r = migrateConfig(db, configPath, errors);
configMigrated = r.ok;
downloadedVodsCount = r.vodCount;
}
const queuePath = path.join(appDataDir, 'download_queue.json');
if (fs.existsSync(queuePath)) {
queueMigrated = migrateQueue(db, queuePath, errors);
}
const streamersCount = db.get<{ c: number }>('SELECT COUNT(*) AS c FROM streamers')?.c ?? 0;
db.run(
'INSERT INTO migrations_applied(name, payload) VALUES (?, ?)',
[
MIGRATION_NAME,
JSON.stringify({ configMigrated, queueMigrated, downloadedVodsCount, streamersCount, errorCount: errors.length }),
]
);
return {
alreadyApplied: false,
configMigrated,
queueMigrated,
downloadedVodsCount,
streamersCount,
errors,
};
}