diff --git a/lib/hosters.js b/lib/hosters.js index fd577c2..cee9680 100644 --- a/lib/hosters.js +++ b/lib/hosters.js @@ -499,24 +499,27 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s return null; } -async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle) { +async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle, opts) { const config = HOSTER_CONFIGS[hosterName]; if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`); - // For byse: snapshot the current file-code list so the post-upload poller - // can identify new arrivals even when the initial POST response has an - // empty filecode. let byseBaseline = null; if (hosterName === 'byse.sx') { - const baseline = await _fetchByseFileList(apiKey, signal); - byseBaseline = new Set(baseline.map(f => f.file_code)); + if (opts && opts.byseBaseline instanceof Set) { + byseBaseline = opts.byseBaseline; + } else { + const baseline = await _fetchByseFileList(apiKey, signal); + byseBaseline = new Set(baseline.map(f => f.file_code)); + } } - // Doodstream: same snapshot so a codeless upload response can be recovered by - // matching a newly-appeared file in the account by name (see below). let doodBaseline = null; if (hosterName === 'doodstream.com') { - const baseline = await _fetchDoodstreamFileList(apiKey, signal); - doodBaseline = new Set(baseline.map(f => f.file_code)); + if (opts && opts.doodBaseline instanceof Set) { + doodBaseline = opts.doodBaseline; + } else { + const baseline = await _fetchDoodstreamFileList(apiKey, signal); + doodBaseline = new Set(baseline.map(f => f.file_code)); + } } // Step 1: Get upload server @@ -647,8 +650,23 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro throw new Error(msg); } +async function prefetchBaseline(hosterName, apiKey, signal) { + try { + if (hosterName === 'byse.sx') { + const baseline = await _fetchByseFileList(apiKey, signal); + return new Set(baseline.map(f => f.file_code)); + } + if (hosterName === 'doodstream.com') { + const baseline = await _fetchDoodstreamFileList(apiKey, signal); + return new Set(baseline.map(f => f.file_code)); + } + } catch { /* leave caller to fall back to per-job fetch */ } + return null; +} + module.exports = { uploadFile, + prefetchBaseline, HOSTER_CONFIGS, __test: { extractUploadServerUrl, diff --git a/lib/upload-manager.js b/lib/upload-manager.js index 676e9a2..220a249 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -2,7 +2,7 @@ const { EventEmitter } = require('events'); const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -const { uploadFile } = require('./hosters'); +const { uploadFile, prefetchBaseline } = require('./hosters'); const VidmolyUploader = require('./vidmoly-upload'); const VoeUploader = require('./voe-upload'); const DoodstreamUploader = require('./doodstream-upload'); @@ -42,6 +42,7 @@ class UploadManager extends EventEmitter { this._failedAccounts = new Map(); // hoster -> Set of failed accountIds this._accountOverrides = new Map(); // hoster -> fallback account object this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none) + this._baselineCache = new Map(); // hoster:apiKey -> Promise> (one fetch shared across all jobs in batch) } switchAccount(hoster, fallbackAccount) { @@ -278,6 +279,7 @@ class UploadManager extends EventEmitter { this.jobAbortControllers.clear(); this.cancelledJobIds.clear(); this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch + this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance) this.semaphores = {}; this.globalSemaphore = null; this.globalThrottle = null; @@ -919,7 +921,9 @@ class UploadManager extends EventEmitter { const apiKey = await this._resolveDoodstreamApiKey(task); if (apiKey) { this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) }); - return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle); + return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, { + doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal) + }); } this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) }); const dood = new DoodstreamUploader(); @@ -929,10 +933,23 @@ class UploadManager extends EventEmitter { const clouddrop = new ClouddropUploader(task.apiKey); return clouddrop.upload(task.file, progressCb, signal, throttle); } else { - return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle); + const baselineOpts = {}; + if (task.hoster === 'byse.sx') baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal); + if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal); + return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts); } } + _getBaseline(hosterName, apiKey, signal) { + if (!apiKey) return Promise.resolve(null); + const key = `${hosterName}:${apiKey}`; + let pending = this._baselineCache.get(key); + if (pending) return pending; + pending = prefetchBaseline(hosterName, apiKey, signal); + this._baselineCache.set(key, pending); + return pending; + } + // Resolve (and cache per batch) the doodstream API key for a login-only // account by logging in once and scraping+validating it from the session. // Returns the key string, or '' when none could be derived (cached either way diff --git a/main.js b/main.js index c4ac6e3..b5448da 100644 --- a/main.js +++ b/main.js @@ -1297,33 +1297,32 @@ ipcMain.handle('select-folder', async () => { }); if (result.canceled || !result.filePaths.length) return null; - // Recursively collect all files from selected folders const files = []; - const walk = (dir) => { - try { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) walk(full); - else if (entry.isFile()) files.push(full); - } - } catch {} - }; - for (const folder of result.filePaths) walk(folder); + for (const folder of result.filePaths) await walkFolderAsync(folder, files); return files.length > 0 ? files : null; }); +async function walkFolderAsync(rootDir, outFiles) { + const fsp = fs.promises; + const stack = [rootDir]; + let scanned = 0; + while (stack.length > 0) { + const dir = stack.pop(); + let entries; + try { entries = await fsp.readdir(dir, { withFileTypes: true }); } + catch { continue; } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.isFile()) outFiles.push(full); + } + if ((++scanned % 8) === 0) await new Promise(setImmediate); + } +} + ipcMain.handle('resolve-folder-files', async (_event, folderPath) => { const files = []; - const walk = (dir) => { - try { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) walk(full); - else if (entry.isFile()) files.push(full); - } - } catch {} - }; - walk(folderPath); + await walkFolderAsync(folderPath, files); return files; }); @@ -1383,8 +1382,27 @@ ipcMain.handle('start-upload', (_event, payload) => { // Pass hoster settings to the upload manager uploadManager = new UploadManager(config.hosterSettings || {}, config.globalSettings || {}); + const _progressByJob = new Map(); + const _progressTerminalQueue = []; + let _progressFlushTimer = null; + const PROGRESS_BATCH_INTERVAL_MS = 100; + function _scheduleProgressFlush() { + if (_progressFlushTimer) return; + _progressFlushTimer = setTimeout(() => { + _progressFlushTimer = null; + if (!mainWindow || mainWindow.isDestroyed()) { + _progressByJob.clear(); + _progressTerminalQueue.length = 0; + return; + } + const batch = _progressTerminalQueue.splice(0); + for (const v of _progressByJob.values()) batch.push(v); + _progressByJob.clear(); + if (batch.length) mainWindow.webContents.send('upload-progress-batch', batch); + }, PROGRESS_BATCH_INTERVAL_MS); + } + uploadManager.on('progress', (data) => { - // Only log state changes, not continuous progress updates if (data.status !== 'uploading') { debugLog(`progress: ${data.fileName} ${data.hoster} ${data.status} ${data.error || ''}`); _appendJobLog(data.jobId, { @@ -1393,10 +1411,6 @@ ipcMain.handle('start-upload', (_event, payload) => { error: data.error || null, attempt: data.attempt || 0, maxAttempts: data.maxAttempts || 0 }); } - // Write to fileuploader.log immediately when a single upload finishes — - // unless the user disabled logging for this hoster (per-hoster toggle). - // Read from the live uploadManager.hosterSettings so a mid-batch toggle - // (which calls updateSettings) takes effect immediately. if (data.status === 'done' && data.result) { const link = data.result.download_url || data.result.embed_url || data.result.file_code || ''; if (link) { @@ -1409,9 +1423,16 @@ ipcMain.handle('start-upload', (_event, payload) => { debugLog(`WARNING: done but no link for ${data.fileName} @ ${data.hoster}: ${JSON.stringify(data.result)}`); } } - if (mainWindow && !mainWindow.isDestroyed()) { - mainWindow.webContents.send('upload-progress', data); + const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped'; + if (isTerminal) { + if (data.jobId) _progressByJob.delete(data.jobId); + _progressTerminalQueue.push(data); + } else if (data.jobId) { + _progressByJob.set(data.jobId, data); + } else { + _progressTerminalQueue.push(data); } + _scheduleProgressFlush(); }); uploadManager.on('stats', (data) => { diff --git a/preload.js b/preload.js index c3e6446..abfd86d 100644 --- a/preload.js +++ b/preload.js @@ -93,6 +93,9 @@ contextBridge.exposeInMainWorld('api', { onUploadProgress: (callback) => { ipcRenderer.on('upload-progress', (_event, data) => callback(data)); }, + onUploadProgressBatch: (callback) => { + ipcRenderer.on('upload-progress-batch', (_event, batch) => callback(batch)); + }, onUploadBatchDone: (callback) => { ipcRenderer.on('upload-batch-done', (_event, data) => callback(data)); }, diff --git a/renderer/app.js b/renderer/app.js index f4356b5..e63f596 100644 --- a/renderer/app.js +++ b/renderer/app.js @@ -111,6 +111,12 @@ async function init() { window.api.onUploadProgress((data) => { handleProgress(data); }); + if (window.api.onUploadProgressBatch) { + window.api.onUploadProgressBatch((batch) => { + if (!Array.isArray(batch)) return; + for (let i = 0; i < batch.length; i++) handleProgress(batch[i]); + }); + } window.api.onUploadBatchDone((data) => { handleBatchDone(data); }); @@ -4097,17 +4103,32 @@ function renderHistoryTable(container) { ${headerCell('date', 'Date')}${headerCell('filename', 'Filename')}${headerCell('host', 'Host')}${headerCell('link', 'Link')} `; - rows.forEach(row => { - html += ` - ${escapeHtml(row.date)} - ${escapeHtml(row.filename)} - ${escapeHtml(row.host)} - ${escapeHtml(row.link)} - `; - }); - - html += ''; - container.innerHTML = html; + const parts = [html]; + const len = rows.length; + for (let i = 0; i < len; i++) { + const row = rows[i]; + const link = row.link || ''; + const date = escapeHtml(row.date); + const filename = escapeHtml(row.filename); + const host = escapeHtml(row.host); + const linkHtml = escapeHtml(link); + const linkAttr = escapeAttr(link); + parts.push(''); + parts.push(date); + parts.push(''); + parts.push(filename); + parts.push(''); + parts.push(host); + parts.push(''); + parts.push(linkHtml); + parts.push(''); + } + parts.push(''); + container.innerHTML = parts.join(''); // Delegated listeners: bind once per render-target instead of once per // row/header. With a 5000-row history the per-row bind path was a diff --git a/tests/upload-manager.test.js b/tests/upload-manager.test.js index 6a9f761..7c10273 100644 --- a/tests/upload-manager.test.js +++ b/tests/upload-manager.test.js @@ -31,6 +31,7 @@ describe('UploadManager', () => { const origRequire = module.constructor.prototype.require; const hosters = require('../lib/hosters'); hosters.uploadFile = mockUploadFile; + hosters.prefetchBaseline = async () => null; // Mock fs.statSync for test file paths const fs = require('fs');