perf: per-batch baseline cache, async folder walk, history-table fast path, progress IPC batching

lib/hosters.js

@@ -499,24 +499,27 @@ async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, s
   return null;
 }
 
-async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle) {
+async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle, opts) {
   const config = HOSTER_CONFIGS[hosterName];
   if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`);
 
-  // For byse: snapshot the current file-code list so the post-upload poller
-  // can identify new arrivals even when the initial POST response has an
-  // empty filecode.
   let byseBaseline = null;
   if (hosterName === 'byse.sx') {
-    const baseline = await _fetchByseFileList(apiKey, signal);
+    if (opts && opts.byseBaseline instanceof Set) {
-    byseBaseline = new Set(baseline.map(f => f.file_code));
+      byseBaseline = opts.byseBaseline;
+    } else {
+      const baseline = await _fetchByseFileList(apiKey, signal);
+      byseBaseline = new Set(baseline.map(f => f.file_code));
+    }
   }
-  // Doodstream: same snapshot so a codeless upload response can be recovered by
-  // matching a newly-appeared file in the account by name (see below).
   let doodBaseline = null;
   if (hosterName === 'doodstream.com') {
-    const baseline = await _fetchDoodstreamFileList(apiKey, signal);
+    if (opts && opts.doodBaseline instanceof Set) {
-    doodBaseline = new Set(baseline.map(f => f.file_code));
+      doodBaseline = opts.doodBaseline;
+    } else {
+      const baseline = await _fetchDoodstreamFileList(apiKey, signal);
+      doodBaseline = new Set(baseline.map(f => f.file_code));
+    }
   }
 
   // Step 1: Get upload server
@@ -647,8 +650,23 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
   throw new Error(msg);
 }
 
+async function prefetchBaseline(hosterName, apiKey, signal) {
+  try {
+    if (hosterName === 'byse.sx') {
+      const baseline = await _fetchByseFileList(apiKey, signal);
+      return new Set(baseline.map(f => f.file_code));
+    }
+    if (hosterName === 'doodstream.com') {
+      const baseline = await _fetchDoodstreamFileList(apiKey, signal);
+      return new Set(baseline.map(f => f.file_code));
+    }
+  } catch { /* leave caller to fall back to per-job fetch */ }
+  return null;
+}
+
 module.exports = {
   uploadFile,
+  prefetchBaseline,
   HOSTER_CONFIGS,
   __test: {
     extractUploadServerUrl,

lib/upload-manager.js

@@ -2,7 +2,7 @@ const { EventEmitter } = require('events');
 const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');
-const { uploadFile } = require('./hosters');
+const { uploadFile, prefetchBaseline } = require('./hosters');
 const VidmolyUploader = require('./vidmoly-upload');
 const VoeUploader = require('./voe-upload');
 const DoodstreamUploader = require('./doodstream-upload');
@@ -42,6 +42,7 @@ class UploadManager extends EventEmitter {
     this._failedAccounts = new Map(); // hoster -> Set of failed accountIds
     this._accountOverrides = new Map(); // hoster -> fallback account object
     this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none)
+    this._baselineCache = new Map(); // hoster:apiKey -> Promise<Set<file_code>> (one fetch shared across all jobs in batch)
   }
 
   switchAccount(hoster, fallbackAccount) {
@@ -278,6 +279,7 @@ class UploadManager extends EventEmitter {
     this.jobAbortControllers.clear();
     this.cancelledJobIds.clear();
     this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
+    this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
     this.semaphores = {};
     this.globalSemaphore = null;
     this.globalThrottle = null;
@@ -919,7 +921,9 @@ class UploadManager extends EventEmitter {
       const apiKey = await this._resolveDoodstreamApiKey(task);
       if (apiKey) {
         this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) });
-        return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle);
+        return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, {
+          doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal)
+        });
       }
       this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) });
       const dood = new DoodstreamUploader();
@@ -929,10 +933,23 @@ class UploadManager extends EventEmitter {
       const clouddrop = new ClouddropUploader(task.apiKey);
       return clouddrop.upload(task.file, progressCb, signal, throttle);
     } else {
-      return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle);
+      const baselineOpts = {};
+      if (task.hoster === 'byse.sx') baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal);
+      if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal);
+      return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts);
     }
   }
 
+  _getBaseline(hosterName, apiKey, signal) {
+    if (!apiKey) return Promise.resolve(null);
+    const key = `${hosterName}:${apiKey}`;
+    let pending = this._baselineCache.get(key);
+    if (pending) return pending;
+    pending = prefetchBaseline(hosterName, apiKey, signal);
+    this._baselineCache.set(key, pending);
+    return pending;
+  }
+
   // Resolve (and cache per batch) the doodstream API key for a login-only
   // account by logging in once and scraping+validating it from the session.
   // Returns the key string, or '' when none could be derived (cached either way

main.js

@@ -1297,33 +1297,32 @@ ipcMain.handle('select-folder', async () => {
   });
   if (result.canceled || !result.filePaths.length) return null;
 
-  // Recursively collect all files from selected folders
   const files = [];
-  const walk = (dir) => {
+  for (const folder of result.filePaths) await walkFolderAsync(folder, files);
-    try {
-      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-        const full = path.join(dir, entry.name);
-        if (entry.isDirectory()) walk(full);
-        else if (entry.isFile()) files.push(full);
-      }
-    } catch {}
-  };
-  for (const folder of result.filePaths) walk(folder);
   return files.length > 0 ? files : null;
 });
 
+async function walkFolderAsync(rootDir, outFiles) {
+  const fsp = fs.promises;
+  const stack = [rootDir];
+  let scanned = 0;
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    let entries;
+    try { entries = await fsp.readdir(dir, { withFileTypes: true }); }
+    catch { continue; }
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) stack.push(full);
+      else if (entry.isFile()) outFiles.push(full);
+    }
+    if ((++scanned % 8) === 0) await new Promise(setImmediate);
+  }
+}
+
 ipcMain.handle('resolve-folder-files', async (_event, folderPath) => {
   const files = [];
-  const walk = (dir) => {
+  await walkFolderAsync(folderPath, files);
-    try {
-      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-        const full = path.join(dir, entry.name);
-        if (entry.isDirectory()) walk(full);
-        else if (entry.isFile()) files.push(full);
-      }
-    } catch {}
-  };
-  walk(folderPath);
   return files;
 });
 
@@ -1383,8 +1382,27 @@ ipcMain.handle('start-upload', (_event, payload) => {
   // Pass hoster settings to the upload manager
   uploadManager = new UploadManager(config.hosterSettings || {}, config.globalSettings || {});
 
+  const _progressByJob = new Map();
+  const _progressTerminalQueue = [];
+  let _progressFlushTimer = null;
+  const PROGRESS_BATCH_INTERVAL_MS = 100;
+  function _scheduleProgressFlush() {
+    if (_progressFlushTimer) return;
+    _progressFlushTimer = setTimeout(() => {
+      _progressFlushTimer = null;
+      if (!mainWindow || mainWindow.isDestroyed()) {
+        _progressByJob.clear();
+        _progressTerminalQueue.length = 0;
+        return;
+      }
+      const batch = _progressTerminalQueue.splice(0);
+      for (const v of _progressByJob.values()) batch.push(v);
+      _progressByJob.clear();
+      if (batch.length) mainWindow.webContents.send('upload-progress-batch', batch);
+    }, PROGRESS_BATCH_INTERVAL_MS);
+  }
+
   uploadManager.on('progress', (data) => {
-    // Only log state changes, not continuous progress updates
     if (data.status !== 'uploading') {
       debugLog(`progress: ${data.fileName} ${data.hoster} ${data.status} ${data.error || ''}`);
       _appendJobLog(data.jobId, {
@@ -1393,10 +1411,6 @@ ipcMain.handle('start-upload', (_event, payload) => {
         error: data.error || null, attempt: data.attempt || 0, maxAttempts: data.maxAttempts || 0
       });
     }
-    // Write to fileuploader.log immediately when a single upload finishes —
-    // unless the user disabled logging for this hoster (per-hoster toggle).
-    // Read from the live uploadManager.hosterSettings so a mid-batch toggle
-    // (which calls updateSettings) takes effect immediately.
     if (data.status === 'done' && data.result) {
       const link = data.result.download_url || data.result.embed_url || data.result.file_code || '';
       if (link) {
@@ -1409,9 +1423,16 @@ ipcMain.handle('start-upload', (_event, payload) => {
         debugLog(`WARNING: done but no link for ${data.fileName} @ ${data.hoster}: ${JSON.stringify(data.result)}`);
       }
     }
-    if (mainWindow && !mainWindow.isDestroyed()) {
+    const isTerminal = data.status === 'done' || data.status === 'error' || data.status === 'aborted' || data.status === 'skipped';
-      mainWindow.webContents.send('upload-progress', data);
+    if (isTerminal) {
+      if (data.jobId) _progressByJob.delete(data.jobId);
+      _progressTerminalQueue.push(data);
+    } else if (data.jobId) {
+      _progressByJob.set(data.jobId, data);
+    } else {
+      _progressTerminalQueue.push(data);
     }
+    _scheduleProgressFlush();
   });
 
   uploadManager.on('stats', (data) => {

preload.js

@@ -93,6 +93,9 @@ contextBridge.exposeInMainWorld('api', {
   onUploadProgress: (callback) => {
     ipcRenderer.on('upload-progress', (_event, data) => callback(data));
   },
+  onUploadProgressBatch: (callback) => {
+    ipcRenderer.on('upload-progress-batch', (_event, batch) => callback(batch));
+  },
   onUploadBatchDone: (callback) => {
     ipcRenderer.on('upload-batch-done', (_event, data) => callback(data));
   },

renderer/app.js

@@ -111,6 +111,12 @@ async function init() {
   window.api.onUploadProgress((data) => {
     handleProgress(data);
   });
+  if (window.api.onUploadProgressBatch) {
+    window.api.onUploadProgressBatch((batch) => {
+      if (!Array.isArray(batch)) return;
+      for (let i = 0; i < batch.length; i++) handleProgress(batch[i]);
+    });
+  }
   window.api.onUploadBatchDone((data) => {
     handleBatchDone(data);
   });
@@ -4097,17 +4103,32 @@ function renderHistoryTable(container) {
     ${headerCell('date', 'Date')}${headerCell('filename', 'Filename')}${headerCell('host', 'Host')}${headerCell('link', 'Link')}
   </tr></thead><tbody>`;
 
-  rows.forEach(row => {
+  const parts = [html];
-    html += `<tr class="history-row${row.isError ? ' error' : ''}" data-link="${escapeAttr(row.link)}">
+  const len = rows.length;
-      <td class="col-date">${escapeHtml(row.date)}</td>
+  for (let i = 0; i < len; i++) {
-      <td class="col-filename">${escapeHtml(row.filename)}</td>
+    const row = rows[i];
-      <td class="col-host">${escapeHtml(row.host)}</td>
+    const link = row.link || '';
-      <td class="col-link">${escapeHtml(row.link)}</td>
+    const date = escapeHtml(row.date);
-    </tr>`;
+    const filename = escapeHtml(row.filename);
-  });
+    const host = escapeHtml(row.host);
-
+    const linkHtml = escapeHtml(link);
-  html += '</tbody></table>';
+    const linkAttr = escapeAttr(link);
-  container.innerHTML = html;
+    parts.push('<tr class="history-row');
+    if (row.isError) parts.push(' error');
+    parts.push('" data-link="');
+    parts.push(linkAttr);
+    parts.push('"><td class="col-date">');
+    parts.push(date);
+    parts.push('</td><td class="col-filename">');
+    parts.push(filename);
+    parts.push('</td><td class="col-host">');
+    parts.push(host);
+    parts.push('</td><td class="col-link">');
+    parts.push(linkHtml);
+    parts.push('</td></tr>');
+  }
+  parts.push('</tbody></table>');
+  container.innerHTML = parts.join('');
 
   // Delegated listeners: bind once per render-target instead of once per
   // row/header. With a 5000-row history the per-row bind path was a

tests/upload-manager.test.js

@@ -31,6 +31,7 @@ describe('UploadManager', () => {
     const origRequire = module.constructor.prototype.require;
     const hosters = require('../lib/hosters');
     hosters.uploadFile = mockUploadFile;
+    hosters.prefetchBaseline = async () => null;
 
     // Mock fs.statSync for test file paths
     const fs = require('fs');