feat: enforce automatic queue admission

2026-08-26 10:46:28 +02:00
parent 10bf08ea55
commit cfe7395104
2 changed files with 684 additions and 71 deletions
@@ -64,6 +64,16 @@ test('Windows compositor paints the full hidden surface with an RDP session envi
 const { contextBridge } = require('electron');
 const managedOnlineBackupProbeCalls = [];
 const folderMonitorProbeCalls = [];
+let automationProbe = {
+  history: [],
+  uploadLog: [],
+  paused: false,
+  dryScan: { files: [], reachable: true, trigger: 'test' },
+  readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
+  mutationCalls: [],
+  logs: [],
+  savedSettings: []
+};
 const managedOnlineBackupIds = {
  a: 'AAAAAAAAAAAAAAAAAAAAAA',
  b: 'AQEBAQEBAQEBAQEBAQEBAQ',
@@ -144,14 +154,80 @@ contextBridge.exposeInMainWorld('api', {
    pending.resolve({ ok: true, removedId: pending.id, notFound: false });
  },
  getManagedOnlineBackupProbeCalls() { return managedOnlineBackupProbeCalls; },
-  debugLog() {},
+  configureAutomationProbe(value = {}) {
+    automationProbe = {
+      history: Array.isArray(value.history) ? value.history : [],
+      uploadLog: Array.isArray(value.uploadLog) ? value.uploadLog : [],
+      paused: value.paused === true,
+      dryScan: value.dryScan || { files: [], reachable: true, trigger: 'test' },
+      readCalls: { history: 0, uploadLog: 0, inspect: 0, status: 0, testScan: 0, reconcile: 0 },
+      mutationCalls: [],
+      logs: [],
+      savedSettings: []
+    };
+  },
+  getAutomationProbeState() {
+    return {
+      readCalls: { ...automationProbe.readCalls },
+      mutationCalls: automationProbe.mutationCalls.map(value => [...value]),
+      logs: [...automationProbe.logs],
+      savedSettings: automationProbe.savedSettings.map(value => JSON.parse(JSON.stringify(value)))
+    };
+  },
+  inspectImportFiles(entries) {
+    automationProbe.readCalls.inspect++;
+    const candidates = Array.isArray(entries) ? entries : [];
+    const unavailable = candidates.filter(entry => entry?.unavailable).map(entry => ({ ...entry, reason: 'unreadable' }));
+    const accepted = candidates.filter(entry => !entry?.unavailable).map(entry => ({ ...entry }));
+    return Promise.resolve({
+      candidateCount: candidates.length,
+      duplicateCount: 0,
+      unavailableCount: unavailable.length,
+      acceptedCount: accepted.length,
+      accepted,
+      duplicates: [],
+      unavailable
+    });
+  },
+  getHistory() {
+    automationProbe.readCalls.history++;
+    return Promise.resolve(automationProbe.history);
+  },
+  readOwnUploadLog() {
+    automationProbe.readCalls.uploadLog++;
+    return Promise.resolve(automationProbe.uploadLog);
+  },
+  automationGetStatus() {
+    automationProbe.readCalls.status++;
+    return Promise.resolve({ paused: automationProbe.paused });
+  },
+  folderMonitorTestScan() {
+    automationProbe.readCalls.testScan++;
+    return Promise.resolve(automationProbe.dryScan);
+  },
+  folderMonitorReconcile() {
+    automationProbe.readCalls.reconcile++;
+    return Promise.resolve(automationProbe.dryScan);
+  },
+  debugLog(value) { automationProbe.logs.push(String(value)); },
+  saveGlobalSettings(value) {
+    automationProbe.savedSettings.push(value);
+    automationProbe.mutationCalls.push(['settings']);
+    return Promise.resolve(true);
+  },
  savePendingQueue(payload) {
    folderMonitorProbeCalls.push(['save', payload?.queueJobs?.length || 0]);
+    automationProbe.mutationCalls.push(['save', payload?.queueJobs?.length || 0]);
    return Promise.resolve(true);
  },
  addJobsToBatch(payload) {
    folderMonitorProbeCalls.push(['inject', payload?.jobs?.length || 0]);
-    return Promise.resolve({});
+    automationProbe.mutationCalls.push(['inject', payload?.jobs?.length || 0]);
+    return Promise.resolve({ added: payload?.jobs?.length || 0 });
+  },
+  startUpload(payload) {
+    automationProbe.mutationCalls.push(['start', payload?.jobs?.length || 0]);
+    return Promise.resolve({ started: true });
  },
  getFolderMonitorProbeCalls() { return folderMonitorProbeCalls; }
 });
@@ -310,14 +386,14 @@ contextBridge.exposeInMainWorld('api', {
      rebuildJobIndex();
    };
    resetQueue(false);
-    handleFolderMonitorFiles(['C:\\\\folder-monitor-queue-only.mkv']);
+    await handleFolderMonitorFiles(['C:\\\\folder-monitor-queue-only.mkv']);
    await new Promise(resolve => setTimeout(resolve, 0));
    const queueOnly = {
      statuses: queueJobs.map(job => job.status),
      injectCalls: window.api.getFolderMonitorProbeCalls().filter(call => call[0] === 'inject').length
    };
    resetQueue(true);
-    handleFolderMonitorFiles(['C:\\\\folder-monitor-inject.mkv']);
+    await handleFolderMonitorFiles(['C:\\\\folder-monitor-inject.mkv']);
    await new Promise(resolve => setTimeout(resolve, 0));
    const autoStart = {
      statuses: queueJobs.map(job => job.status),
@@ -371,6 +447,212 @@ contextBridge.exposeInMainWorld('api', {
    const sameBasenameResult = Object.fromEntries(queueJobs.map(job => [job.id, { status: job.status, code: job.result?.file_code || null }]));
    return { germanTooltips, englishTooltips, gridScope, actions, queueOnly, autoStart, manualQueueOnly, manualAutoStartRunning, manualAutoStartIdle, sameBasenameResult };
  })()`;
+  const automationPipelineScript = `(async () => {
+    const clone = value => JSON.parse(JSON.stringify(value));
+    const captureMutationFingerprint = async () => {
+      const api = await window.api.getAutomationProbeState();
+      return {
+        queueJobs: clone(queueJobs),
+        selectedFiles: clone(selectedFiles),
+        counters: { sessionDone: _sessionDoneCount, sessionError: _sessionErrorCount },
+        pendingFiles: clone(_pendingFiles),
+        pendingInspection: clone(_pendingImportInspection),
+        pendingInspections: _pendingImportInspections,
+        pendingAutoStart: clone([..._pendingFolderMonitorAutoStart]),
+        config: clone(config),
+        monitorSettings: clone(config.globalSettings?.folderMonitor || {}),
+        api: { mutationCalls: api.mutationCalls, logs: api.logs, savedSettings: api.savedSettings }
+      };
+    };
+    const hosters = ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx'];
+    const candidates = Array.from({ length: 500 }, (_, index) => ({
+      path: 'C:\\\\watch\\\\candidate-' + String(index).padStart(3, '0') + '.mkv',
+      name: 'candidate-' + String(index).padStart(3, '0') + '.mkv',
+      size: index >= 55 && index < 75 ? 2 * 1024 * 1024 : 512 * 1024,
+      mtimeMs: index,
+      filterMatched: index < 430,
+      unavailable: index >= 50 && index < 55
+    }));
+    const history = [{
+      id: 'history',
+      files: candidates.slice(0, 25).map(file => ({ path: file.path, name: file.name, results: [{ hoster: hosters[0], status: 'done' }] }))
+    }];
+    const uploadLog = candidates.slice(25, 50).map(file => ({ fileName: file.name, hoster: hosters[0] }));
+    config = {
+      hosters: Object.fromEntries(HOSTERS.map(hoster => [hoster, []])),
+      hosterSettings: {},
+      globalSettings: {
+        folderMonitor: {
+          enabled: true,
+          folderPath: 'C:\\\\watch',
+          hosters,
+          autoStart: false,
+          queueLimitJobs: 1500,
+          paused: false,
+          telemetry: { dateKey: '2026-08-26', detected: 7, queued: 3, skipped: 2, deferred: 1 }
+        }
+      }
+    };
+    hosterSettings = { 'doodstream.com': { maxSizeMb: 1 } };
+    selectedUploadHosters = [];
+    selectedFiles = [{ path: 'C:\\\\manual\\\\selected.mkv', name: 'selected.mkv', size: 1 }];
+    queueJobs = Array.from({ length: 300 }, (_, index) => ({
+      id: 'existing-' + index,
+      file: 'C:\\\\queue\\\\existing-' + index + '.mkv',
+      fileName: 'existing-' + index + '.mkv',
+      hoster: hosters[index % hosters.length],
+      status: 'queued',
+      bytesTotal: 1
+    }));
+    _sessionDoneCount = 4;
+    _sessionErrorCount = 5;
+    _pendingFiles = [{ path: 'C:\\\\pending\\\\pending.mkv', name: 'pending.mkv', size: 1 }];
+    _pendingImportInspection = { candidateCount: 1, accepted: clone(_pendingFiles) };
+    _pendingImportInspections = 1;
+    _pendingFolderMonitorAutoStart.clear();
+    _pendingFolderMonitorAutoStart.set('C:\\\\pending\\\\pending.mkv', true);
+    rebuildJobIndex();
+    window.api.configureAutomationProbe({ history, uploadLog, paused: false });
+    const before = await captureMutationFingerprint();
+    const preview = await evaluateAutomationCandidates(candidates, { dryRun: true, trigger: 'test' });
+    const after = await captureMutationFingerprint();
+    const dryReads = (await window.api.getAutomationProbeState()).readCalls;
+    const dry = {
+      fingerprintEqual: JSON.stringify(after) === JSON.stringify(before),
+      summary: preview.summary,
+      frozen: Object.isFrozen(preview) && Object.isFrozen(preview.summary) && Object.isFrozen(preview.admittedFiles) && Object.isFrozen(preview.deferredFiles),
+      reads: dryReads
+    };
+    window.api.configureAutomationProbe({
+      dryScan: { files: [candidates[55]], reachable: true, trigger: 'test' },
+      paused: false
+    });
+    const manualTestBefore = await captureMutationFingerprint();
+    const manualTestPreview = await runFolderMonitorTestScan();
+    const manualTestAfter = await captureMutationFingerprint();
+    const manualTestProbe = await window.api.getAutomationProbeState();
+    const manualTest = {
+      fingerprintEqual: JSON.stringify(manualTestAfter) === JSON.stringify(manualTestBefore),
+      summary: manualTestPreview.summary,
+      reads: manualTestProbe.readCalls
+    };
+
+    const configureAtomicState = currentCount => {
+      config.globalSettings.folderMonitor = {
+        enabled: true,
+        folderPath: 'C:\\\\watch',
+        hosters,
+        autoStart: false,
+        queueLimitJobs: 15000,
+        paused: false,
+        telemetry: { dateKey: new Date().toLocaleDateString('en-CA'), detected: 0, queued: 0, skipped: 0, deferred: 0 }
+      };
+      hosterSettings = {
+        'doodstream.com': { maxSizeMb: 2 },
+        'voe.sx': { maxSizeMb: 2 }
+      };
+      selectedUploadHosters = [];
+      selectedFiles = [{ path: 'C:\\\\manual\\\\unplanned.mkv', name: 'unplanned.mkv', size: 1 }];
+      queueJobs = Array.from({ length: currentCount }, (_, index) => ({
+        id: 'capacity-' + index,
+        file: 'C:\\\\capacity\\\\existing-' + index + '.mkv',
+        fileName: 'existing-' + index + '.mkv',
+        hoster: hosters[index % hosters.length],
+        status: 'queued',
+        bytesTotal: 1
+      }));
+      _pendingFiles = [];
+      _pendingImportInspection = null;
+      _pendingImportInspections = 0;
+      _pendingFolderMonitorAutoStart.clear();
+      uploading = false;
+      rebuildJobIndex();
+      window.api.configureAutomationProbe({ paused: false });
+    };
+    const atomicCandidates = [
+      { path: 'C:\\\\watch\\\\a.mkv', name: 'a.mkv', size: 1024 * 1024, mtimeMs: 1, filterMatched: true },
+      { path: 'C:\\\\watch\\\\b.mkv', name: 'b.mkv', size: 3 * 1024 * 1024, mtimeMs: 2, filterMatched: true }
+    ];
+    configureAtomicState(14998);
+    const atomicEvaluation = await evaluateAutomationCandidates(atomicCandidates, { dryRun: false, trigger: 'watcher' });
+    const atomicResult = await applyAutomationEvaluation(atomicEvaluation);
+    const atomic = {
+      newQueueFiles: [...new Set(queueJobs.filter(job => job.file === atomicCandidates[0].path || job.file === atomicCandidates[1].path).map(job => job.fileName))],
+      admittedFiles: atomicResult.admittedFiles.map(file => file.name),
+      deferred: config.globalSettings.folderMonitor.telemetry.deferred,
+      queued: config.globalSettings.folderMonitor.telemetry.queued,
+      currentJobCount: window.AutomationControl.countAutomaticQueueJobs(queueJobs),
+      unplannedJobs: queueJobs.filter(job => job.fileName === 'unplanned.mkv').length
+    };
+    const statusSnapshot = createAutomationStatusSnapshot();
+    const status = {
+      state: statusSnapshot.state,
+      currentJobCount: statusSnapshot.currentJobCount,
+      availableSlots: statusSnapshot.availableSlots,
+      queueLimited: statusSnapshot.queueLimited,
+      frozen: Object.isFrozen(statusSnapshot) && Object.isFrozen(statusSnapshot.telemetry)
+    };
+
+    configureAtomicState(14994);
+    const staleEvaluation = await evaluateAutomationCandidates(atomicCandidates, { dryRun: false, trigger: 'watcher' });
+    queueJobs.push(...Array.from({ length: 4 }, (_, index) => ({
+      id: 'stale-' + index,
+      file: 'C:\\\\capacity\\\\stale-' + index + '.mkv',
+      fileName: 'stale-' + index + '.mkv',
+      hoster: hosters[index],
+      status: 'queued',
+      bytesTotal: 1
+    })));
+    rebuildJobIndex();
+    const staleResult = await applyAutomationEvaluation(staleEvaluation);
+    const stale = {
+      plannedBeforeApply: staleEvaluation.admittedFiles.map(file => file.name),
+      admittedAfterApply: staleResult.admittedFiles.map(file => file.name),
+      newQueueFiles: [...new Set(queueJobs.filter(job => job.file === atomicCandidates[0].path || job.file === atomicCandidates[1].path).map(job => job.fileName))]
+    };
+
+    configureAtomicState(0);
+    config.globalSettings.folderMonitor.paused = true;
+    window.api.configureAutomationProbe({ paused: true });
+    const pausedJob = {
+      id: 'paused-preview',
+      file: 'C:\\\\manual\\\\paused-preview.mkv',
+      fileName: 'paused-preview.mkv',
+      hoster: hosters[0],
+      status: 'preview',
+      bytesTotal: 1
+    };
+    queueJobs = [pausedJob];
+    selectedFiles = [{ path: pausedJob.file, name: pausedJob.fileName, size: 1 }];
+    selectedUploadHosters = [hosters[0]];
+    rebuildJobIndex();
+    await startUpload();
+    uploading = true;
+    await startSelectedUpload([pausedJob]);
+    uploading = false;
+    const pausedAutomaticEvaluation = await evaluateAutomationCandidates([
+      { path: 'C:\\\\watch\\\\paused-auto.mkv', name: 'paused-auto.mkv', size: 1, mtimeMs: 1, filterMatched: true }
+    ], { dryRun: false, trigger: 'watcher' });
+    const pausedAutomaticResult = await applyAutomationEvaluation(pausedAutomaticEvaluation);
+    await coordinateImportEntries([
+      { path: 'C:\\\\manual\\\\allowed-preview.mkv', name: 'allowed-preview.mkv', size: 1 }
+    ]);
+    const input = document.createElement('input');
+    input.type = 'checkbox';
+    input.dataset.hosterModal = hosters[0];
+    input.checked = true;
+    document.getElementById('hosterModalList').replaceChildren(input);
+    await applyHosterSelection();
+    const pausedProbe = await window.api.getAutomationProbeState();
+    const paused = {
+      uploading,
+      statuses: Object.fromEntries(queueJobs.map(job => [job.fileName, job.status])),
+      automaticApplied: pausedAutomaticResult.admittedFiles.length,
+      startCalls: pausedProbe.mutationCalls.filter(call => call[0] === 'start').length,
+      injectCalls: pausedProbe.mutationCalls.filter(call => call[0] === 'inject').length
+    };
+    return { dry, manualTest, atomic, status, stale, paused };
+  })()`;
  const onlineBackupBehaviorScript = `(async () => {
    const ids = {
      a: 'AAAAAAAAAAAAAAAAAAAAAA',
@@ -577,6 +859,7 @@ app.whenReady().then(async () => {
  const onlineBackupBehavior = await window.webContents.executeJavaScript(${JSON.stringify(onlineBackupBehaviorScript)});
  const settingsSearchBehavior = await window.webContents.executeJavaScript(${JSON.stringify(settingsSearchBehaviorScript)});
  const folderMonitorBehavior = await window.webContents.executeJavaScript(${JSON.stringify(folderMonitorBehaviorScript)});
+  const automationPipeline = await window.webContents.executeJavaScript(${JSON.stringify(automationPipelineScript)});
  const onlineBackupLayout = await window.webContents.executeJavaScript(${JSON.stringify(onlineBackupLayoutScript)});
  window.setContentSize(760, Math.min(900, display.workAreaSize.height));
  await new Promise(resolve => setTimeout(resolve, 50));
@@ -595,6 +878,7 @@ app.whenReady().then(async () => {
    appDialogBehavior,
    settingsSearchBehavior,
    folderMonitorBehavior,
+    automationPipeline,
    onlineBackupBehavior,
    onlineBackupLayout,
    onlineBackupNarrowLayout
@@ -734,6 +1018,69 @@ app.whenReady().then(async () => {
        'waiting-same-name': { status: 'preview', code: null }
      }
    });
+    assert.deepEqual(result.automationPipeline.dry, {
+      fingerprintEqual: true,
+      summary: {
+        found: 500,
+        filterMatched: 430,
+        alreadyProcessed: 50,
+        unavailable: 5,
+        sizeLimitedJobs: 20,
+        acceptedFiles: 375,
+        selectedTargets: 4,
+        resultingJobs: 1480,
+        availableSlots: 1200,
+        deferredFiles: 70
+      },
+      frozen: true,
+      reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 0, reconcile: 0 }
+    });
+    assert.deepEqual(result.automationPipeline.manualTest, {
+      fingerprintEqual: true,
+      summary: {
+        found: 1,
+        filterMatched: 1,
+        alreadyProcessed: 0,
+        unavailable: 0,
+        sizeLimitedJobs: 1,
+        acceptedFiles: 1,
+        selectedTargets: 4,
+        resultingJobs: 3,
+        availableSlots: 1200,
+        deferredFiles: 0
+      },
+      reads: { history: 1, uploadLog: 1, inspect: 1, status: 0, testScan: 1, reconcile: 0 }
+    });
+    assert.deepEqual(result.automationPipeline.atomic, {
+      newQueueFiles: ['b.mkv'],
+      admittedFiles: ['b.mkv'],
+      deferred: 1,
+      queued: 1,
+      currentJobCount: 15000,
+      unplannedJobs: 0
+    });
+    assert.deepEqual(result.automationPipeline.status, {
+      state: 'queue-limited',
+      currentJobCount: 15000,
+      availableSlots: 0,
+      queueLimited: true,
+      frozen: true
+    });
+    assert.deepEqual(result.automationPipeline.stale, {
+      plannedBeforeApply: ['a.mkv', 'b.mkv'],
+      admittedAfterApply: ['b.mkv'],
+      newQueueFiles: ['b.mkv']
+    });
+    assert.deepEqual(result.automationPipeline.paused, {
+      uploading: false,
+      statuses: {
+        'paused-preview.mkv': 'preview',
+        'allowed-preview.mkv': 'preview'
+      },
+      automaticApplied: 0,
+      startCalls: 0,
+      injectCalls: 0
+    });
    assert.deepEqual(result.onlineBackupBehavior.initialKeys, ['MHU2-ZYXW…9876', 'MHU2-ABCD…1234']);
    assert.deepEqual(result.onlineBackupBehavior.initialWarning, {
      hidden: false,