fix: harden upload cancellation and audit logging
Preserve pre-start and batch cancellation requests, reject late upload success after cancellation, and wait for cancellation acknowledgements before removing queue entries. Separate formatted link logs from privacy-safe source cleanup and upload plan audits, persist audit fallback paths, redact support bundles, and expose audit diagnostics safely. Improve queue selection and destructive-action clarity, show the Settings save action only while changes are pending, and add regression coverage for all updated behavior.
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { getRotatedLogPath } = require('./log-rotation');
|
||||
|
||||
const READABLE_LOGS = {
|
||||
debug: 'debug',
|
||||
fileuploader: 'fileuploader',
|
||||
uploadAudit: 'uploadAudit',
|
||||
accountRotation: 'accountRotation',
|
||||
crash: 'crashLog'
|
||||
};
|
||||
@@ -38,7 +40,7 @@ function createCollectors(deps) {
|
||||
const paths = getAllLogPaths();
|
||||
let p = paths[key];
|
||||
if (!p) return null;
|
||||
if (backup === 1 || backup === 2) p = `${p}.${backup}`;
|
||||
if (backup === 1 || backup === 2) p = getRotatedLogPath(p, backup);
|
||||
return p;
|
||||
}
|
||||
|
||||
@@ -67,15 +69,17 @@ function createCollectors(deps) {
|
||||
const paths = getAllLogPaths();
|
||||
const dir = paths.logDir;
|
||||
const files = [];
|
||||
const readableNames = new Set();
|
||||
for (const [name, key] of Object.entries(READABLE_LOGS)) {
|
||||
const base = paths[key];
|
||||
if (!base) continue;
|
||||
const variants = [];
|
||||
for (const suffix of ['', '.1', '.2']) {
|
||||
const fp = base + suffix;
|
||||
for (const backup of [0, 1, 2]) {
|
||||
const fp = backup === 0 ? base : getRotatedLogPath(base, backup);
|
||||
readableNames.add(path.basename(fp));
|
||||
try {
|
||||
const st = fs.statSync(fp);
|
||||
variants.push({ backup: suffix === '' ? 0 : Number(suffix.slice(1)), sizeBytes: st.size, mtime: st.mtime.toISOString() });
|
||||
variants.push({ backup, sizeBytes: st.size, mtime: st.mtime.toISOString() });
|
||||
} catch {}
|
||||
}
|
||||
files.push({ name, path: base, readable: true, present: variants.length > 0, variants });
|
||||
@@ -84,7 +88,7 @@ function createCollectors(deps) {
|
||||
try {
|
||||
siblings = fs.readdirSync(dir)
|
||||
.filter(f => /\.log(\.\d+)?$/i.test(f))
|
||||
.filter(f => !files.some(x => path.basename(x.path) === f || f.startsWith(path.basename(x.path))));
|
||||
.filter(f => !readableNames.has(f));
|
||||
siblings = siblings.map(f => {
|
||||
let size = 0, mtime = null;
|
||||
try { const st = fs.statSync(path.join(dir, f)); size = st.size; mtime = st.mtime.toISOString(); } catch {}
|
||||
|
||||
+15
-9
@@ -14,6 +14,14 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function getRotatedLogPath(filePath, backup) {
|
||||
const index = Number(backup);
|
||||
if (!Number.isInteger(index) || index < 1) return filePath;
|
||||
const ext = path.extname(filePath);
|
||||
const base = filePath.slice(0, filePath.length - ext.length);
|
||||
return `${base}.${index}${ext}`;
|
||||
}
|
||||
|
||||
function maybeRotateLogFile(filePath, maxBytes, maxBackups = 3, log = () => {}) {
|
||||
if (!filePath || !Number.isFinite(maxBytes) || maxBytes <= 0) return false;
|
||||
let size = 0;
|
||||
@@ -29,24 +37,22 @@ function maybeRotateLogFile(filePath, maxBytes, maxBackups = 3, log = () => {})
|
||||
}
|
||||
if (size <= maxBytes) return false;
|
||||
|
||||
const ext = path.extname(filePath);
|
||||
const base = filePath.slice(0, filePath.length - ext.length);
|
||||
|
||||
// Drop the oldest backup if it exists, then shift each numbered backup up
|
||||
// one slot. Errors are ignored: missing intermediate backups are normal,
|
||||
// failed renames just mean we'll rotate again next time.
|
||||
try { fs.unlinkSync(`${base}.${maxBackups}${ext}`); } catch {}
|
||||
try { fs.unlinkSync(getRotatedLogPath(filePath, maxBackups)); } catch {}
|
||||
for (let i = maxBackups - 1; i >= 1; i--) {
|
||||
try { fs.renameSync(`${base}.${i}${ext}`, `${base}.${i + 1}${ext}`); } catch {}
|
||||
try { fs.renameSync(getRotatedLogPath(filePath, i), getRotatedLogPath(filePath, i + 1)); } catch {}
|
||||
}
|
||||
try {
|
||||
fs.renameSync(filePath, `${base}.1${ext}`);
|
||||
log(`logRotation: rotated ${filePath} (${(size / 1024 / 1024).toFixed(1)} MB) → ${base}.1${ext}`);
|
||||
const backupPath = getRotatedLogPath(filePath, 1);
|
||||
fs.renameSync(filePath, backupPath);
|
||||
log(`logRotation: rotated ${filePath} (${(size / 1024 / 1024).toFixed(1)} MB) → ${backupPath}`);
|
||||
return true;
|
||||
} catch (err) {
|
||||
log(`logRotation: rename ${filePath} → ${base}.1${ext} failed: ${err.message}`);
|
||||
log(`logRotation: rename ${filePath} → ${getRotatedLogPath(filePath, 1)} failed: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { maybeRotateLogFile };
|
||||
module.exports = { getRotatedLogPath, maybeRotateLogFile };
|
||||
|
||||
+13
-7
@@ -42,6 +42,9 @@ function redactLogText(text, secrets) {
|
||||
}
|
||||
}
|
||||
out = out
|
||||
.replace(/("(?:file|fileName|stagedFile|sourceFile|targetFile|path|[A-Za-z0-9_]*Path)"\s*:\s*")[^"]*(")/gi, '$1<redacted-path>$2')
|
||||
.replace(/\b[A-Za-z]:(?:\\+|\/+)[^\r\n"'<>|]*?(?=\s+(?:trigger|error|outcome|hoster|attempt|status|code)=|\r?\n|$|["'])/gi, '<redacted-path>')
|
||||
.replace(/\\{2,}[A-Za-z0-9._$-]+\\+[^\r\n"'<>|]*?(?=\s+(?:trigger|error|outcome|hoster|attempt|status|code)=|\r?\n|$|["'])/g, '<redacted-path>')
|
||||
.replace(/https?:\/\/(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/[\w-]+/gi, 'https://discord.com/api/webhooks/' + REDACTED)
|
||||
.replace(/(\/\/[^\s/:@]+:)[^\s/@]+(@)/g, '$1' + REDACTED + '$2')
|
||||
.replace(/(authorization:\s*(?:bearer|basic)\s+)\S+/gi, '$1' + REDACTED)
|
||||
@@ -66,13 +69,15 @@ function valueScrub(value, secrets) {
|
||||
return JSON.parse(scrubbed);
|
||||
}
|
||||
|
||||
function collectFile(filePath, label, maxBytes) {
|
||||
function collectFile(filePath, label, maxBytes, options) {
|
||||
const includePath = !options || options.includePath !== false;
|
||||
if (!filePath) return `=== ${label} ===\n<no path configured>\n\n`;
|
||||
let stat;
|
||||
try { stat = fs.statSync(filePath); }
|
||||
catch (err) {
|
||||
if (err && err.code === 'ENOENT') return `=== ${label} (${filePath}) ===\n<file does not exist yet>\n\n`;
|
||||
return `=== ${label} (${filePath}) ===\n<stat error: ${err.message}>\n\n`;
|
||||
const context = includePath ? ` (${filePath})` : '';
|
||||
if (err && err.code === 'ENOENT') return `=== ${label}${context} ===\n<file does not exist yet>\n\n`;
|
||||
return `=== ${label}${context} ===\n<stat error: ${err.message}>\n\n`;
|
||||
}
|
||||
const cap = Number.isFinite(maxBytes) && maxBytes > 0 ? maxBytes : 5 * 1024 * 1024;
|
||||
let content;
|
||||
@@ -90,10 +95,11 @@ function collectFile(filePath, label, maxBytes) {
|
||||
} catch (err) {
|
||||
content = `<read error: ${err.message}>`;
|
||||
}
|
||||
return `=== ${label} (${filePath}, size=${stat.size} bytes) ===\n${content}\n\n`;
|
||||
const metadata = includePath ? `${filePath}, size=${stat.size} bytes` : `size=${stat.size} bytes`;
|
||||
return `=== ${label} (${metadata}) ===\n${content}\n\n`;
|
||||
}
|
||||
|
||||
function buildSupportBundleText({ header, sanitizedConfig, files }) {
|
||||
function buildSupportBundleText({ header, sanitizedConfig, files, secrets }) {
|
||||
const parts = [];
|
||||
parts.push('=== Multi-Hoster-Upload Support Bundle ===\n');
|
||||
if (header && typeof header === 'object') {
|
||||
@@ -101,10 +107,10 @@ function buildSupportBundleText({ header, sanitizedConfig, files }) {
|
||||
}
|
||||
parts.push('\n');
|
||||
parts.push('=== Config (sanitized — password/apiKey/token/cookie/sessionId redacted) ===\n');
|
||||
parts.push(JSON.stringify(sanitizedConfig, null, 2));
|
||||
parts.push(redactLogText(JSON.stringify(sanitizedConfig, null, 2), secrets));
|
||||
parts.push('\n\n');
|
||||
for (const f of (files || [])) {
|
||||
parts.push(collectFile(f.path, f.label || f.path, f.maxBytes));
|
||||
parts.push(redactLogText(collectFile(f.path, f.label || f.path, f.maxBytes, { includePath: false }), secrets));
|
||||
}
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
const nodePath = require('path');
|
||||
|
||||
function getUploadAuditLogPath(uploadLogPath, pathApi = nodePath) {
|
||||
if (typeof uploadLogPath !== 'string' || !uploadLogPath.trim()) return null;
|
||||
return pathApi.join(pathApi.dirname(uploadLogPath), 'upload-audit.log');
|
||||
}
|
||||
|
||||
function createUploadAuditWriter(options) {
|
||||
const source = options && typeof options === 'object' ? options : {};
|
||||
const fs = source.fs;
|
||||
const path = source.path || nodePath;
|
||||
const resolveUploadLogTarget = source.resolveUploadLogTarget;
|
||||
const rotateLogFile = typeof source.rotateLogFile === 'function' ? source.rotateLogFile : () => {};
|
||||
const invalidateUploadLogTarget = typeof source.invalidateUploadLogTarget === 'function' ? source.invalidateUploadLogTarget : () => {};
|
||||
const persistFallbackLogPath = typeof source.persistFallbackLogPath === 'function' ? source.persistFallbackLogPath : async () => {};
|
||||
const reportError = typeof source.reportError === 'function' ? source.reportError : () => {};
|
||||
const retryDelays = Array.isArray(source.retryDelays) && source.retryDelays.length > 0 ? source.retryDelays : [0, 100, 250];
|
||||
const maxBytes = Number.isFinite(source.maxBytes) ? source.maxBytes : 10 * 1024 * 1024;
|
||||
const maxBackups = Number.isFinite(source.maxBackups) ? source.maxBackups : 2;
|
||||
let activePath = null;
|
||||
|
||||
if (!fs || !fs.promises || typeof fs.promises.appendFile !== 'function' || typeof resolveUploadLogTarget !== 'function') {
|
||||
throw new TypeError('createUploadAuditWriter requires fs and resolveUploadLogTarget');
|
||||
}
|
||||
|
||||
async function append(line, label) {
|
||||
let excludedPath = null;
|
||||
for (const delay of retryDelays) {
|
||||
if (delay) await new Promise(resolve => setTimeout(resolve, delay));
|
||||
const uploadTarget = resolveUploadLogTarget(excludedPath);
|
||||
const targetPath = uploadTarget && getUploadAuditLogPath(uploadTarget.path, path);
|
||||
if (!targetPath) continue;
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
||||
rotateLogFile(targetPath, maxBytes, maxBackups);
|
||||
await fs.promises.appendFile(targetPath, line, 'utf-8');
|
||||
activePath = targetPath;
|
||||
if (uploadTarget.isFallback) {
|
||||
try {
|
||||
await persistFallbackLogPath(uploadTarget.path);
|
||||
} catch (error) {
|
||||
reportError('audit-fallback-persist', error);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
excludedPath = uploadTarget.path;
|
||||
invalidateUploadLogTarget();
|
||||
reportError(label, error);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return { append, getActivePath: () => activePath };
|
||||
}
|
||||
|
||||
module.exports = { getUploadAuditLogPath, createUploadAuditWriter };
|
||||
+46
-1
@@ -28,7 +28,52 @@
|
||||
return { hoster, fileName, ts };
|
||||
}
|
||||
|
||||
const api = { formatUploadLogLine, parseUploadLogLine };
|
||||
function summarizeBatchPlan(payload) {
|
||||
const source = payload && typeof payload === 'object' ? payload : {};
|
||||
const jobs = Array.isArray(source.jobs) ? source.jobs : [];
|
||||
if (jobs.length > 0) {
|
||||
const files = new Set();
|
||||
const destinations = new Set();
|
||||
let plannedUploadCount = 0;
|
||||
for (const job of jobs) {
|
||||
if (!job || typeof job !== 'object') continue;
|
||||
const file = typeof job.file === 'string' ? job.file.trim() : '';
|
||||
const hoster = typeof job.hoster === 'string' ? job.hoster.trim() : '';
|
||||
if (!file || !hoster) continue;
|
||||
files.add(file);
|
||||
destinations.add(hoster);
|
||||
plannedUploadCount++;
|
||||
}
|
||||
return {
|
||||
fileCount: files.size,
|
||||
destinationCount: destinations.size,
|
||||
plannedUploadCount
|
||||
};
|
||||
}
|
||||
|
||||
const files = new Set((Array.isArray(source.files) ? source.files : []).filter(value => typeof value === 'string' && value.trim()));
|
||||
const destinations = new Set((Array.isArray(source.hosters) ? source.hosters : []).filter(value => typeof value === 'string' && value.trim()));
|
||||
return {
|
||||
fileCount: files.size,
|
||||
destinationCount: destinations.size,
|
||||
plannedUploadCount: files.size * destinations.size
|
||||
};
|
||||
}
|
||||
|
||||
function formatUploadPlanLogLine(date, plan, mode) {
|
||||
const inputDate = date instanceof Date && !Number.isNaN(date.getTime()) ? date : new Date();
|
||||
const source = plan && typeof plan === 'object' ? plan : {};
|
||||
const count = value => Number.isFinite(Number(value)) ? Math.max(0, Math.floor(Number(value))) : 0;
|
||||
return `# UPLOAD-PLAN ${JSON.stringify({
|
||||
timestamp: inputDate.toISOString(),
|
||||
mode: mode === 'add' ? 'add' : 'start',
|
||||
fileCount: count(source.fileCount),
|
||||
destinationCount: count(source.destinationCount),
|
||||
plannedUploadCount: count(source.plannedUploadCount)
|
||||
})}\r\n`;
|
||||
}
|
||||
|
||||
const api = { formatUploadLogLine, parseUploadLogLine, summarizeBatchPlan, formatUploadPlanLogLine };
|
||||
if (typeof module !== 'undefined' && module.exports) module.exports = api;
|
||||
else if (root) root.UploadLog = api;
|
||||
})(typeof window !== 'undefined' ? window : this);
|
||||
|
||||
+31
-2
@@ -39,6 +39,8 @@ class UploadManager extends EventEmitter {
|
||||
this.activeJobs = new Map(); // uploadId -> { jobId, speedKbs, bytesUploaded, hoster }
|
||||
this.jobAbortControllers = new Map(); // jobId -> AbortController
|
||||
this.cancelledJobIds = new Set();
|
||||
this.pendingCancelledJobIds = new Set();
|
||||
this.pendingCancelAll = false;
|
||||
this.sessionBytes = 0;
|
||||
this._transientErrorTotal = 0;
|
||||
this.lastStartTime = {}; // hoster -> timestamp of last upload start
|
||||
@@ -330,14 +332,20 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
async startBatch(tasks, opts = {}) {
|
||||
const pendingCancelledJobIds = new Set(this.pendingCancelledJobIds);
|
||||
const pendingCancelAll = this.pendingCancelAll;
|
||||
this.pendingCancelledJobIds.clear();
|
||||
this.pendingCancelAll = false;
|
||||
this.running = true;
|
||||
this.stopAfterActive = false;
|
||||
this.stopAfterActive = pendingCancelAll;
|
||||
this.abortController = new AbortController();
|
||||
if (pendingCancelAll) this.abortController.abort();
|
||||
this.startTime = Date.now();
|
||||
this.sessionBytes = 0;
|
||||
this.activeJobs.clear();
|
||||
this.jobAbortControllers.clear();
|
||||
this.cancelledJobIds.clear();
|
||||
for (const jobId of pendingCancelledJobIds) this.cancelledJobIds.add(jobId);
|
||||
this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch
|
||||
this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance)
|
||||
this.semaphores = {};
|
||||
@@ -447,6 +455,7 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
const maxAttempts = Math.max(1, (settings.retries || 0) + 1);
|
||||
const jobAbortController = new AbortController();
|
||||
if (this.cancelledJobIds.has(jobId)) jobAbortController.abort();
|
||||
const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal);
|
||||
this.jobAbortControllers.set(jobId, jobAbortController);
|
||||
|
||||
@@ -500,6 +509,12 @@ class UploadManager extends EventEmitter {
|
||||
};
|
||||
|
||||
try {
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) {
|
||||
const error = 'Abgebrochen';
|
||||
emitFinalStatus('aborted', { error, attempt: 0 });
|
||||
recordFinalResult('aborted', { error });
|
||||
return;
|
||||
}
|
||||
if (fileNotFound) {
|
||||
const error = 'Datei nicht gefunden';
|
||||
emitFinalStatus('skipped', { error, attempt: 0 });
|
||||
@@ -714,6 +729,8 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe);
|
||||
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
|
||||
const elapsed = Math.round((Date.now() - jobStart) / 1000);
|
||||
this.sessionBytes += fileSize;
|
||||
this.activeJobs.delete(uploadId);
|
||||
@@ -847,6 +864,12 @@ class UploadManager extends EventEmitter {
|
||||
this._noteSuspectReject(task.hoster, task.accountId, fileSize);
|
||||
const alt = await this._trySuspectRejectAlternates(task, { uploadId, jobId, fileName, fileSize, settings, signal, fileProbe });
|
||||
if (alt) {
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) {
|
||||
const error = 'Abgebrochen';
|
||||
emitFinalStatus('aborted', { error });
|
||||
recordFinalResult('aborted', { error });
|
||||
return;
|
||||
}
|
||||
emitFinalStatus('done', { result: alt.result, speedKbs: alt.speedKbs, elapsed: alt.elapsed, attempt: 1 });
|
||||
recordFinalResult('done', { result: alt.result });
|
||||
return;
|
||||
@@ -1017,6 +1040,7 @@ class UploadManager extends EventEmitter {
|
||||
: hosterThrottle || globalThrottle;
|
||||
|
||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.sessionBytes += fileSize;
|
||||
emitFinalStatus('done', { result, speedKbs: currentSpeedKbs, elapsed: Math.round((Date.now() - jobStart) / 1000), attempt });
|
||||
@@ -1159,6 +1183,7 @@ class UploadManager extends EventEmitter {
|
||||
: hosterThrottle || globalThrottle;
|
||||
try {
|
||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.sessionBytes += fileSize;
|
||||
this._suspectGoodAccounts.set(task.hoster, account.id);
|
||||
@@ -1442,6 +1467,7 @@ class UploadManager extends EventEmitter {
|
||||
cancelJobs(jobIds) {
|
||||
for (const jobId of jobIds || []) {
|
||||
if (!jobId) continue;
|
||||
if (!this.running) this.pendingCancelledJobIds.add(jobId);
|
||||
this.cancelledJobIds.add(jobId);
|
||||
const controller = this.jobAbortControllers.get(jobId);
|
||||
if (controller && !controller.signal.aborted) {
|
||||
@@ -1455,7 +1481,10 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
|
||||
cancel() {
|
||||
if (!this.running) return;
|
||||
if (!this.running) {
|
||||
this.pendingCancelAll = true;
|
||||
return;
|
||||
}
|
||||
this.abortController.abort();
|
||||
this.stopAfterActive = true;
|
||||
for (const controller of this.jobAbortControllers.values()) {
|
||||
|
||||
Reference in New Issue
Block a user