Administrator/Multi-Hoster-Upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 204 Wiki Activity

feat(diagnostics): file-format probe + structured upload-start/failure rot-log

Browse Source
This commit is contained in:
Administrator 2026-06-07 18:49:54 +02:00
parent c44dde5396
commit 5fb313273d
4 changed files with 212 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
lib/file-probe.js Normal file
View File

@ -0,0 +1,73 @@
const fs = require('fs');
const SIGNATURES = [
{ kind: 'mp4-iso', test: (b) => b.length >= 12 && b.slice(4, 8).toString('ascii') === 'ftyp' },
{ kind: 'matroska', test: (b) => b.length >= 4 && b[0] === 0x1A && b[1] === 0x45 && b[2] === 0xDF && b[3] === 0xA3 },
{ kind: 'avi', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'AVI ' },
{ kind: 'wav', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'WAVE' },
{ kind: 'flv', test: (b) => b.length >= 3 && b.slice(0, 3).toString('ascii') === 'FLV' },
{ kind: 'asf-wmv', test: (b) => b.length >= 4 && b[0] === 0x30 && b[1] === 0x26 && b[2] === 0xB2 && b[3] === 0x75 },
{ kind: 'mpeg-ps', test: (b) => b.length >= 4 && b[0] === 0x00 && b[1] === 0x00 && b[2] === 0x01 && (b[3] === 0xBA || b[3] === 0xB3) },
{ kind: 'mpeg-ts', test: (b) => b.length >= 1 && b[0] === 0x47 },
{ kind: 'mp3', test: (b) => b.length >= 3 && (b.slice(0, 3).toString('ascii') === 'ID3' || (b[0] === 0xFF && (b[1] & 0xE0) === 0xE0)) },
{ kind: 'ogg', test: (b) => b.length >= 4 && b.slice(0, 4).toString('ascii') === 'OggS' },
{ kind: 'jpeg', test: (b) => b.length >= 3 && b[0] === 0xFF && b[1] === 0xD8 && b[2] === 0xFF },
{ kind: 'png', test: (b) => b.length >= 8 && b[0] === 0x89 && b.slice(1, 4).toString('ascii') === 'PNG' },
{ kind: 'pdf', test: (b) => b.length >= 5 && b.slice(0, 5).toString('ascii') === '%PDF-' },
{ kind: 'zip', test: (b) => b.length >= 4 && b[0] === 0x50 && b[1] === 0x4B && (b[2] === 0x03 || b[2] === 0x05 || b[2] === 0x07) },
{ kind: 'html', test: (b) => {
const s = b.toString('ascii', 0, Math.min(b.length, 64)).trimStart().toLowerCase();
return s.startsWith('<!doctype html') || s.startsWith('<html');
} }
];
const VIDEO_KINDS = new Set(['mp4-iso', 'matroska', 'avi', 'flv', 'asf-wmv', 'mpeg-ps', 'mpeg-ts']);
function detectKind(buf) {
if (!buf || buf.length === 0) return 'empty';
for (const sig of SIGNATURES) {
try { if (sig.test(buf)) return sig.kind; } catch { /* ignore malformed buffer slice */ }
}
return 'unknown';
}
function isVideoLikeKind(kind) {
return VIDEO_KINDS.has(kind);
}
function probeFileHead(filePath, bytes) {
const want = Number.isFinite(bytes) && bytes > 0 ? bytes : 64;
return new Promise((resolve) => {
fs.open(filePath, 'r', (err, fd) => {
if (err) return resolve({ ok: false, error: err.message, kind: 'unreadable' });
const buf = Buffer.alloc(want);
fs.read(fd, buf, 0, want, 0, (rerr, bytesRead) => {
fs.close(fd, () => {});
if (rerr) return resolve({ ok: false, error: rerr.message, kind: 'unreadable' });
const slice = buf.slice(0, bytesRead);
resolve({
ok: true,
bytesRead,
kind: detectKind(slice),
isVideoLike: isVideoLikeKind(detectKind(slice)),
headHex: slice.toString('hex')
});
});
});
});
}
function summarizeFileStat(filePath) {
try {
const st = fs.statSync(filePath);
return {
size: st.size,
mtime: st.mtime.toISOString(),
isFile: st.isFile()
};
} catch (err) {
return { error: err.message };
}
}
module.exports = { detectKind, isVideoLikeKind, probeFileHead, summarizeFileStat, VIDEO_KINDS, SIGNATURES };

11
lib/hosters.js
View File

@ -580,6 +580,17 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
try { try {
result = config.parseResult(payload); result = config.parseResult(payload);
} catch (err) { } catch (err) {
if (err && typeof err === 'object' && !err.diagnostic) {
try {
err.diagnostic = {
hoster: hosterName,
http: statusCode,
contentType: (headers && headers['content-type']) || null,
payloadSnippet: JSON.stringify(payload).slice(0, 1000),
uploadUrl: targetUrl
};
} catch { /* JSON cycle — skip diagnostic */ }
}
parseErr = err; parseErr = err;
} }
if (result && (result.file_code || result.download_url || result.embed_url)) { if (result && (result.file_code || result.download_url || result.embed_url)) {

28
lib/upload-manager.js
View File

@ -9,6 +9,7 @@ const DoodstreamUploader = require('./doodstream-upload');
const ClouddropUploader = require('./clouddrop-upload'); const ClouddropUploader = require('./clouddrop-upload');
const Semaphore = require('./semaphore'); const Semaphore = require('./semaphore');
const Throttle = require('./throttle'); const Throttle = require('./throttle');
const { probeFileHead, summarizeFileStat } = require('./file-probe');
const DEFAULT_SETTINGS = { const DEFAULT_SETTINGS = {
retries: 3, retries: 3,
@ -424,6 +425,16 @@ class UploadManager extends EventEmitter {
maxAttempts maxAttempts
}); });
const fileStat = summarizeFileStat(task.file);
const fileProbe = await probeFileHead(task.file, 64).catch((err) => ({ ok: false, error: err.message, kind: 'unreadable' }));
this._rotLog('upload-start', {
jobId, hoster: task.hoster, accountId: task.accountId, fileName,
fileSize: fileStat.size, fileMtime: fileStat.mtime,
detectedKind: fileProbe && fileProbe.kind ? fileProbe.kind : 'unknown',
isVideoLike: !!(fileProbe && fileProbe.isVideoLike),
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
});
// Acquire hoster semaphore first so jobs waiting for a hoster slot // Acquire hoster semaphore first so jobs waiting for a hoster slot
// don't waste global slots (prevents underutilization) // don't waste global slots (prevents underutilization)
await hosterSemaphore.acquire(signal); await hosterSemaphore.acquire(signal);
@ -596,6 +607,23 @@ class UploadManager extends EventEmitter {
this.activeJobs.delete(uploadId); this.activeJobs.delete(uploadId);
const isSpeedRestart = speedAbort && speedAbort.signal.aborted && !signal.aborted; const isSpeedRestart = speedAbort && speedAbort.signal.aborted && !signal.aborted;
if (!signal.aborted && !isSpeedRestart) {
const diag = (err && typeof err === 'object' && err.diagnostic) || {};
this._rotLog('upload-failure', {
jobId, hoster: task.hoster, accountId: task.accountId, fileName,
attempt,
error: err && err.message ? err.message : String(err),
fileRejected: !!(err && err.fileRejected),
accountError: !!(err && err.accountError),
hosterTransient: !!(err && err.hosterTransient),
http: diag.http || null,
contentType: diag.contentType || null,
detectedKind: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.kind) ? fileProbe.kind : null,
isVideoLike: !!(typeof fileProbe !== 'undefined' && fileProbe && fileProbe.isVideoLike),
headHex: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.headHex) ? fileProbe.headHex.slice(0, 32) : null,
payloadSnippet: diag.payloadSnippet || null
});
}
if (signal.aborted) { if (signal.aborted) {
lastError = new Error('Abgebrochen'); lastError = new Error('Abgebrochen');
break; break;

100
tests/file-probe.test.js Normal file
View File

@ -0,0 +1,100 @@
const test = require('node:test');
const assert = require('node:assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const { detectKind, isVideoLikeKind, probeFileHead, summarizeFileStat } = require('../lib/file-probe');
function tmpWrite(name, buf) {
const p = path.join(os.tmpdir(), `mhu-probe-${Date.now()}-${name}`);
fs.writeFileSync(p, buf);
return p;
}
test('detectKind recognizes ISO-MP4 (ftyp box at offset 4)', () => {
const buf = Buffer.concat([Buffer.from([0x00, 0x00, 0x00, 0x20]), Buffer.from('ftypisom', 'ascii'), Buffer.alloc(8, 0)]);
assert.strictEqual(detectKind(buf), 'mp4-iso');
assert.strictEqual(isVideoLikeKind('mp4-iso'), true);
});
test('detectKind recognizes Matroska / WebM EBML header', () => {
const buf = Buffer.from([0x1A, 0x45, 0xDF, 0xA3, 0x01, 0x00]);
assert.strictEqual(detectKind(buf), 'matroska');
assert.strictEqual(isVideoLikeKind('matroska'), true);
});
test('detectKind recognizes AVI (RIFF...AVI )', () => {
const buf = Buffer.concat([Buffer.from('RIFF', 'ascii'), Buffer.from([0x00, 0x00, 0x00, 0x00]), Buffer.from('AVI ', 'ascii')]);
assert.strictEqual(detectKind(buf), 'avi');
});
test('detectKind recognizes FLV', () => {
const buf = Buffer.concat([Buffer.from('FLV', 'ascii'), Buffer.from([0x01])]);
assert.strictEqual(detectKind(buf), 'flv');
});
test('detectKind recognizes ASF (WMV)', () => {
const buf = Buffer.from([0x30, 0x26, 0xB2, 0x75, 0x00, 0x00]);
assert.strictEqual(detectKind(buf), 'asf-wmv');
});
test('detectKind recognizes MPEG-PS (00 00 01 BA)', () => {
const buf = Buffer.from([0x00, 0x00, 0x01, 0xBA, 0x00]);
assert.strictEqual(detectKind(buf), 'mpeg-ps');
});
test('detectKind recognizes JPEG (non-video)', () => {
const buf = Buffer.from([0xFF, 0xD8, 0xFF, 0xE0]);
assert.strictEqual(detectKind(buf), 'jpeg');
assert.strictEqual(isVideoLikeKind('jpeg'), false);
});
test('detectKind recognizes HTML response (non-video)', () => {
const buf = Buffer.from('<!DOCTYPE html><html><head>', 'ascii');
assert.strictEqual(detectKind(buf), 'html');
assert.strictEqual(isVideoLikeKind('html'), false);
});
test('detectKind returns empty for zero-length and unknown for noise', () => {
assert.strictEqual(detectKind(Buffer.alloc(0)), 'empty');
assert.strictEqual(detectKind(Buffer.from([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF])), 'unknown');
});
test('probeFileHead reads first bytes and returns hex + kind for an MP4-like file', async () => {
const mp4Head = Buffer.concat([Buffer.from([0x00, 0x00, 0x00, 0x20]), Buffer.from('ftypisom', 'ascii'), Buffer.alloc(16, 0xAA)]);
const p = tmpWrite('fake.mp4', mp4Head);
try {
const res = await probeFileHead(p, 64);
assert.strictEqual(res.ok, true);
assert.strictEqual(res.kind, 'mp4-iso');
assert.strictEqual(res.isVideoLike, true);
assert.ok(res.headHex.startsWith('0000002066747970'));
assert.strictEqual(res.bytesRead, mp4Head.length);
} finally {
fs.unlinkSync(p);
}
});
test('probeFileHead returns ok:false with kind=unreadable for missing file', async () => {
const res = await probeFileHead(path.join(os.tmpdir(), `does-not-exist-${Date.now()}.mp4`), 32);
assert.strictEqual(res.ok, false);
assert.strictEqual(res.kind, 'unreadable');
assert.ok(res.error);
});
test('summarizeFileStat returns size + mtime for a real file', () => {
const p = tmpWrite('stat.bin', Buffer.alloc(123, 0xCC));
try {
const stat = summarizeFileStat(p);
assert.strictEqual(stat.size, 123);
assert.strictEqual(stat.isFile, true);
assert.ok(stat.mtime);
} finally {
fs.unlinkSync(p);
}
});
test('summarizeFileStat returns error for missing file', () => {
const stat = summarizeFileStat(path.join(os.tmpdir(), `does-not-exist-${Date.now()}.bin`));
assert.ok(stat.error);
});