diff --git a/lib/file-probe.js b/lib/file-probe.js new file mode 100644 index 0000000..4a94a50 --- /dev/null +++ b/lib/file-probe.js @@ -0,0 +1,73 @@ +const fs = require('fs'); + +const SIGNATURES = [ + { kind: 'mp4-iso', test: (b) => b.length >= 12 && b.slice(4, 8).toString('ascii') === 'ftyp' }, + { kind: 'matroska', test: (b) => b.length >= 4 && b[0] === 0x1A && b[1] === 0x45 && b[2] === 0xDF && b[3] === 0xA3 }, + { kind: 'avi', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'AVI ' }, + { kind: 'wav', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'WAVE' }, + { kind: 'flv', test: (b) => b.length >= 3 && b.slice(0, 3).toString('ascii') === 'FLV' }, + { kind: 'asf-wmv', test: (b) => b.length >= 4 && b[0] === 0x30 && b[1] === 0x26 && b[2] === 0xB2 && b[3] === 0x75 }, + { kind: 'mpeg-ps', test: (b) => b.length >= 4 && b[0] === 0x00 && b[1] === 0x00 && b[2] === 0x01 && (b[3] === 0xBA || b[3] === 0xB3) }, + { kind: 'mpeg-ts', test: (b) => b.length >= 1 && b[0] === 0x47 }, + { kind: 'mp3', test: (b) => b.length >= 3 && (b.slice(0, 3).toString('ascii') === 'ID3' || (b[0] === 0xFF && (b[1] & 0xE0) === 0xE0)) }, + { kind: 'ogg', test: (b) => b.length >= 4 && b.slice(0, 4).toString('ascii') === 'OggS' }, + { kind: 'jpeg', test: (b) => b.length >= 3 && b[0] === 0xFF && b[1] === 0xD8 && b[2] === 0xFF }, + { kind: 'png', test: (b) => b.length >= 8 && b[0] === 0x89 && b.slice(1, 4).toString('ascii') === 'PNG' }, + { kind: 'pdf', test: (b) => b.length >= 5 && b.slice(0, 5).toString('ascii') === '%PDF-' }, + { kind: 'zip', test: (b) => b.length >= 4 && b[0] === 0x50 && b[1] === 0x4B && (b[2] === 0x03 || b[2] === 0x05 || b[2] === 0x07) }, + { kind: 'html', test: (b) => { + const s = b.toString('ascii', 0, Math.min(b.length, 64)).trimStart().toLowerCase(); + return s.startsWith(' 0 ? bytes : 64; + return new Promise((resolve) => { + fs.open(filePath, 'r', (err, fd) => { + if (err) return resolve({ ok: false, error: err.message, kind: 'unreadable' }); + const buf = Buffer.alloc(want); + fs.read(fd, buf, 0, want, 0, (rerr, bytesRead) => { + fs.close(fd, () => {}); + if (rerr) return resolve({ ok: false, error: rerr.message, kind: 'unreadable' }); + const slice = buf.slice(0, bytesRead); + resolve({ + ok: true, + bytesRead, + kind: detectKind(slice), + isVideoLike: isVideoLikeKind(detectKind(slice)), + headHex: slice.toString('hex') + }); + }); + }); + }); +} + +function summarizeFileStat(filePath) { + try { + const st = fs.statSync(filePath); + return { + size: st.size, + mtime: st.mtime.toISOString(), + isFile: st.isFile() + }; + } catch (err) { + return { error: err.message }; + } +} + +module.exports = { detectKind, isVideoLikeKind, probeFileHead, summarizeFileStat, VIDEO_KINDS, SIGNATURES }; diff --git a/lib/hosters.js b/lib/hosters.js index 5dbdafe..fd577c2 100644 --- a/lib/hosters.js +++ b/lib/hosters.js @@ -580,6 +580,17 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro try { result = config.parseResult(payload); } catch (err) { + if (err && typeof err === 'object' && !err.diagnostic) { + try { + err.diagnostic = { + hoster: hosterName, + http: statusCode, + contentType: (headers && headers['content-type']) || null, + payloadSnippet: JSON.stringify(payload).slice(0, 1000), + uploadUrl: targetUrl + }; + } catch { /* JSON cycle — skip diagnostic */ } + } parseErr = err; } if (result && (result.file_code || result.download_url || result.embed_url)) { diff --git a/lib/upload-manager.js b/lib/upload-manager.js index 25bb33e..ce99002 100644 --- a/lib/upload-manager.js +++ b/lib/upload-manager.js @@ -9,6 +9,7 @@ const DoodstreamUploader = require('./doodstream-upload'); const ClouddropUploader = require('./clouddrop-upload'); const Semaphore = require('./semaphore'); const Throttle = require('./throttle'); +const { probeFileHead, summarizeFileStat } = require('./file-probe'); const DEFAULT_SETTINGS = { retries: 3, @@ -424,6 +425,16 @@ class UploadManager extends EventEmitter { maxAttempts }); + const fileStat = summarizeFileStat(task.file); + const fileProbe = await probeFileHead(task.file, 64).catch((err) => ({ ok: false, error: err.message, kind: 'unreadable' })); + this._rotLog('upload-start', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + fileSize: fileStat.size, fileMtime: fileStat.mtime, + detectedKind: fileProbe && fileProbe.kind ? fileProbe.kind : 'unknown', + isVideoLike: !!(fileProbe && fileProbe.isVideoLike), + headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null + }); + // Acquire hoster semaphore first so jobs waiting for a hoster slot // don't waste global slots (prevents underutilization) await hosterSemaphore.acquire(signal); @@ -596,6 +607,23 @@ class UploadManager extends EventEmitter { this.activeJobs.delete(uploadId); const isSpeedRestart = speedAbort && speedAbort.signal.aborted && !signal.aborted; + if (!signal.aborted && !isSpeedRestart) { + const diag = (err && typeof err === 'object' && err.diagnostic) || {}; + this._rotLog('upload-failure', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + attempt, + error: err && err.message ? err.message : String(err), + fileRejected: !!(err && err.fileRejected), + accountError: !!(err && err.accountError), + hosterTransient: !!(err && err.hosterTransient), + http: diag.http || null, + contentType: diag.contentType || null, + detectedKind: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.kind) ? fileProbe.kind : null, + isVideoLike: !!(typeof fileProbe !== 'undefined' && fileProbe && fileProbe.isVideoLike), + headHex: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.headHex) ? fileProbe.headHex.slice(0, 32) : null, + payloadSnippet: diag.payloadSnippet || null + }); + } if (signal.aborted) { lastError = new Error('Abgebrochen'); break; diff --git a/tests/file-probe.test.js b/tests/file-probe.test.js new file mode 100644 index 0000000..efd3ed0 --- /dev/null +++ b/tests/file-probe.test.js @@ -0,0 +1,100 @@ +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { detectKind, isVideoLikeKind, probeFileHead, summarizeFileStat } = require('../lib/file-probe'); + +function tmpWrite(name, buf) { + const p = path.join(os.tmpdir(), `mhu-probe-${Date.now()}-${name}`); + fs.writeFileSync(p, buf); + return p; +} + +test('detectKind recognizes ISO-MP4 (ftyp box at offset 4)', () => { + const buf = Buffer.concat([Buffer.from([0x00, 0x00, 0x00, 0x20]), Buffer.from('ftypisom', 'ascii'), Buffer.alloc(8, 0)]); + assert.strictEqual(detectKind(buf), 'mp4-iso'); + assert.strictEqual(isVideoLikeKind('mp4-iso'), true); +}); + +test('detectKind recognizes Matroska / WebM EBML header', () => { + const buf = Buffer.from([0x1A, 0x45, 0xDF, 0xA3, 0x01, 0x00]); + assert.strictEqual(detectKind(buf), 'matroska'); + assert.strictEqual(isVideoLikeKind('matroska'), true); +}); + +test('detectKind recognizes AVI (RIFF...AVI )', () => { + const buf = Buffer.concat([Buffer.from('RIFF', 'ascii'), Buffer.from([0x00, 0x00, 0x00, 0x00]), Buffer.from('AVI ', 'ascii')]); + assert.strictEqual(detectKind(buf), 'avi'); +}); + +test('detectKind recognizes FLV', () => { + const buf = Buffer.concat([Buffer.from('FLV', 'ascii'), Buffer.from([0x01])]); + assert.strictEqual(detectKind(buf), 'flv'); +}); + +test('detectKind recognizes ASF (WMV)', () => { + const buf = Buffer.from([0x30, 0x26, 0xB2, 0x75, 0x00, 0x00]); + assert.strictEqual(detectKind(buf), 'asf-wmv'); +}); + +test('detectKind recognizes MPEG-PS (00 00 01 BA)', () => { + const buf = Buffer.from([0x00, 0x00, 0x01, 0xBA, 0x00]); + assert.strictEqual(detectKind(buf), 'mpeg-ps'); +}); + +test('detectKind recognizes JPEG (non-video)', () => { + const buf = Buffer.from([0xFF, 0xD8, 0xFF, 0xE0]); + assert.strictEqual(detectKind(buf), 'jpeg'); + assert.strictEqual(isVideoLikeKind('jpeg'), false); +}); + +test('detectKind recognizes HTML response (non-video)', () => { + const buf = Buffer.from('', 'ascii'); + assert.strictEqual(detectKind(buf), 'html'); + assert.strictEqual(isVideoLikeKind('html'), false); +}); + +test('detectKind returns empty for zero-length and unknown for noise', () => { + assert.strictEqual(detectKind(Buffer.alloc(0)), 'empty'); + assert.strictEqual(detectKind(Buffer.from([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF])), 'unknown'); +}); + +test('probeFileHead reads first bytes and returns hex + kind for an MP4-like file', async () => { + const mp4Head = Buffer.concat([Buffer.from([0x00, 0x00, 0x00, 0x20]), Buffer.from('ftypisom', 'ascii'), Buffer.alloc(16, 0xAA)]); + const p = tmpWrite('fake.mp4', mp4Head); + try { + const res = await probeFileHead(p, 64); + assert.strictEqual(res.ok, true); + assert.strictEqual(res.kind, 'mp4-iso'); + assert.strictEqual(res.isVideoLike, true); + assert.ok(res.headHex.startsWith('0000002066747970')); + assert.strictEqual(res.bytesRead, mp4Head.length); + } finally { + fs.unlinkSync(p); + } +}); + +test('probeFileHead returns ok:false with kind=unreadable for missing file', async () => { + const res = await probeFileHead(path.join(os.tmpdir(), `does-not-exist-${Date.now()}.mp4`), 32); + assert.strictEqual(res.ok, false); + assert.strictEqual(res.kind, 'unreadable'); + assert.ok(res.error); +}); + +test('summarizeFileStat returns size + mtime for a real file', () => { + const p = tmpWrite('stat.bin', Buffer.alloc(123, 0xCC)); + try { + const stat = summarizeFileStat(p); + assert.strictEqual(stat.size, 123); + assert.strictEqual(stat.isFile, true); + assert.ok(stat.mtime); + } finally { + fs.unlinkSync(p); + } +}); + +test('summarizeFileStat returns error for missing file', () => { + const stat = summarizeFileStat(path.join(os.tmpdir(), `does-not-exist-${Date.now()}.bin`)); + assert.ok(stat.error); +});