commit 6d0ad84d2d36353df69bbedf620f9cf36c67fd11 Author: Sucukdeluxe <259325684+Sucukdeluxe@users.noreply.github.com> Date: Sat Aug 1 17:46:29 2026 +0200 Release Multi-Hoster Uploader 3.3.108 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42a6a46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +node_modules/ +release/ +.worktrees/ +__pycache__/ +*.pyc + +electron-config.json +electron-config.json.bak +electron-config.json.tmp +electron-config.json.pre-history-split.bak +electron-config.pre-import-*.json +electron-history.json +electron-history.json.tmp +*.log +debug.log +fileuploader.log +account-rotation.log +doodstream-debug.log +upload-debug.log +release-*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..7609905 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Multi-Hoster-Upload + +Multi-Hoster-Upload is a Windows desktop app for managing large file batches across multiple video hosting services from one queue. + +![Multi-Hoster-Upload product overview](assets/product-overview.png) + +[Download the latest release](https://github.com/Sucukdeluxe/Multi-Hoster-Upload/releases/latest) + +## Capabilities + +- Upload one batch to several supported hosters in parallel. +- Manage multiple accounts per hoster with validation, health checks, and automatic rotation. +- Add files by drag and drop or file selection and monitor live queue progress. +- Control per-hoster concurrency, bandwidth limits, retries, and folder monitoring. +- Keep local upload history and copy completed links in bulk. + +## Supported hosters + +| Hoster | Authentication | +| --- | --- | +| Doodstream | Web login or API key | +| VOE | Web login or API key | +| Vidmoly | Web login | +| Byse | API key | +| Clouddrop | API key | + +## Installation + +1. Download the Setup or portable executable from the [latest GitHub release](https://github.com/Sucukdeluxe/Multi-Hoster-Upload/releases/latest). +2. Run the installer, or launch the portable executable directly. +3. Add and validate at least one hoster account in Settings, then select files and start the queue. + +## Local data and credentials + +Settings, queue state, and upload history are stored locally in the app's user-data directory. Hoster passwords and API keys are encrypted with Electron safeStorage before being written when operating-system encryption is available; on Windows this uses DPAPI for the current user profile. + +## Development + +```powershell +npm install +npm start +npm test +npm run lint +npm run release:win +``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..aa8784a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +Report security vulnerabilities through GitHub private vulnerability reporting when it is available for this repository: open the Security tab, choose Advisories, and select "Report a vulnerability." + +Do not post credentials, passwords, API keys, tokens, cookies, private logs, configuration files, or private upload links in public issues. Redact sensitive values from reproduction steps and attachments. + +If private vulnerability reporting is unavailable, contact the repository owner through a private channel before sharing technical details. diff --git a/assets/app_icon.ico b/assets/app_icon.ico new file mode 100644 index 0000000..ed6b808 Binary files /dev/null and b/assets/app_icon.ico differ diff --git a/assets/app_icon.png b/assets/app_icon.png new file mode 100644 index 0000000..ac13bab Binary files /dev/null and b/assets/app_icon.png differ diff --git a/assets/product-overview.png b/assets/product-overview.png new file mode 100644 index 0000000..e719b86 Binary files /dev/null and b/assets/product-overview.png differ diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..513008a --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,105 @@ +import security from 'eslint-plugin-security'; + +const sharedRules = { + // Security rules + // detect-object-injection disabled: 78 false positives from config lookups like obj[hosterName] + 'security/detect-object-injection': 'off', + 'security/detect-non-literal-regexp': 'warn', + 'security/detect-unsafe-regex': 'warn', + 'security/detect-buffer-noassert': 'warn', + 'security/detect-eval-with-expression': 'error', + 'security/detect-no-csrf-before-method-override': 'warn', + 'security/detect-possible-timing-attacks': 'warn', + 'security/detect-pseudoRandomBytes': 'warn', + // Code quality + 'no-unused-vars': ['warn', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }], + 'no-undef': 'error', + 'no-constant-condition': 'warn', + 'no-debugger': 'error', + 'no-duplicate-case': 'error', + 'no-empty': ['warn', { allowEmptyCatch: true }], + 'no-ex-assign': 'error', + 'no-extra-boolean-cast': 'warn', + 'no-func-assign': 'error', + 'no-inner-declarations': 'error', + 'no-irregular-whitespace': 'error', + 'no-unreachable': 'error', + 'use-isnan': 'error', + 'valid-typeof': 'error', + 'eqeqeq': ['warn', 'always'], + 'no-caller': 'error', + 'no-eval': 'error', + 'no-implied-eval': 'error', + 'no-new-func': 'error', + 'no-throw-literal': 'warn', + 'no-self-assign': 'error', + 'no-self-compare': 'error', + 'no-loss-of-precision': 'error', + 'no-dupe-keys': 'error', + 'no-unsafe-finally': 'error', + 'no-unmodified-loop-condition': 'warn', + 'no-template-curly-in-string': 'warn', +}; + +const nodeGlobals = { + process: 'readonly', + console: 'readonly', + setTimeout: 'readonly', + clearTimeout: 'readonly', + setInterval: 'readonly', + clearInterval: 'readonly', + setImmediate: 'readonly', + Buffer: 'readonly', + URL: 'readonly', + URLSearchParams: 'readonly', + fetch: 'readonly', + crypto: 'readonly', + structuredClone: 'readonly', +}; + +export default [ + { ignores: ['**/node_modules/**', 'release/**', 'tests/**'] }, + { + files: ['**/*.js'], + ignores: ['gateway/**'], + plugins: { security }, + languageOptions: { + ecmaVersion: 2022, + sourceType: 'commonjs', + globals: { + require: 'readonly', + module: 'readonly', + exports: 'readonly', + __dirname: 'readonly', + __filename: 'readonly', + ...nodeGlobals, + AbortController: 'readonly', + AbortSignal: 'readonly', + navigator: 'readonly', + document: 'readonly', + window: 'readonly', + localStorage: 'readonly', + HTMLElement: 'readonly', + alert: 'readonly', + confirm: 'readonly', + requestAnimationFrame: 'readonly', + queueMicrotask: 'readonly', + Intl: 'readonly', + performance: 'readonly', + EventSource: 'readonly', + } + }, + rules: sharedRules + }, + { + files: ['gateway/**/*.js', 'gateway/**/*.mjs'], + ignores: ['gateway/node_modules/**'], + plugins: { security }, + languageOptions: { + ecmaVersion: 2022, + sourceType: 'module', + globals: nodeGlobals + }, + rules: sharedRules + } +]; diff --git a/lib/account-auth.js b/lib/account-auth.js new file mode 100644 index 0000000..1fd0a4a --- /dev/null +++ b/lib/account-auth.js @@ -0,0 +1,36 @@ +// Decides which credential an upload task should use for a given hoster. +// Extracted from main.js buildTaskFromAccount so the routing can be unit-tested +// without Electron. +// +// DOODSTREAM SPECIAL CASE: prefer the official doodapi.co API key whenever the +// account has one. The web-login path (username/password) drives doodstream's +// browser upload flow, which hands the filecode back inside an XFileSharing +// HTML form. On long/large uploads that form comes back empty (no fn) because a +// per-page-load sess_id token ages out over the multi-minute upload and/or the +// server-side file-registration callback times out — the upload then "succeeds" +// (bytes sent, HTTP 200) but yields no link. The JSON API returns the filecode +// directly in result[0].filecode and authenticates with a persistent api_key, +// so it has no empty-form failure mode for result retrieval. The API path was +// doodstream's ORIGINAL upload path (present since the initial commit); web +// login was added later only as an alternative for keyless accounts — so +// preferring the key here restores the intended primary path, it doesn't fight +// a deliberate choice. Keyless accounts keep using web login unchanged. +function selectUploadAuth(hoster, account) { + if (!account || typeof account !== 'object') return {}; + + if (hoster === 'doodstream.com' && account.apiKey) { + return { apiKey: account.apiKey }; + } + if (account.authType === 'api' && account.apiKey) { + return { apiKey: account.apiKey }; + } + if (account.username && account.password) { + return { username: account.username, password: account.password }; + } + if (account.apiKey) { + return { apiKey: account.apiKey }; + } + return {}; +} + +module.exports = { selectUploadAuth }; diff --git a/lib/account-rotation.js b/lib/account-rotation.js new file mode 100644 index 0000000..de6ee7c --- /dev/null +++ b/lib/account-rotation.js @@ -0,0 +1,27 @@ +function enabledAccountsFor(hosters, hoster, hasCreds) { + const list = hosters && hosters[hoster]; + if (!Array.isArray(list)) return []; + return list.filter(a => a && a.enabled !== false && hasCreds(hoster, a)); +} + +function createAccountPicker({ hosters, hosterSettings, hasCreds, indices }) { + const rotIdx = Object.assign(Object.create(null), indices || {}); + let dirty = false; + function pick(hoster) { + const enabled = enabledAccountsFor(hosters, hoster, hasCreds); + if (enabled.length === 0) return null; + const hs = (hosterSettings && hosterSettings[hoster]) || {}; + if (hs.rotateAccounts === true && enabled.length > 1) { + const cursor = Number.isFinite(rotIdx[hoster]) ? rotIdx[hoster] : 0; + rotIdx[hoster] = cursor + 1; + dirty = true; + return enabled[cursor % enabled.length]; + } + return enabled[0]; + } + pick.indices = () => ({ ...rotIdx }); + pick.dirty = () => dirty; + return pick; +} + +module.exports = { createAccountPicker, enabledAccountsFor }; diff --git a/lib/backup-crypto.js b/lib/backup-crypto.js new file mode 100644 index 0000000..9fd9f22 --- /dev/null +++ b/lib/backup-crypto.js @@ -0,0 +1,95 @@ +const crypto = require('crypto'); + +const MAGIC = Buffer.from('MHU1'); +const SALT_LEN = 16; +const IV_LEN = 12; +const TAG_LEN = 16; +const KEY_LEN = 32; +const ITERATIONS = 100_000; +const DIGEST = 'sha512'; +const ALGO = 'aes-256-gcm'; + +// Fixed app-internal passphrase — backups are opaque without the app, which is +// enough protection for API keys stored locally. We keep the AES-GCM envelope +// (with random salt/iv) so each export is still distinct and authenticated. +const APP_PASSPHRASE = 'multi-hoster-upload::backup::v1'; + +function deriveKey(passphrase, salt) { + return crypto.pbkdf2Sync(passphrase, salt, ITERATIONS, KEY_LEN, DIGEST); +} + +/** + * Encrypt a config object. + * Returns a Buffer: MHU1 | salt(16) | iv(12) | tag(16) | ciphertext + */ +function encrypt(config) { + const plaintext = Buffer.from(JSON.stringify(config), 'utf-8'); + const salt = crypto.randomBytes(SALT_LEN); + const iv = crypto.randomBytes(IV_LEN); + const key = deriveKey(APP_PASSPHRASE, salt); + + const cipher = crypto.createCipheriv(ALGO, key, iv); + const encrypted = Buffer.concat([cipher.update(plaintext), cipher.final()]); + const tag = cipher.getAuthTag(); + + plaintext.fill(0); + key.fill(0); + return Buffer.concat([MAGIC, salt, iv, tag, encrypted]); +} + +/** + * Decrypt a .mhu buffer. + * Tries the app's built-in key first; if that fails and a user password is + * provided, falls back to legacy password-based decryption. Throws a special + * error with `needsPassword = true` if the app key fails and no password was + * given, so callers can prompt the user for the legacy password. + */ +function decrypt(buffer, userPassword) { + if (buffer.length < MAGIC.length + SALT_LEN + IV_LEN + TAG_LEN + 1) { + throw new Error('Ungültiges Backup-Format'); + } + + const magic = buffer.subarray(0, 4); + if (!magic.equals(MAGIC)) { + throw new Error('Keine gültige .mhu Backup-Datei'); + } + + let offset = MAGIC.length; + const salt = buffer.subarray(offset, offset += SALT_LEN); + const iv = buffer.subarray(offset, offset += IV_LEN); + const tag = buffer.subarray(offset, offset += TAG_LEN); + const ciphertext = buffer.subarray(offset); + + const tryPassphrase = (passphrase) => { + const key = deriveKey(passphrase, salt); + const decipher = crypto.createDecipheriv(ALGO, key, iv); + decipher.setAuthTag(tag); + try { + const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]); + const result = JSON.parse(decrypted.toString('utf-8')); + decrypted.fill(0); + key.fill(0); + return result; + } catch { + key.fill(0); + return null; + } + }; + + // 1) Try the app-internal key (new format, no password required). + const fromApp = tryPassphrase(APP_PASSPHRASE); + if (fromApp) return fromApp; + + // 2) Legacy format: user had set their own password. + if (userPassword) { + const fromUser = tryPassphrase(userPassword); + if (fromUser) return fromUser; + throw new Error('Falsches Passwort oder beschädigte Datei'); + } + + const err = new Error('Dieses Backup wurde mit einem Passwort verschlüsselt'); + err.needsPassword = true; + throw err; +} + +module.exports = { encrypt, decrypt }; diff --git a/lib/clouddrop-upload.js b/lib/clouddrop-upload.js new file mode 100644 index 0000000..5223b88 --- /dev/null +++ b/lib/clouddrop-upload.js @@ -0,0 +1,239 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request, Agent } = require('undici'); + +const BASE_URL = 'https://clouddrop.cc'; +const API_BASE = `${BASE_URL}/api/cloud`; +const CHUNK_UPLOAD_BASE = 'https://upload.clouddrop.cc/api/cloud'; +const USER_AGENT = 'multi-hoster-uploader/1.0'; + +const SIMPLE_UPLOAD_LIMIT = 16 * 1024 * 1024; // 16 MB +const CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB — server's fixed chunk size +const INIT_TIMEOUT = 60_000; +const CHUNK_TIMEOUT = 30 * 60_000; // 30 min per chunk +const COMPLETE_TIMEOUT = 5 * 60_000; +const SIMPLE_UPLOAD_TIMEOUT = 30 * 60_000; + +// Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under +// the server's per-IP limit of 100 concurrent connections (cd_conn). +// Shared across all ClouddropUploader instances via module-level agent. +const clouddropAgent = new Agent({ + connections: 50, + pipelining: 1, + keepAliveTimeout: 30_000, + keepAliveMaxTimeout: 60_000 +}); + +/** + * Clouddrop.cc uploader — uses API Key (Bearer) authentication. + * Files > 16 MB use the chunked protocol, smaller files use simple upload. + * After upload, a share link is created and returned as download_url. + */ +class ClouddropUploader { + constructor(apiKey) { + this.apiKey = String(apiKey || '').trim(); + } + + _headers(extra) { + return { + 'Authorization': `Bearer ${this.apiKey}`, + 'User-Agent': USER_AGENT, + 'Accept': 'application/json', + ...(extra || {}) + }; + } + + async _parseJsonResponse(res) { + const text = await res.body.text(); + let payload = null; + try { payload = text ? JSON.parse(text) : {}; } catch { + throw new Error(`Clouddrop: API-Antwort war kein JSON (HTTP ${res.statusCode}): ${text.slice(0, 200)}`); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + const msg = (payload && (payload.error || payload.message)) + || `HTTP ${res.statusCode}`; + const err = new Error(`Clouddrop: ${msg}`); + err.status = res.statusCode; + throw err; + } + return payload; + } + + /** + * Upload a file. Returns { download_url, embed_url, file_code }. + */ + async upload(filePath, progressCb, signal, throttle) { + if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt'); + const fileName = path.basename(filePath); + let fileSize = 0; + try { fileSize = fs.statSync(filePath).size; } + catch { throw new Error(`Clouddrop: Datei nicht lesbar: ${fileName}`); } + if (fileSize <= 0) throw new Error('Clouddrop: Datei ist leer'); + + let fileId; + if (fileSize <= SIMPLE_UPLOAD_LIMIT) { + fileId = await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle); + } else { + fileId = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle); + } + + return { + download_url: `${BASE_URL}/share/${fileId}`, + embed_url: null, + file_code: fileId + }; + } + + /** + * Simple upload for files < 16 MB — single multipart POST. + */ + async _uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle) { + const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex'); + const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + + const preamble = + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${safeFileName}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`; + const epilogue = `\r\n--${boundary}--\r\n`; + + const preambleBuf = Buffer.from(preamble, 'utf-8'); + const epilogueBuf = Buffer.from(epilogue, 'utf-8'); + const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; + + let bytesRead = 0; + async function* generate() { + yield preambleBuf; + const fileStream = fs.createReadStream(filePath, { highWaterMark: 1024 * 1024 }); + for await (const chunk of fileStream) { + if (signal && signal.aborted) throw new Error('Aborted'); + if (throttle) await throttle.consume(chunk.length, signal); + bytesRead += chunk.length; + yield chunk; + if (progressCb) progressCb(bytesRead, fileSize); + } + yield epilogueBuf; + } + + const res = await request(`${API_BASE}/upload?mode=rename`, { + method: 'POST', + dispatcher: clouddropAgent, + body: generate(), + signal, + headers: this._headers({ + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize) + }), + headersTimeout: SIMPLE_UPLOAD_TIMEOUT, + bodyTimeout: SIMPLE_UPLOAD_TIMEOUT + }); + + const payload = await this._parseJsonResponse(res); + if (!payload.fileId) throw new Error(`Clouddrop: Keine fileId in Upload-Antwort`); + return payload.fileId; + } + + /** + * Chunked upload for files > 16 MB. + * Flow: POST /upload/init → PUT /upload/:sessionId/chunk/:n (0-based) → POST /upload/:sessionId/complete + */ + async _uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle) { + // 1. Init session + const initRes = await request(`${API_BASE}/upload/init`, { + method: 'POST', + dispatcher: clouddropAgent, + signal, + headers: this._headers({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ filename: fileName, size: fileSize, parentId: null }), + headersTimeout: INIT_TIMEOUT, + bodyTimeout: INIT_TIMEOUT + }); + const initPayload = await this._parseJsonResponse(initRes); + const sessionId = initPayload.sessionId; + const chunkSize = initPayload.chunkSize || CHUNK_SIZE; + const totalChunks = initPayload.totalChunks || Math.ceil(fileSize / chunkSize); + if (!sessionId) throw new Error('Clouddrop: Keine sessionId von /upload/init'); + + // 2. Read file and PUT chunks sequentially. + // Reuse a single buffer for all chunks (only the last chunk may be smaller, + // in which case we slice a view). Avoids 64× 16 MB allocations on a 1 GB + // file — real GC pressure during busy uploads. + const fh = await fs.promises.open(filePath, 'r'); + let bytesSent = 0; + const reusableBuf = Buffer.allocUnsafe(chunkSize); + try { + for (let i = 0; i < totalChunks; i++) { + if (signal && signal.aborted) throw new Error('Aborted'); + + const offset = i * chunkSize; + const remaining = fileSize - offset; + const thisChunkSize = Math.min(chunkSize, remaining); + await fh.read(reusableBuf, 0, thisChunkSize, offset); + const body = thisChunkSize === chunkSize + ? reusableBuf + : reusableBuf.subarray(0, thisChunkSize); + + if (throttle) await throttle.consume(thisChunkSize, signal); + + const chunkRes = await request(`${CHUNK_UPLOAD_BASE}/upload/${sessionId}/chunk/${i}`, { + method: 'PUT', + dispatcher: clouddropAgent, + signal, + body, + headers: this._headers({ + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(thisChunkSize) + }), + headersTimeout: CHUNK_TIMEOUT, + bodyTimeout: CHUNK_TIMEOUT + }); + await this._parseJsonResponse(chunkRes); + + bytesSent += thisChunkSize; + if (progressCb) progressCb(bytesSent, fileSize); + } + } finally { + try { await fh.close(); } catch {} + } + + // 3. Complete session — all bytes are already on the server at this point. + // We MUST NOT throw here, otherwise the upload-manager would retry the entire + // multi-GB upload. Any failure (timeout, non-JSON, missing fileId, server still + // post-processing) is swallowed and we fall back to sessionId as file_code. + try { + const completeRes = await request(`${API_BASE}/upload/${sessionId}/complete`, { + method: 'POST', + dispatcher: clouddropAgent, + signal, + headers: this._headers({ 'Content-Type': 'application/json' }), + body: '{}', + headersTimeout: COMPLETE_TIMEOUT, + bodyTimeout: COMPLETE_TIMEOUT + }); + const completePayload = await this._parseJsonResponse(completeRes).catch(() => ({})); + return completePayload.fileId || completePayload.id || sessionId; + } catch { + return sessionId; + } + } + + /** + * Lightweight auth check — GET /api/cloud/files (list root, small response). + */ + async checkAuth(signal) { + if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt'); + const res = await request(`${API_BASE}/files/?limit=1`, { + method: 'GET', + dispatcher: clouddropAgent, + signal, + headers: this._headers(), + headersTimeout: 15_000, + bodyTimeout: 15_000 + }); + await this._parseJsonResponse(res); + return true; + } +} + +module.exports = ClouddropUploader; diff --git a/lib/coalesced-set.js b/lib/coalesced-set.js new file mode 100644 index 0000000..4136d3f --- /dev/null +++ b/lib/coalesced-set.js @@ -0,0 +1,73 @@ +// Microtask-coalesced set. Adds are O(1); the apply callback runs once per +// scheduler tick with every id collected since the last flush. +// +// Used by the renderer to merge a burst of done-jobs (e.g. 500 jobs all +// finishing within milliseconds) into a single queueJobs.filter() pass — +// without this each event was its own O(N) sweep, so 500 finishes were +// O(N²) and visibly froze the UI on completion. +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag). + +(function (root) { + 'use strict'; + + /** + * Build a coalesced set. + * @param {{ apply: (Set) => void, scheduler?: (cb: () => void) => void }} opts + * apply: called once per scheduler tick with the accumulated ids. + * scheduler: defaults to queueMicrotask. Tests can pass a synchronous + * stand-in to avoid async waits. + */ + function makeCoalescedSet(opts) { + if (!opts || typeof opts.apply !== 'function') { + throw new TypeError('makeCoalescedSet: { apply: fn } required'); + } + const apply = opts.apply; + const scheduler = typeof opts.scheduler === 'function' + ? opts.scheduler + : (typeof queueMicrotask === 'function' ? queueMicrotask : (cb) => Promise.resolve().then(cb)); + let pending = new Set(); + let scheduled = false; + + function flush() { + scheduled = false; + if (pending.size === 0) return; + const drop = pending; + pending = new Set(); + try { apply(drop); } catch (e) { + // Don't let a failing apply lock out the next batch — surface it + // but keep the coalescer usable. + if (typeof console !== 'undefined' && console.error) console.error(e); + } + } + + return { + add(id) { + pending.add(id); + if (!scheduled) { + scheduled = true; + scheduler(flush); + } + }, + /** + * Synchronously consume any pending ids. Used by beforeunload paths + * where we can't wait for the next microtask before persisting. + */ + drainSync() { + if (pending.size === 0) return; + const drop = pending; + pending = new Set(); + scheduled = false; + apply(drop); + }, + /** Introspection for tests + diagnostics. */ + pendingSize() { return pending.size; }, + isScheduled() { return scheduled; } + }; + } + + const api = { makeCoalescedSet }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.CoalescedSet = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/config-store.js b/lib/config-store.js new file mode 100644 index 0000000..566372f --- /dev/null +++ b/lib/config-store.js @@ -0,0 +1,654 @@ +const fs = require('fs'); +const path = require('path'); +const secretStore = require('./secret-store'); +const { normalizeLogMode } = require('./log-mode'); + +const HOSTER_SETTINGS_DEFAULTS = { + retries: 3, + maxSpeedKbs: 0, // 0 = unlimited + parallelCount: 2, // 1-100 + restartBelowKbs: 0, // 0 = off + timeIntervalSec: 0, // delay between jobs + maxSizeMb: 0, // 0 = unlimited + logToFile: true, // write this hoster's successful links to fileuploader.log + rotateAccounts: false, + sizeMemoEnabled: true +}; + +// Template for each hoster type (used as defaults for new accounts) +const HOSTER_ACCOUNT_TEMPLATES = { + 'doodstream.com': { enabled: true, authType: 'login', username: '', password: '' }, + 'doodstream.com:api': { enabled: true, authType: 'api', apiKey: '' }, + 'voe.sx': { enabled: true, authType: 'login', username: '', password: '' }, + 'voe.sx:api': { enabled: true, authType: 'api', apiKey: '' }, + 'vidmoly.me': { enabled: true, authType: 'login', username: '', password: '' }, + 'byse.sx': { enabled: true, authType: 'api', apiKey: '' }, + 'clouddrop.cc': { enabled: true, authType: 'api', apiKey: '' } +}; + +// All known hoster names (used for iteration) +const HOSTER_NAMES = ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx', 'clouddrop.cc']; + +// Dropdown options for "Add Account" modal: value -> label +const HOSTER_ADD_OPTIONS = [ + { value: 'doodstream.com', label: 'Doodstream (Web Login)', hoster: 'doodstream.com', authType: 'login' }, + { value: 'doodstream.com:api', label: 'Doodstream (API)', hoster: 'doodstream.com', authType: 'api' }, + { value: 'voe.sx', label: 'Voe (Web Login)', hoster: 'voe.sx', authType: 'login' }, + { value: 'voe.sx:api', label: 'Voe (API)', hoster: 'voe.sx', authType: 'api' }, + { value: 'vidmoly.me', label: 'Vidmoly (Web Login)', hoster: 'vidmoly.me', authType: 'login' }, + { value: 'byse.sx', label: 'Byse (API)', hoster: 'byse.sx', authType: 'api' }, + { value: 'clouddrop.cc', label: 'Clouddrop (API)', hoster: 'clouddrop.cc', authType: 'api' } +]; + +const DEFAULTS = { + hosters: { + 'doodstream.com': [], + 'voe.sx': [], + 'vidmoly.me': [], + 'byse.sx': [], + 'clouddrop.cc': [] + }, + hosterSettings: { + 'doodstream.com': { ...HOSTER_SETTINGS_DEFAULTS }, + 'voe.sx': { ...HOSTER_SETTINGS_DEFAULTS }, + 'vidmoly.me': { ...HOSTER_SETTINGS_DEFAULTS }, + 'byse.sx': { ...HOSTER_SETTINGS_DEFAULTS }, + 'clouddrop.cc': { ...HOSTER_SETTINGS_DEFAULTS } + }, + globalSettings: { + alwaysOnTop: false, + shutdownAfterFinish: 'nothing', // nothing | sleep | shutdown | restart + logFilePath: '', + sessionLog: false, // legacy boolean (kept for back-compat reads); normalized into logMode on load + logVerbose: false, // when true, [DEBUG] level entries are written to debug.log + webhookUrl: '', // POST target on batch-done (Discord or generic JSON) + webhookMention: '', // optional Discord ping target: user-id, role:id, @here, @everyone + autoRetryRounds: 0, // 0 = off; 1-5 automatic retry rounds for transient failures after batch end + autoRetryDelayMin: 5, // base delay in minutes between auto-retry rounds (linear backoff: round N waits N*delay) + historyRetention: 'all', // 'all' | '7d' | '30d' | '90d' | '1000' | '100' — storage cap for upload history + // NOTE: logMode is intentionally NOT in DEFAULTS. If it were, the deep-merge + // would seed logMode='single' for every load, which would beat (and silently + // erase) the legacy sessionLog:true → "daily" migration. normalizeLogMode in + // load() sets logMode after the merge, looking at the saved-only data. + resumeQueueOnLaunch: true, + parallelUploadCount: 0, // 0 = use per-hoster limits only + scaleParallelUploads: false, + removeFromQueueOnDone: false, + showDropTarget: false, + globalMaxSpeedKbs: 0, // 0 = unlimited global speed + pendingQueue: null, + scramble: { + active: false, + prefix: '', + suffix: '', + chars: 'both', // 'letters' | 'numbers' | 'both' + length: 0 // 0 = same as original basename length + }, + folderMonitor: { + enabled: false, + folderPath: '', + recursive: false, + filterMode: 'include', // 'include' | 'exclude' + extensions: '', // comma-separated: 'mp4,mkv,avi' + skipDuplicates: true, + delaySec: 3, + autoStart: true, + hosters: [] // pre-selected hosters, empty = ask via modal + }, + remote: { + enabled: false, + port: 9100, + token: '', + allowInput: true + }, + diagnostics: { + enabled: false, + port: 9110, + token: '', + label: '', + codeIssuedAt: 0, + bindMode: 'local', + publicHost: '', + allowlist: [], + bindAddress: '127.0.0.1' + } + }, + history: [], + rotationCursors: {} +}; + +const HISTORY_RETENTION_OPTIONS = [ + { value: 'all', label: 'Alles behalten' }, + { value: '7d', label: 'Letzte 7 Tage' }, + { value: '30d', label: 'Letzte 30 Tage' }, + { value: '90d', label: 'Letzte 90 Tage' }, + { value: '1000', label: 'Letzte 1000 Uploads' }, + { value: '100', label: 'Letzte 100 Uploads' } +]; + +function batchTimestampMs(batch) { + const raw = batch && batch.timestamp; + if (raw === null || raw === undefined || raw === '') return null; + const ms = typeof raw === 'number' ? raw : Date.parse(raw); + return Number.isFinite(ms) ? ms : null; +} + +function batchRowCount(batch) { + let n = 0; + const files = (batch && batch.files) || []; + for (const file of files) { + for (const result of (file.results || [])) { + if (result.status === 'aborted' || result.status === 'error') continue; + n++; + } + } + return n; +} + +function countHistoryRows(history) { + let n = 0; + for (const batch of (history || [])) n += batchRowCount(batch); + return n; +} + +function applyHistoryRetention(history, retention, nowMs) { + if (!Array.isArray(history) || history.length === 0) return history; + const policy = String(retention || 'all'); + if (policy === 'all') return history; + + if (/^\d+d$/.test(policy)) { + const days = parseInt(policy, 10); + if (!Number.isFinite(days) || days <= 0) return history; + const cutoff = nowMs - days * 86400000; + return history.filter(b => { + const ts = batchTimestampMs(b); + return ts === null || ts >= cutoff; + }); + } + + const maxRows = parseInt(policy, 10); + if (!Number.isFinite(maxRows) || maxRows <= 0) return history; + const keptReversed = []; + let acc = 0; + for (let i = history.length - 1; i >= 0; i--) { + keptReversed.push(history[i]); + acc += batchRowCount(history[i]); + if (acc >= maxRows) break; + } + return keptReversed.reverse(); +} + +class ConfigStore { + constructor(app) { + const dir = app && app.isPackaged + ? app.getPath('userData') + : path.join(__dirname, '..'); + this.filePath = path.join(dir, 'electron-config.json'); + this.historyPath = path.join(dir, 'electron-history.json'); + this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions + this._historyWriteQueue = Promise.resolve(); + this._historyMigrated = false; + this._cache = null; + this._cacheKey = ''; + this._perfLog = null; + this._wqDepth = 0; + + // Migrate config from old location if current doesn't exist + if (!fs.existsSync(this.filePath) && app && app.isPackaged) { + this._migrateFromOldPath(app); + } + if (app && app.isPackaged) { + this._migrateHistory(); + } + } + + _readHistoryFile() { + try { + const raw = fs.readFileSync(this.historyPath, 'utf-8'); + if (!raw || raw.trim().length < 2) return []; + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + if (parsed && Array.isArray(parsed.history)) return parsed.history; + return []; + } catch { + return null; + } + } + + _writeHistoryFileDurable(arr) { + const tmp = this.historyPath + '.tmp'; + const fd = fs.openSync(tmp, 'w'); + try { + fs.writeSync(fd, JSON.stringify(arr)); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(tmp, this.historyPath); + } + + _writeHistoryFileAtomic(arr) { + return new Promise((resolve, reject) => { + const tmp = this.historyPath + '.tmp'; + fs.writeFile(tmp, JSON.stringify(arr), 'utf-8', (err) => { + if (err) return reject(err); + try { fs.renameSync(tmp, this.historyPath); } catch (e) { return reject(e); } + resolve(); + }); + }); + } + + _enqueueHistoryWrite(fn) { + this._historyWriteQueue = this._historyWriteQueue.then(fn, fn); + return this._historyWriteQueue; + } + + _migrateHistory() { + try { + if (fs.existsSync(this.historyPath)) { + this._historyMigrated = Array.isArray(this._readHistoryFile()); + return; + } + let cfg = null; + try { cfg = this._readAndParse(this.filePath); } catch {} + const hist = (cfg && Array.isArray(cfg.history)) ? cfg.history : []; + this._writeHistoryFileDurable(hist); + const check = this._readHistoryFile(); + if (Array.isArray(check) && check.length === hist.length) { + if (hist.length > 0) { + try { fs.copyFileSync(this.filePath, this.filePath + '.pre-history-split.bak'); } catch {} + } + this._historyMigrated = true; + } else { + this._historyMigrated = false; + } + } catch { + this._historyMigrated = false; + } + } + + _migrateFromOldPath(app) { + try { + const appDataDir = path.dirname(app.getPath('userData')); + // Check alternate folder names that may have been used + const candidates = ['multi-hoster-uploader', 'Multi-Hoster-Upload']; + for (const name of candidates) { + const oldPath = path.join(appDataDir, name, 'electron-config.json'); + if (oldPath !== this.filePath && fs.existsSync(oldPath)) { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true }); + fs.copyFileSync(oldPath, this.filePath); + return; + } + } + // Also check next to the executable (portable mode previous location) + const exeDir = path.dirname(app.getPath('exe')); + const portablePath = path.join(exeDir, 'electron-config.json'); + if (portablePath !== this.filePath && fs.existsSync(portablePath)) { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true }); + fs.copyFileSync(portablePath, this.filePath); + } + } catch {} + } + + _readAndParse(filePath) { + const raw = fs.readFileSync(filePath, 'utf-8'); + if (!raw || raw.trim().length < 2) return null; + return JSON.parse(raw); + } + + _clone(obj) { + try { return structuredClone(obj); } + catch { return JSON.parse(JSON.stringify(obj)); } + } + + setPerfLog(fn) { this._perfLog = typeof fn === 'function' ? fn : null; } + + _pqLen(globalSettings) { + const pq = globalSettings && globalSettings.pendingQueue; + return pq && Array.isArray(pq.queueJobs) ? pq.queueJobs.length : 0; + } + + _callerTag() { + const lines = (new Error().stack || '').split('\n'); + const out = []; + for (let i = 2; i < lines.length && out.length < 3; i++) { + const line = lines[i].trim(); + if (/config-store\.js/.test(line)) continue; + const m = line.match(/at (?:async )?([^ (]+)/); + if (m) out.push(m[1].split('.').pop()); + } + return out.join('<') || '?'; + } + + load() { + if (!this._perfLog) return this._loadImpl(); + const hadCache = !!this._cache; + const t0 = performance.now(); + const r = this._loadImpl(); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = this._pqLen(r && r.globalSettings); + const h = (r && r.history || []).length; + this._perfLog(`config-load wall=${dt.toFixed(0)}ms cache=${hadCache ? 'hit' : 'miss'} hist=${h} queue=${q} via=${this._callerTag()}`); + } + return r; + } + + _loadImpl() { + try { + // In-memory cache keyed on the file's mtime+size. The processed config + // (merged + credential-decrypted) is reparsed/re-decrypted from disk ONLY + // when the file actually changes. Our own writes refresh the cache (see + // _commit), and an external edit changes mtime/size so the cache misses + // and we reread. Without this, every one of the ~38 main.js load() call + // sites (incl. the per-500ms log-flush path) re-read disk + JSON.parse the + // whole growing history + DPAPI-decrypt every credential — the dominant + // long-running main-thread drag. load() always returns a CLONE so callers + // can mutate the result without corrupting the cache. + let stat = null; + try { stat = fs.statSync(this.filePath); } catch {} + const statKey = stat ? `${stat.mtimeMs}:${stat.size}` : ''; + if (stat && this._cache && this._cacheKey === statKey) { + return this._clone(this._cache); + } + + let data = null; + // Try main config + try { data = this._readAndParse(this.filePath); } catch {} + // Fallback to backup if main is empty/corrupt + if (!data) { + try { data = this._readAndParse(this.filePath + '.bak'); } catch {} + } + if (!data) { + try { data = this._readAndParse(this.filePath + '.pre-history-split.bak'); } catch {} + } + if (!data) { + const fresh = JSON.parse(JSON.stringify(DEFAULTS)); + fresh.globalSettings.logMode = normalizeLogMode(fresh.globalSettings); + return fresh; + } + + // Migrate old single-object format to array format + for (const [name, val] of Object.entries(data.hosters || {})) { + if (val && !Array.isArray(val)) { + if (!val.id) val.id = `${name}-migrated-${Date.now()}`; + // Infer authType for old format accounts + if (!val.authType) { + if (name === 'byse.sx') val.authType = 'api'; + else if (name === 'vidmoly.me') val.authType = 'login'; + else if (val.username && val.password) val.authType = 'login'; + else if (val.apiKey) val.authType = 'api'; + else val.authType = 'login'; + } + data.hosters[name] = [val]; + } + } + + // Merge hosters: ensure all known hosters exist as arrays + const hosters = {}; + for (const name of HOSTER_NAMES) { + const saved = data.hosters && data.hosters[name]; + if (Array.isArray(saved) && saved.length > 0) { + hosters[name] = saved.map((acc, i) => { + // Ensure authType is set on every account + if (!acc.authType) { + if (name === 'byse.sx') acc.authType = 'api'; + else if (name === 'vidmoly.me') acc.authType = 'login'; + else if (acc.username && acc.password) acc.authType = 'login'; + else if (acc.apiKey) acc.authType = 'api'; + else acc.authType = 'login'; + } + return { + ...acc, + id: acc.id || `${name}-${Date.now()}-${i}` + }; + }); + } else { + hosters[name] = []; + } + } + + // Merge hoster settings with defaults + const hosterSettings = {}; + for (const name of Object.keys(DEFAULTS.hosterSettings)) { + hosterSettings[name] = { + ...HOSTER_SETTINGS_DEFAULTS, + ...(data.hosterSettings && data.hosterSettings[name] || {}) + }; + } + const savedGlobal = data.globalSettings || {}; + const globalSettings = { + ...DEFAULTS.globalSettings, + ...savedGlobal + }; + // Deep-merge nested objects so new keys are always present + for (const key of Object.keys(DEFAULTS.globalSettings)) { + const def = DEFAULTS.globalSettings[key]; + if (def && typeof def === 'object' && !Array.isArray(def)) { + globalSettings[key] = { ...def, ...(savedGlobal[key] || {}) }; + } + } + // Normalize logMode at this single boundary. Legacy sessionLog: true + // means *daily* (the old field was named after a misnomer); see log-mode.js. + // Downstream readers consume logMode only and must NOT derive from + // sessionLog at call sites. + globalSettings.logMode = normalizeLogMode(globalSettings); + const rotationCursors = (data.rotationCursors && typeof data.rotationCursors === 'object' && !Array.isArray(data.rotationCursors)) + ? data.rotationCursors + : {}; + const result = { hosters, hosterSettings, globalSettings, history: this._historyMigrated ? [] : (data.history || []), rotationCursors }; + // Decrypt credentials stored with safeStorage so the rest of the app + // keeps working with plaintext in memory. + secretStore.decryptCredentials(result); + if (stat) { + this._cache = result; + this._cacheKey = statKey; + } + return this._clone(result); + } catch { + const fresh = JSON.parse(JSON.stringify(DEFAULTS)); + fresh.globalSettings.logMode = normalizeLogMode(fresh.globalSettings); + return fresh; + } + } + + // Encrypt credential fields without mutating the caller's plaintext object. + // Only `hosters` carries credentials, so we clone ONLY that subtree — the rest + // (history, globalSettings, …) is referenced read-only into the stringified + // object. Deep-cloning the whole config here (incl. an ever-growing history) + // on every write was a primary long-running main-thread stall. + _serializeForDisk(config) { + const hosters = this._clone(config.hosters || {}); + secretStore.encryptCredentials({ hosters }); + return JSON.stringify({ ...config, hosters }, null, 2); + } + + _commit(config) { + if (!this._perfLog) return this._atomicWrite(this._serializeForDisk(config)); + const t0 = performance.now(); + const data = this._serializeForDisk(config); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = this._pqLen(config.globalSettings); + const h = (config.history || []).length; + this._perfLog(`config-serialize wall=${dt.toFixed(0)}ms bytes=${data.length} hist=${h} queue=${q} wqDepth=${this._wqDepth} via=${this._callerTag()}`); + } + return this._atomicWrite(data); + } + + _enqueueWrite(fn) { + this._wqDepth++; + const done = () => { this._wqDepth--; }; + this._writeQueue = this._writeQueue.then(fn, fn).then(done, done); + return this._writeQueue; + } + + _anyHosters(cfg) { + const h = cfg && cfg.hosters; + return !!h && typeof h === 'object' && Object.values(h).some(a => Array.isArray(a) && a.length > 0); + } + + _recoverHostersFromDisk() { + for (const p of [this.filePath, this.filePath + '.bak', this.filePath + '.pre-history-split.bak']) { + try { + const raw = fs.readFileSync(p, 'utf-8'); + if (!raw || raw.trim().length < 2) continue; + const data = JSON.parse(raw); + if (this._anyHosters(data)) return data.hosters; + } catch {} + } + return null; + } + + _guardHosters(current, hostersIntentional) { + if (!hostersIntentional && !this._anyHosters(current)) { + const recovered = this._recoverHostersFromDisk(); + if (recovered) { + current.hosters = recovered; + if (this._perfLog) this._perfLog('config-guard: prevented account wipe — restored hosters from on-disk backup after a corrupt/empty read'); + } + } + return current; + } + + save(config) { + return this._enqueueWrite(() => { + const current = this.load(); + if (config.hosters) current.hosters = config.hosters; + if (config.hosterSettings) current.hosterSettings = config.hosterSettings; + if (config.globalSettings) current.globalSettings = config.globalSettings; + this._guardHosters(current, !!config.hosters); + return this._commit(current); + }); + } + + loadHistory() { + if (this._historyMigrated) { + return this._readHistoryFile() || []; + } + const config = this.load(); + return config.history || []; + } + + _atomicWrite(data) { + return new Promise((resolve, reject) => { + const tmpPath = this.filePath + '.tmp'; + const backupPath = this.filePath + '.bak'; + let fd; + try { + fd = fs.openSync(tmpPath, 'w'); + fs.writeSync(fd, data); + fs.fsyncSync(fd); + } catch (e) { + try { if (fd !== undefined) fs.closeSync(fd); } catch {} + return reject(e); + } + try { fs.closeSync(fd); } catch {} + Promise.resolve().then(() => { + try { + try { + if (fs.existsSync(this.filePath)) { + const cur = fs.readFileSync(this.filePath, 'utf-8'); + if (cur && cur.trim().length > 2) fs.writeFileSync(backupPath, cur, 'utf-8'); + } + } catch {} + fs.renameSync(tmpPath, this.filePath); + } catch (e) { return reject(e); } + // Invalidate the read cache: the next load() re-reads + re-merges the + // freshly-written file (the on-disk format is sparse — load() fills + // defaults — so we must NOT serve a pre-merge in-memory object). + this._cache = null; + this._cacheKey = ''; + resolve(); + }); + }); + } + + appendHistory(entry) { + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => { + const cur = this._readHistoryFile(); + if (cur === null && fs.existsSync(this.historyPath)) return; + const arr = cur || []; + arr.push(entry); + const gs = this.load().globalSettings; + const retention = (gs && gs.historyRetention) || 'all'; + const pruned = applyHistoryRetention(arr, retention, Date.now()); + return this._writeHistoryFileAtomic(pruned); + }); + } + return this._enqueueWrite(() => { + const config = this.load(); + config.history.push(entry); + const retention = (config.globalSettings && config.globalSettings.historyRetention) || 'all'; + config.history = applyHistoryRetention(config.history, retention, Date.now()); + return this._commit(config); + }); + } + + pruneHistory(retention, opts = {}) { + const dryRun = !!opts.dryRun; + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => { + const current = this._readHistoryFile() || []; + const beforeBatches = current.length; + const beforeRows = countHistoryRows(current); + const pruned = applyHistoryRetention(current, retention, Date.now()); + const result = { + removedBatches: beforeBatches - pruned.length, + removedRows: beforeRows - countHistoryRows(pruned), + keptBatches: pruned.length, + keptRows: countHistoryRows(pruned) + }; + if (dryRun) return result; + return this._writeHistoryFileAtomic(pruned) + .then(() => this.save({ globalSettings: { ...this.load().globalSettings, historyRetention: String(retention || 'all') } })) + .then(() => result); + }); + } + return this._enqueueWrite(() => { + const config = this.load(); + const beforeBatches = config.history.length; + const beforeRows = countHistoryRows(config.history); + const pruned = applyHistoryRetention(config.history, retention, Date.now()); + const result = { + removedBatches: beforeBatches - pruned.length, + removedRows: beforeRows - countHistoryRows(pruned), + keptBatches: pruned.length, + keptRows: countHistoryRows(pruned) + }; + if (dryRun) return result; + config.history = pruned; + if (config.globalSettings) config.globalSettings.historyRetention = String(retention || 'all'); + return this._commit(config).then(() => result); + }); + } + + clearHistory() { + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => this._writeHistoryFileAtomic([])); + } + return this._enqueueWrite(() => { + const config = this.load(); + config.history = []; + return this._commit(config); + }); + } + + saveRotationCursors(cursors) { + return this._enqueueWrite(() => { + const config = this.load(); + config.rotationCursors = (cursors && typeof cursors === 'object' && !Array.isArray(cursors)) ? cursors : {}; + this._guardHosters(config, false); + return this._commit(config); + }); + } +} + +module.exports = ConfigStore; +module.exports.HOSTER_ACCOUNT_TEMPLATES = HOSTER_ACCOUNT_TEMPLATES; +module.exports.HOSTER_NAMES = HOSTER_NAMES; +module.exports.HOSTER_ADD_OPTIONS = HOSTER_ADD_OPTIONS; +module.exports.HISTORY_RETENTION_OPTIONS = HISTORY_RETENTION_OPTIONS; +module.exports.applyHistoryRetention = applyHistoryRetention; +module.exports.countHistoryRows = countHistoryRows; diff --git a/lib/diagnostics-agent.js b/lib/diagnostics-agent.js new file mode 100644 index 0000000..f5c2239 --- /dev/null +++ b/lib/diagnostics-agent.js @@ -0,0 +1,32 @@ +function createAgent(collectors) { + const OPS = { + get_system_info: (a) => collectors.getSystemInfo(a), + server_health: (a) => collectors.serverHealth(a), + get_config_redacted: (a) => collectors.getConfigRedacted(a), + list_logs: () => collectors.listLogs(), + read_log: (a) => collectors.readLog(a), + tail_log: (a) => collectors.readLog(a), + get_app_events: (a) => collectors.getAppEvents(a), + list_errors: (a) => collectors.listErrors(a), + get_queue_state: (a) => collectors.getQueueState(a), + get_history: (a) => collectors.getHistory(a), + get_rotation_state: () => collectors.getRotationState(), + get_health: () => collectors.getHealth() + }; + + function handle(op, args) { + const fn = (typeof op === 'string' && Object.prototype.hasOwnProperty.call(OPS, op)) ? OPS[op] : null; + if (typeof fn !== 'function') return { ok: false, error: `unknown or non-readonly op: ${op}` }; + try { + const data = fn(args || {}); + if (data && data.ok === false) return data; + return { ok: true, data }; + } catch (e) { + return { ok: false, error: String((e && e.message) || e) }; + } + } + + return { handle, ops: Object.keys(OPS) }; +} + +module.exports = { createAgent }; diff --git a/lib/diagnostics-collectors.js b/lib/diagnostics-collectors.js new file mode 100644 index 0000000..dcfaf23 --- /dev/null +++ b/lib/diagnostics-collectors.js @@ -0,0 +1,277 @@ +const fs = require('fs'); +const path = require('path'); + +const READABLE_LOGS = { + debug: 'debug', + fileuploader: 'fileuploader', + accountRotation: 'accountRotation', + crash: 'crashLog' +}; + +const QUEUE_STATUSES = ['preview', 'queued', 'getting-server', 'uploading', 'retrying', 'done', 'error', 'aborted', 'skipped']; + +function createCollectors(deps) { + const { loadConfig, loadHistory, getAllLogPaths, support, stats, appInfo, systemInfo, agentInfo } = deps; + + function _secrets() { + try { return support.collectSecretValues(loadConfig()); } catch { return []; } + } + + function _deepRedact(value, secrets) { + const s = secrets || _secrets(); + const walk = (v) => { + if (typeof v === 'string') return support.redactLogText(v, s); + if (Array.isArray(v)) return v.map(walk); + if (v && typeof v === 'object') { + const o = {}; + for (const k of Object.keys(v)) o[k] = walk(v[k]); + return o; + } + return v; + }; + try { return walk(value); } catch { return value; } + } + + function _resolveLogPath(name, backup) { + const key = READABLE_LOGS[name]; + if (!key) return null; + const paths = getAllLogPaths(); + let p = paths[key]; + if (!p) return null; + if (backup === 1 || backup === 2) p = `${p}.${backup}`; + return p; + } + + function getSystemInfo() { + return { app: appInfo(), system: systemInfo(), agent: agentInfo() }; + } + + function getConfigRedacted(args) { + const section = (args && args.section) || 'all'; + const cfg = loadConfig(); + const secrets = support.collectSecretValues(cfg); + const sanitized = support.sanitizeConfig(cfg); + let pick; + let note; + if (section === 'all') { + pick = { ...sanitized }; + delete pick.history; + note = 'history omitted from config — use get_history'; + } else { + pick = sanitized[section] !== undefined ? sanitized[section] : null; + } + return { section, note, config: _deepRedact(pick, secrets) }; + } + + function listLogs() { + const paths = getAllLogPaths(); + const dir = paths.logDir; + const files = []; + for (const [name, key] of Object.entries(READABLE_LOGS)) { + const base = paths[key]; + if (!base) continue; + const variants = []; + for (const suffix of ['', '.1', '.2']) { + const fp = base + suffix; + try { + const st = fs.statSync(fp); + variants.push({ backup: suffix === '' ? 0 : Number(suffix.slice(1)), sizeBytes: st.size, mtime: st.mtime.toISOString() }); + } catch {} + } + files.push({ name, path: base, readable: true, present: variants.length > 0, variants }); + } + let siblings = []; + try { + siblings = fs.readdirSync(dir) + .filter(f => /\.log(\.\d+)?$/i.test(f)) + .filter(f => !files.some(x => path.basename(x.path) === f || f.startsWith(path.basename(x.path)))); + siblings = siblings.map(f => { + let size = 0, mtime = null; + try { const st = fs.statSync(path.join(dir, f)); size = st.size; mtime = st.mtime.toISOString(); } catch {} + return { name: f, readable: false, sizeBytes: size, mtime }; + }); + } catch {} + return { dir, files, otherLogs: siblings }; + } + + function readLog(args) { + const a = args || {}; + const name = a.name; + const p = _resolveLogPath(name, a.backup); + if (!p) return { ok: false, error: `unknown or non-readable log: ${name}` }; + const tailKb = Math.min(Math.max(Number(a.tailKb) || 256, 1), 1024); + const raw = support.collectFile(p, name, tailKb * 1024); + let content = support.redactLogText(raw, _secrets()); + let matchedLines; + if (a.grep && typeof a.grep === 'string' && a.grep.length <= 200) { + const terms = a.grep.split('|').map(s => s.trim().toLowerCase()).filter(Boolean); + if (terms.length) { + const lines = content.split('\n').filter(l => { + const low = l.toLowerCase(); + return terms.some(t => low.includes(t)); + }); + matchedLines = lines.length; + content = lines.join('\n'); + } + } + let sizeBytes = null; + try { sizeBytes = fs.statSync(p).size; } catch {} + return { name, path: p, sizeBytes, returnedBytes: Buffer.byteLength(content), tailKb, matchedLines, content }; + } + + function getAppEvents(args) { + const limit = Math.min(Math.max(Number(args && args.limit) || 50, 1), 500); + const out = []; + const secrets = _secrets(); + for (const name of ['crash', 'debug']) { + const p = _resolveLogPath(name); + if (!p) continue; + const raw = support.redactLogText(support.collectFile(p, name, 256 * 1024), secrets); + const lines = raw.split('\n').filter(l => l.trim() && !l.startsWith('===')); + for (const line of lines.slice(-limit)) out.push({ source: name, text: line }); + } + return { events: out.slice(-limit), truncated: out.length > limit }; + } + + function _historyErrors(history, opts) { + const o = opts || {}; + const sinceMs = Number.isFinite(o.sinceMs) ? o.sinceMs : null; + const secrets = _secrets(); + const errors = []; + const byCategory = {}; + for (const batch of (Array.isArray(history) ? history : [])) { + if (!batch || !Array.isArray(batch.files)) continue; + const ts = batch.timestamp ? Date.parse(batch.timestamp) : null; + if (sinceMs !== null && ts !== null && ts < sinceMs) continue; + for (const file of batch.files) { + if (!file || !Array.isArray(file.results)) continue; + for (const r of file.results) { + if (!r || r.status === 'done') continue; + const category = stats.classifyErrorCategory(r.error); + if (o.category && o.category !== category) continue; + if (o.hoster && o.hoster !== r.hoster) continue; + byCategory[category] = (byCategory[category] || 0) + 1; + errors.push({ + ts: batch.timestamp || null, + fileName: file.name || file.fileName || '', + hoster: r.hoster || '', + accountId: r.accountId || undefined, + category, + error: support.redactLogText(String(r.error || ''), secrets) + }); + } + } + } + return { errors, byCategory }; + } + + function listErrors(args) { + const a = args || {}; + const cfg = loadConfig(); + const { errors, byCategory } = _historyErrors(cfg.history, a); + const limit = Math.min(Math.max(Number(a.limit) || 100, 1), 1000); + const window = Number.isFinite(a.sinceMs) ? `since ${new Date(a.sinceMs).toISOString()}` : 'all history'; + return { window, total: errors.length, byCategory, errors: errors.slice(-limit) }; + } + + function getQueueState(args) { + const a = args || {}; + const cfg = loadConfig(); + const pending = cfg.globalSettings && cfg.globalSettings.pendingQueue; + if (!pending || typeof pending !== 'object') { + return { source: 'empty', stale: false, counts: {}, selectedHosters: [] }; + } + const counts = {}; + for (const s of QUEUE_STATUSES) counts[s] = 0; + const jobs = Array.isArray(pending.queueJobs) ? pending.queueJobs : []; + for (const j of jobs) { if (counts[j.status] !== undefined) counts[j.status]++; } + const result = { + source: 'persisted', + stale: true, + savedAt: pending.savedAt || null, + selectedHosters: Array.isArray(pending.selectedUploadHosters) ? pending.selectedUploadHosters : [], + fileCount: Array.isArray(pending.selectedFiles) ? pending.selectedFiles.length : 0, + counts + }; + if (a.includeJobs !== false) { + const maxJobs = Math.min(Math.max(Number(a.maxJobs) || 200, 1), 2000); + result.jobs = _deepRedact(jobs.slice(0, maxJobs).map(j => ({ + file: j.file, fileName: j.fileName, hoster: j.hoster, status: j.status, error: j.error || null + }))); + result.jobsTruncated = jobs.length > maxJobs; + } + return result; + } + + function getHistory(args) { + const a = args || {}; + const history = typeof loadHistory === 'function' + ? (loadHistory() || []) + : (Array.isArray(loadConfig().history) ? loadConfig().history : []); + const limit = Math.min(Math.max(Number(a.limit) || 20, 1), 200); + const perHoster = stats.summarizePerHoster(history); + const recent = [...history].slice(-limit).reverse(); + const secrets = _secrets(); + const batches = recent.map(b => { + const out = { timestamp: b.timestamp || null, fileCount: Array.isArray(b.files) ? b.files.length : 0 }; + if (a.includeFiles) { + out.files = (b.files || []).map(f => ({ + name: f.name || f.fileName || '', + results: (f.results || []).map(r => { + const rr = { hoster: r.hoster, status: r.status }; + if (r.error) rr.error = support.redactLogText(String(r.error), secrets); + if (a.includeUrls && r.url) rr.url = r.url; + return rr; + }) + })); + } + return out; + }); + return { totalBatches: history.length, returned: batches.length, perHoster, batches }; + } + + function getRotationState() { + const cfg = loadConfig(); + return { rotationCursors: _deepRedact(cfg.rotationCursors || {}) }; + } + + function getHealth() { + const cfg = loadConfig(); + const hosters = cfg.hosters && typeof cfg.hosters === 'object' ? Object.keys(cfg.hosters).filter(h => Array.isArray(cfg.hosters[h]) && cfg.hosters[h].length > 0) : []; + return { + reachabilityKnown: false, + hint: 'Live hoster probing (run_health_check) is disabled in this build. Configured hosters with at least one account are listed.', + configuredHosters: hosters + }; + } + + function serverHealth(args) { + const a = args || {}; + const errorLimit = Math.min(Math.max(Number(a.errorLimit) || 20, 1), 200); + const errArgs = Number.isFinite(a.errorSinceMs) ? { sinceMs: a.errorSinceMs, limit: errorLimit } : { limit: errorLimit }; + const errors = listErrors(errArgs); + const queue = getQueueState({ includeJobs: false }); + const history = getHistory({ limit: 5 }); + const warnings = []; + if (queue.source === 'persisted' && queue.stale) warnings.push('queue state is from the persisted snapshot (may lag live state; UploadManager not introspected in this build).'); + if (errors.total > 0) warnings.push(`${errors.total} non-success result(s) in the error window.`); + return { + server: getSystemInfo(), + queue, + recentBatches: history.batches, + perHoster: history.perHoster, + errors, + hosters: getHealth(), + logs: listLogs(), + warnings + }; + } + + return { + getSystemInfo, getConfigRedacted, listLogs, readLog, getAppEvents, + listErrors, getQueueState, getHistory, getRotationState, getHealth, serverHealth, + READABLE_LOGS + }; +} + +module.exports = { createCollectors, READABLE_LOGS }; diff --git a/lib/doodstream-upload.js b/lib/doodstream-upload.js new file mode 100644 index 0000000..1c3821b --- /dev/null +++ b/lib/doodstream-upload.js @@ -0,0 +1,705 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request } = require('undici'); + +const BASE_URL = 'https://doodstream.com'; +const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const UPLOAD_TIMEOUT = 1800000; // 30 min + +// Cap doodstream's per-hoster debug log alongside the main log files so +// dev-mode sessions don't accumulate gigabytes of upload trace. +const { maybeRotateLogFile } = require('./log-rotation'); +const _DOODSTREAM_LOG_MAX_BYTES = 10 * 1024 * 1024; +const _DOODSTREAM_LOG_MAX_BACKUPS = 1; + +// Resolve the log path at write-time. In a packaged build __dirname lives +// inside app.asar (read-only) — writing there fails silently and we lose every +// production trace. Prefer Electron's writable userData dir, fall back to the +// repo root only when running outside Electron (tests / plain node). +function _doodstreamLogPath() { + try { + const { app } = require('electron'); + if (app && typeof app.getPath === 'function') { + return path.join(app.getPath('userData'), 'doodstream-debug.log'); + } + } catch { /* not running under Electron */ } + return path.join(__dirname, '..', 'doodstream-debug.log'); +} + +let _debugVerbose = false; +function setDebugVerbose(v) { _debugVerbose = !!v; } + +function _debugLog(msg) { + if (!_debugVerbose) return; + try { + const logPath = _doodstreamLogPath(); + maybeRotateLogFile(logPath, _DOODSTREAM_LOG_MAX_BYTES, _DOODSTREAM_LOG_MAX_BACKUPS); + const ts = new Date().toISOString(); + fs.appendFileSync(logPath, `[${ts}] ${msg}\n`); + } catch {} +} + +class DoodstreamUploader { + constructor() { + this.cookies = new Map(); + this.sessId = ''; + this.apiKey = ''; // optionally derived from the logged-in session (deriveApiKey) + } + + _cookieHeader() { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + _parseCookiesFromHeaders(headers) { + let setCookies; + if (typeof headers.getSetCookie === 'function') { + setCookies = headers.getSetCookie(); + } else if (headers['set-cookie']) { + setCookies = Array.isArray(headers['set-cookie']) ? headers['set-cookie'] : [headers['set-cookie']]; + } else { + return; + } + for (const raw of setCookies) { + const pair = raw.split(';')[0]; + const eq = pair.indexOf('='); + if (eq > 0) { + this.cookies.set(pair.substring(0, eq).trim(), pair.substring(eq + 1).trim()); + } + } + } + + async _fetch(url, opts = {}, _redirectCount = 0) { + const MAX_REDIRECTS = 10; + const headers = { + 'User-Agent': USER_AGENT, + ...(opts.headers || {}) + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + // The small discovery/result requests that bookend a multi-minute upload + // occasionally hit a transient blip ("fetch failed", ECONNRESET, a hung TLS + // handshake). A blip here shouldn't throw away the whole upload, so retry a + // few times with short backoff. Each attempt gets its own 20s timeout — + // Node's fetch has none by default, and a hung socket would otherwise stall + // the attempt for minutes. The big file upload (undici) is retried at the + // upload-manager level, not here. + let res; + for (let attempt = 1; ; attempt++) { + const timeoutSignal = AbortSignal.timeout(20000); + const signal = opts.signal ? AbortSignal.any([opts.signal, timeoutSignal]) : timeoutSignal; + try { + res = await fetch(url, { ...opts, headers, redirect: 'manual', signal }); + break; + } catch (err) { + if (opts.signal && opts.signal.aborted) throw err; // caller abort: don't retry + if (attempt >= 3) throw err; + _debugLog(`_fetch transient (${attempt}/3) ${url}: ${err && err.message}; retry`); + await new Promise(r => setTimeout(r, 400 * attempt)); + } + } + + this._parseCookiesFromHeaders(res.headers); + + if ([301, 302, 303, 307, 308].includes(res.status)) { + try { await res.text(); } catch {} + if (_redirectCount >= MAX_REDIRECTS) throw new Error('Zu viele Redirects'); + const location = res.headers.get('location'); + if (location) { + const nextUrl = new URL(location, url).href; + return this._fetch(nextUrl, { ...opts, method: 'GET', body: undefined }, _redirectCount + 1); + } + } + + return res; + } + + /** + * Login to DoodStream via web form + */ + async login(username, password, otp) { + // GET homepage first to collect cookies + const homeRes = await this._fetch(BASE_URL); + await homeRes.text(); + + // POST login via AJAX (op in body, XHR header required for JSON response) + const loginData = new URLSearchParams({ + op: 'login_ajax', + login: username, + password: password, + loginotp: otp || '' + }); + + // Use raw fetch with redirect: 'manual' to detect success redirects + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Referer': BASE_URL + '/', + 'X-Requested-With': 'XMLHttpRequest', + 'User-Agent': USER_AGENT + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + const res = await fetch(BASE_URL + '/', { + method: 'POST', + body: loginData.toString(), + headers, + redirect: 'manual' + }); + + this._parseCookiesFromHeaders(res.headers); + + // On successful login, server may redirect (3xx) to dashboard + if ([301, 302, 303, 307, 308].includes(res.status)) { + try { await res.text(); } catch {} + // Redirect means login succeeded + } else { + const body = await res.text(); + let json; + try { json = JSON.parse(body); } catch { json = null; } + + if (json && json.status === 'success') { + // Explicit success response + } else if (json && json.message && /otp/i.test(json.message)) { + // OTP required — signal caller to collect OTP from user + const err = new Error(`Doodstream Login: ${json.message}`); + err.otpRequired = true; + throw err; + } else if (json && json.status === 'fail') { + throw new Error(`Doodstream Login: ${json.message || 'Login fehlgeschlagen'}`); + } else if (body.includes('Dashboard')) { + // Got dashboard HTML directly — login worked + } else { + const msg = (json && json.message) || 'Login fehlgeschlagen'; + throw new Error(`Doodstream Login: ${msg}`); + } + } + + // Extract sess_id from the upload page + await this._extractSessId(); + } + + async _extractSessId() { + const res = await this._fetch(BASE_URL + '/?op=upload'); + const html = await res.text(); + + // Hidden input: + const hiddenMatch = html.match(/name=["']sess_id["'][^>]*value=["']([a-zA-Z0-9]+)["']/); + if (hiddenMatch) { + this.sessId = hiddenMatch[1]; + return; + } + + // Vue component prop or JS: sess_id: "xxx" or sess_id="xxx" + const sessMatch = html.match(/sess_id['":\s]+['"]([a-zA-Z0-9]+)['"]/); + if (sessMatch) { + this.sessId = sessMatch[1]; + return; + } + + // Assignment: sess_id = 'xxx' + const altMatch = html.match(/sess_id\s*=\s*['"]([a-zA-Z0-9]+)['"]/); + if (altMatch) { + this.sessId = altMatch[1]; + return; + } + + throw new Error('Doodstream: sess_id nicht gefunden nach Login'); + } + + /** + * Get upload server URL from web interface + */ + async _getUploadServer() { + // Use the standard upload server endpoint + const res = await this._fetch(BASE_URL + '/?op=upload_server'); + const text = await res.text(); + const ctype = (res.headers && res.headers.get) ? (res.headers.get('content-type') || '') : ''; + _debugLog(`upload_server: status=${res.status} ctype=${ctype} body(800)=${(text || '').slice(0, 800)}`); + let json; + try { json = JSON.parse(text); } catch { json = null; } + + if (json && json.result && /^https?:\/\//i.test(json.result)) { + return json.result; + } + + // Fallback: try fetching from upload page HTML + const pageRes = await this._fetch(BASE_URL + '/?op=upload'); + const html = await pageRes.text(); + + // Current doodstream format: the upload server is the action of the + // multipart upload form, e.g. + //
+ // + // The node is assigned per page-load and the action carries a session token + // in its query string that matches the page's hidden sess_id. We refresh + // this.sessId from THIS page so the multipart sess_id field matches the node + // URL — login-time and node tokens otherwise diverge and the upload comes + // back with an empty filecode. + const actionMatch = html.match(/action=["'](https?:\/\/[^"']+\/upload\/[^"']*)["']/i); + if (actionMatch) { + const url = actionMatch[1].replace(/&/g, '&'); // un-escape HTML entities in query + const freshSess = html.match(/name=["']sess_id["'][^>]*value=["']([a-zA-Z0-9]+)["']/); + if (freshSess) { + this.sessId = freshSess[1]; + } else { + _debugLog('upload_server: form action found but no sess_id on page; keeping existing sessId'); + } + // Capture the form's real fields so upload() submits exactly what the + // browser would (file_title, submit_btn, …) instead of stale hardcoded ones. + this._uploadFormFields = this._parseUploadFormFields(html); + _debugLog(`upload_server: using form action node=${url} sess=${this.sessId} fields=${Object.keys(this._uploadFormFields).join(',')}`); + return url; + } + + // Legacy fallback: srv_url JS variable (older doodstream theme). + const srvMatch = html.match(/srv_url['":\s]+['"]?(https?:\/\/[^'">\s]+)['"]?/i); + if (srvMatch) return srvMatch[1]; + + // No upload server could be extracted. We MUST NOT silently fall back to a + // hardcoded node: that node is stale and accepts the bytes but returns an + // empty form (no filecode) — so the user wastes ~90s uploading 95 MB into a + // dead end and gets a cryptic "kein Filecode" 90s later. Fail fast and put + // the raw responses in the error so the real format change is diagnosable. + const urlHints = (html.match(/https?:\/\/[^'">\s]+/g) || []).slice(0, 4).join(' , '); + _debugLog(`upload_server: NO SERVER. upload-page html(2000)=${(html || '').slice(0, 2000)}`); + throw new Error( + `Doodstream: konnte Upload-Server nicht ermitteln (Endpoint geaendert?). ` + + `op=upload_server status=${res.status} ctype=${ctype} body=${(text || '').slice(0, 300)} ` + + `| upload-page URL-Treffer: ${urlHints || 'keine'}` + ); + } + + /** + * Replicate the non-file fields of doodstream's CURRENT upload form so our + * POST matches what the browser actually submits. Doodstream dropped the old + * `utype` field and added file_title / fakefilepc / submit_btn; submitting a + * stale/incomplete field set can make the node accept the bytes but skip + * registration (→ empty result form). We parse the live form rather than + * hardcode, so we track whatever fields doodstream uses now. The file input + * (type=file) is excluded — the file is streamed separately. + */ + _parseUploadFormFields(html) { + const fields = {}; + if (!html) return fields; + // Narrow to the upload form (its action points at a /upload/ node). + const formMatch = html.match(/]*\baction=["'][^"']*\/upload\/[^"']*["'][\s\S]*?<\/form>/i); + const scope = formMatch ? formMatch[0] : html; + const re = /<(?:input|button)\b([^>]*)>/gi; + let m; + while ((m = re.exec(scope)) !== null) { + const attrs = m[1]; + const typeM = attrs.match(/\btype=["']([^"']*)["']/i); + if (typeM && typeM[1].toLowerCase() === 'file') continue; + const nameM = attrs.match(/\bname=["']([^"']+)["']/i); + if (!nameM) continue; + const valM = attrs.match(/\bvalue=["']([^"']*)["']/i); + fields[nameM[1]] = valM ? valM[1] : ''; + } + return fields; + } + + /** + * Upload file using web session + */ + async upload(filePath, progressCb, signal, throttle) { + const fileName = path.basename(filePath); + const fileSize = fs.statSync(filePath).size; + + // Get upload server + const uploadUrl = await this._getUploadServer(); + // Remember which CDN node handled this upload so a later parse failure can + // report it — failures sometimes correlate with a specific node. + this._lastUploadUrl = uploadUrl; + + // Build multipart form + const boundary = `----WebKitFormBoundary${crypto.randomBytes(16).toString('hex')}`; + + // Build form parts. Submit the live form's fields (parsed in + // _getUploadServer) so our POST matches the browser; merge in sess_id (the + // fresh node token) and keep utype=reg as a harmless compatibility extra. + // Falls back to the minimal known-good set if the form wasn't parsed. + const formFields = { utype: 'reg', ...(this._uploadFormFields || {}) }; + formFields.sess_id = this.sessId; + let preamble = ''; + for (const [name, value] of Object.entries(formFields)) { + preamble += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`; + } + const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + preamble += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${safeFileName}"\r\nContent-Type: application/octet-stream\r\n\r\n`; + + const epilogue = `\r\n--${boundary}--\r\n`; + const preambleBuf = Buffer.from(preamble, 'utf-8'); + const epilogueBuf = Buffer.from(epilogue, 'utf-8'); + const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; + + const CHUNK_SIZE = 1024 * 1024; + let bytesRead = 0; + + async function* generate() { + yield preambleBuf; + const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE }); + for await (const chunk of fileStream) { + if (signal && signal.aborted) throw new Error('Aborted'); + if (throttle) await throttle.consume(chunk.length, signal); + bytesRead += chunk.length; + yield chunk; + if (progressCb) progressCb(bytesRead, fileSize); + } + yield epilogueBuf; + } + + let uploadRes; + try { + uploadRes = await request(uploadUrl, { + method: 'POST', + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize), + 'User-Agent': USER_AGENT, + 'Cookie': this._cookieHeader() + }, + body: generate(), + signal, + bodyTimeout: UPLOAD_TIMEOUT, + headersTimeout: 60000 + }); + } catch (err) { + // Label which phase failed so a future "fetch failed"/"terminated" is + // attributable to the big upload POST vs the small bookend requests. The + // original message is preserved as a substring so upload-manager's + // transient classification still matches. NOTE: undici may surface + // "terminated"/"other side closed", which are not yet in that transient + // list — revisit if logs show them. + const mb = Math.round(bytesRead / 1048576); + throw new Error(`Doodstream Upload-POST (${mb} MB an ${uploadUrl}): ${err && err.message ? err.message : err}`); + } + + const statusCode = uploadRes.statusCode; + _debugLog(`Upload response status: ${statusCode}`); + + // Handle redirects from upload server (undici doesn't follow them) + if ([301, 302, 303, 307, 308].includes(statusCode)) { + const location = uploadRes.headers['location']; + try { await uploadRes.body.text(); } catch {} + _debugLog(`Upload redirect to: ${location}`); + if (location) { + return this._handleUploadResult(location); + } + } + + const resText = await uploadRes.body.text(); + _debugLog(`Upload response body (first 500): ${resText.slice(0, 500)}`); + + if (statusCode >= 400) { + let payload; + try { payload = JSON.parse(resText); } catch {} + const msg = payload && payload.msg ? payload.msg : resText.slice(0, 200); + throw new Error(`Doodstream Upload HTTP ${statusCode}: ${msg}`); + } + + return this._parseUploadResponse(resText); + } + + /** + * Follow a redirect URL from upload server and extract filecode + */ + async _handleUploadResult(url) { + _debugLog(`Following upload result URL: ${url}`); + const res = await this._fetch(url); + const html = await res.text(); + _debugLog(`Result page (first 500): ${html.slice(0, 500)}`); + return this._parseUploadResponse(html); + } + + /** + * Extract hidden form fields from HTML (handles various attribute orders) + */ + _extractHiddenFields(html) { + const fields = {}; + // Textarea fields: + const ta = /]*name=['"]([^'"]+)['"][^>]*>([\s\S]*?)<\/textarea>/gi; + let m; + while ((m = ta.exec(html)) !== null) fields[m[1]] = m[2].trim(); + // Input hidden fields + const p1 = /]*type=['"]hidden['"][^>]*name=['"]([^'"]+)['"][^>]*value=['"]([^'"]*)['"]/gi; + while ((m = p1.exec(html)) !== null) { if (!fields[m[1]]) fields[m[1]] = m[2]; } + const p2 = /]*name=['"]([^'"]+)['"][^>]*value=['"]([^'"]*)['"]/gi; + while ((m = p2.exec(html)) !== null) { if (!fields[m[1]]) fields[m[1]] = m[2]; } + const p3 = /]*value=['"]([^'"]*)['"]\s[^>]*name=['"]([^'"]+)['"]/gi; + while ((m = p3.exec(html)) !== null) { if (!fields[m[2]]) fields[m[2]] = m[1]; } + return fields; + } + + /** + * Parse filecode from upload server response (JSON or HTML) + */ + async _parseUploadResponse(resText) { + // 1. Try JSON + let payload; + try { payload = JSON.parse(resText); } catch {} + + if (payload) { + return this._extractFromJson(payload); + } + + // 2. Try filecode directly in HTML + const code = this._findFilecodeInHtml(resText); + if (code) { + _debugLog(`Found filecode in HTML: ${code}`); + return this._buildResult(code); + } + + // 3. Parse HTML form (XFileSharing two-step upload) + const hiddenFields = this._extractHiddenFields(resText); + _debugLog(`Hidden fields: ${JSON.stringify(hiddenFields)}`); + + // Check if filecode is already in hidden fields + const fnCode = hiddenFields.fn || hiddenFields.filecode || hiddenFields.file_code; + if (fnCode && fnCode.length >= 8) { + _debugLog(`Filecode from hidden field 'fn': ${fnCode}`); + // We still need to submit the form so doodstream registers the file + // But the filecode is the 'fn' value + } + + // XFileSharing standard: form with op=upload_result, fn, st + // Always submit to doodstream.com, not to CDN + if (hiddenFields.fn || hiddenFields.op === 'upload_result') { + // Ensure op=upload_result is set + if (!hiddenFields.op) hiddenFields.op = 'upload_result'; + + _debugLog(`Submitting upload_result to ${BASE_URL}/ with fields: ${JSON.stringify(hiddenFields)}`); + const formData = new URLSearchParams(hiddenFields); + let followText = ''; + try { + const followRes = await this._fetch(BASE_URL + '/', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Referer': BASE_URL + '/' + }, + body: formData.toString() + }); + followText = await followRes.text(); + } catch (err) { + // The file already uploaded to the CDN; this POST only registers it on + // doodstream's side. If it fails transiently (even after _fetch's own + // retries) but we already hold the filecode, the upload succeeded from + // the user's view — return it rather than discarding a done upload. + if (fnCode && fnCode.length >= 8) { + _debugLog(`upload_result submit failed (${err && err.message}); using fn ${fnCode}`); + return this._buildResult(fnCode); + } + throw err; + } + _debugLog(`upload_result response (first 500): ${followText.slice(0, 500)}`); + + // Try to find filecode in result page + const resultCode = this._findFilecodeInHtml(followText); + if (resultCode) { + return this._buildResult(resultCode); + } + + // If we had fn from hidden fields, use that as filecode + if (fnCode && fnCode.length >= 8) { + return this._buildResult(fnCode); + } + + // Try download URL pattern in result page + const dlMatch = followText.match(/https?:\/\/[a-z0-9.]+\/d\/([a-zA-Z0-9]+)/i); + if (dlMatch) { + return this._buildResult(dlMatch[1]); + } + + // No filecode anywhere. Surface WHY: XFileSharing puts the real reason + // in the `st` field (anything other than "OK" means the backend refused + // the file — copyright/hash match, duplicate, size, quota, …). The + // download link being empty while the page structure is unchanged points + // at doodstream's backend, not at a parsing bug on our side. + const st = hiddenFields.st || ''; + const fnInfo = fnCode ? `"${fnCode}"(len ${fnCode.length})` : 'fehlt/leer'; + const node = this._lastUploadUrl || '?'; + _debugLog(`No filecode. st=${st} fn=${fnInfo} node=${node} CDN-body=${(resText || '').slice(0, 400)}`); + if (st && st !== 'OK') { + throw new Error(`Doodstream lehnt Datei ab (Server-Status: ${st}). CDN=${node}`); + } + // Empty form (no fn, no st) is a doodstream-side processing flake — same + // account + same file works on a later attempt. Tag it explicitly so the + // upload-manager classifies this as a hoster-transient error and does NOT + // blacklist the account (otherwise one of these flakes poisons the whole + // session and later batches hit `pre-job-swap-blocked` for no fault of + // the account). The flag is the primary signal; the message text is a + // belt-and-suspenders regex fallback in the classifier. + const emptyLinkErr = new Error(`Doodstream Upload: kein Filecode — Server gab leeren Link zurueck (st=${st || '?'}, fn=${fnInfo}, CDN=${node}). CDN-Antwort: ${(resText || '').slice(0, 200)}`); + emptyLinkErr.hosterTransient = true; + throw emptyLinkErr; + } + + // 4. Fallback: follow form action as-is (for non-XFS forms) + const formAction = resText.match(/]*action=['"]([^'"]+)['"]/i); + if (formAction) { + _debugLog(`Fallback: following form action ${formAction[1]}`); + const formData = new URLSearchParams(hiddenFields); + const followRes = await this._fetch(formAction[1], { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Referer': BASE_URL + '/' + }, + body: formData.toString() + }); + const followText = await followRes.text(); + _debugLog(`Fallback response (first 500): ${followText.slice(0, 500)}`); + + const fallbackCode = this._findFilecodeInHtml(followText); + if (fallbackCode) return this._buildResult(fallbackCode); + + // Check if fn was in original hidden fields + if (fnCode && fnCode.length >= 8) return this._buildResult(fnCode); + + throw new Error(`Doodstream Upload: Redirect-Antwort ungueltig (${followText.slice(0, 150)})`); + } + + throw new Error(`Doodstream Upload: Keine gueltige Antwort (Body: ${resText.slice(0, 150)})`); + } + + /** + * Search for filecode patterns in HTML + */ + _findFilecodeInHtml(html) { + // filecode: "xxx" or filecode = "xxx" + const m1 = html.match(/filecode['":\s]+['"]([a-zA-Z0-9]{8,})['"]/i); + if (m1) return m1[1]; + // file_code: "xxx" + const m2 = html.match(/file_code['":\s]+['"]([a-zA-Z0-9]{8,})['"]/i); + if (m2) return m2[1]; + // Download URL pattern: /d/FILECODE + const m3 = html.match(/\/d\/([a-zA-Z0-9]{8,})/); + if (m3) return m3[1]; + return null; + } + + /** + * Extract result from JSON payload + */ + _extractFromJson(payload) { + if (payload.status && Number(payload.status) !== 200 && payload.msg) { + throw new Error(`Doodstream Upload: ${payload.msg}`); + } + + let item = null; + const result = payload.result; + if (Array.isArray(result) && result.length > 0) { + item = result[0]; + } else if (typeof result === 'object' && result) { + item = result; + } + + if (!item) { + throw new Error(`Doodstream Upload fehlgeschlagen: ${payload.msg || JSON.stringify(payload).slice(0, 150)}`); + } + + const fileCode = item.filecode || item.file_code || ''; + return { + download_url: item.download_url || item.protected_dl || (fileCode ? `https://doodstream.com/d/${fileCode}` : null), + embed_url: item.protected_embed || (fileCode ? `https://doodstream.com/e/${fileCode}` : null), + file_code: fileCode + }; + } + + _buildResult(fileCode) { + return { + download_url: `https://doodstream.com/d/${fileCode}`, + embed_url: `https://doodstream.com/e/${fileCode}`, + file_code: fileCode + }; + } + + /** + * Pull candidate API-key tokens out of a logged-in settings page. We do NOT + * rely on knowing doodstream's exact (cookie-gated, unseen) settings DOM — + * instead we gather every plausible long token from form-field values and + * element contents, ranked so tokens near an "api" mention are tried first. + * The caller validates each against the official API, so a wrong guess is + * harmless (it just fails validation). Returned newest-/most-likely-first. + */ + _extractApiKeyCandidates(html) { + if (!html) return []; + const cands = new Set(); + const patterns = [ + /value=["']([A-Za-z0-9]{20,})["']/gi, // + /<(?:textarea|code|span|pre|input)[^>]*>\s*([A-Za-z0-9]{20,})\s*KEY + /\b(?:api[_-]?key|apikey)\b["':\s=>]*["']?([A-Za-z0-9]{20,})/gi // api_key: "KEY" + ]; + for (const re of patterns) { + let m; + while ((m = re.exec(html)) !== null) cands.add(m[1]); + } + // Rank tokens whose preceding context mentions "api" ahead of the rest. + return [...cands] + .map(t => { + const idx = html.indexOf(t); + const ctx = html.slice(Math.max(0, idx - 160), idx).toLowerCase(); + return { t, near: /api/.test(ctx) ? 0 : 1 }; + }) + .sort((a, b) => a.near - b.near) + .map(s => s.t); + } + + /** + * Validate a candidate key against the official API. Only the account's real + * key returns status 200, so this is what makes the brute-force extraction + * safe regardless of the settings-page markup. + */ + async _validateApiKey(key) { + try { + const res = await fetch(`https://doodapi.co/api/account/info?key=${encodeURIComponent(key)}`, { + method: 'GET', redirect: 'follow', signal: AbortSignal.timeout(15000) + }); + const json = await res.json().catch(() => null); + return !!(json && Number(json.status) === 200); + } catch { + return false; + } + } + + /** + * Derive the account's doodapi API key from the logged-in web session, so a + * login-only account can upload via the reliable JSON API (which returns the + * filecode directly) instead of the fragile web upload form. Best-effort: + * returns null if no valid key can be found, and the caller falls back to the + * web-form upload. Requires login() to have run first (needs the cookies). + */ + async deriveApiKey() { + if (this.apiKey) return this.apiKey; + let html = ''; + for (const page of ['/?op=my_account', '/settings', '/?op=profile']) { + try { + const res = await this._fetch(BASE_URL + page); + const text = await res.text(); + if (text && /api[\s_-]?key/i.test(text)) { html = text; break; } + if (text && !html) html = text; + } catch { /* try next page */ } + } + const candidates = this._extractApiKeyCandidates(html); + // Cap validation calls (rate limit 10/s; settings page yields few tokens). + for (const key of candidates.slice(0, 15)) { + if (await this._validateApiKey(key)) { + this.apiKey = key; + _debugLog(`api-key derive: validated key (len ${key.length})`); + return key; + } + } + _debugLog(`api-key derive: ${candidates.length} candidate(s), none validated. settings html(2500)=${(html || '').slice(0, 2500)}`); + return null; + } +} + +module.exports = DoodstreamUploader; +module.exports.setDebugVerbose = setDebugVerbose; diff --git a/lib/file-probe.js b/lib/file-probe.js new file mode 100644 index 0000000..b312f6a --- /dev/null +++ b/lib/file-probe.js @@ -0,0 +1,77 @@ +const fs = require('fs'); + +const SIGNATURES = [ + { kind: 'mp4-iso', test: (b) => b.length >= 12 && b.slice(4, 8).toString('ascii') === 'ftyp' }, + { kind: 'matroska', test: (b) => b.length >= 4 && b[0] === 0x1A && b[1] === 0x45 && b[2] === 0xDF && b[3] === 0xA3 }, + { kind: 'avi', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'AVI ' }, + { kind: 'wav', test: (b) => b.length >= 12 && b.slice(0, 4).toString('ascii') === 'RIFF' && b.slice(8, 12).toString('ascii') === 'WAVE' }, + { kind: 'flv', test: (b) => b.length >= 3 && b.slice(0, 3).toString('ascii') === 'FLV' }, + { kind: 'asf-wmv', test: (b) => b.length >= 4 && b[0] === 0x30 && b[1] === 0x26 && b[2] === 0xB2 && b[3] === 0x75 }, + { kind: 'mpeg-ps', test: (b) => b.length >= 4 && b[0] === 0x00 && b[1] === 0x00 && b[2] === 0x01 && (b[3] === 0xBA || b[3] === 0xB3) }, + { kind: 'gif', test: (b) => b.length >= 6 && (b.slice(0, 6).toString('ascii') === 'GIF87a' || b.slice(0, 6).toString('ascii') === 'GIF89a') }, + // TS demands the 0x47 sync byte every 188 bytes — a single leading 0x47 + // matches every GIF and every text file starting with "G", so require + // three consecutive packet boundaries before classifying as video. + { kind: 'mpeg-ts', test: (b) => b.length >= 377 && b[0] === 0x47 && b[188] === 0x47 && b[376] === 0x47 }, + { kind: 'mp3', test: (b) => b.length >= 3 && (b.slice(0, 3).toString('ascii') === 'ID3' || (b[0] === 0xFF && (b[1] & 0xE0) === 0xE0)) }, + { kind: 'ogg', test: (b) => b.length >= 4 && b.slice(0, 4).toString('ascii') === 'OggS' }, + { kind: 'jpeg', test: (b) => b.length >= 3 && b[0] === 0xFF && b[1] === 0xD8 && b[2] === 0xFF }, + { kind: 'png', test: (b) => b.length >= 8 && b[0] === 0x89 && b.slice(1, 4).toString('ascii') === 'PNG' }, + { kind: 'pdf', test: (b) => b.length >= 5 && b.slice(0, 5).toString('ascii') === '%PDF-' }, + { kind: 'zip', test: (b) => b.length >= 4 && b[0] === 0x50 && b[1] === 0x4B && (b[2] === 0x03 || b[2] === 0x05 || b[2] === 0x07) }, + { kind: 'html', test: (b) => { + const s = b.toString('ascii', 0, Math.min(b.length, 64)).trimStart().toLowerCase(); + return s.startsWith(' 0 ? bytes : 64; + return new Promise((resolve) => { + fs.open(filePath, 'r', (err, fd) => { + if (err) return resolve({ ok: false, error: err.message, kind: 'unreadable' }); + const buf = Buffer.alloc(want); + fs.read(fd, buf, 0, want, 0, (rerr, bytesRead) => { + fs.close(fd, () => {}); + if (rerr) return resolve({ ok: false, error: rerr.message, kind: 'unreadable' }); + const slice = buf.slice(0, bytesRead); + resolve({ + ok: true, + bytesRead, + kind: detectKind(slice), + isVideoLike: isVideoLikeKind(detectKind(slice)), + headHex: slice.toString('hex') + }); + }); + }); + }); +} + +function summarizeFileStat(filePath) { + try { + const st = fs.statSync(filePath); + return { + size: st.size, + mtime: st.mtime.toISOString(), + isFile: st.isFile() + }; + } catch (err) { + return { error: err.message }; + } +} + +module.exports = { detectKind, isVideoLikeKind, probeFileHead, summarizeFileStat, VIDEO_KINDS, SIGNATURES }; diff --git a/lib/folder-monitor.js b/lib/folder-monitor.js new file mode 100644 index 0000000..1c6914f --- /dev/null +++ b/lib/folder-monitor.js @@ -0,0 +1,103 @@ +const { EventEmitter } = require('events'); +const path = require('path'); +const chokidar = require('chokidar'); + +class FolderMonitor extends EventEmitter { + constructor() { + super(); + this._watcher = null; + this._settings = null; + this._seenFiles = new Set(); + this._batchBuffer = []; + this._batchTimer = null; + } + + get running() { + return !!this._watcher; + } + + start(settings) { + this.stop(); + this._settings = settings; + + const folderPath = String(settings.folderPath || '').trim(); + if (!folderPath) throw new Error('Kein Ordnerpfad angegeben'); + + const watchOptions = { + persistent: true, + ignoreInitial: true, + depth: settings.recursive ? undefined : 0, + awaitWriteFinish: { + stabilityThreshold: Math.max(1000, (settings.delaySec || 3) * 1000), + pollInterval: 500 + } + }; + + this._watcher = chokidar.watch(folderPath, watchOptions); + this._watcher.on('add', (filePath) => this._onNewFile(filePath)); + this._watcher.on('unlink', (filePath) => { + // Allow re-added files (e.g. re-encoded) to be detected again + const normalized = filePath.replace(/\\/g, '/').toLowerCase(); + this._seenFiles.delete(normalized); + }); + this._watcher.on('error', (err) => this.emit('error', err)); + } + + stop() { + if (this._watcher) { + this._watcher.close().catch(() => {}); + this._watcher = null; + } + if (this._batchTimer) { + clearTimeout(this._batchTimer); + this._batchTimer = null; + } + this._batchBuffer = []; + this._seenFiles = new Set(); + } + + status() { + return { + running: this.running, + folderPath: this._settings ? this._settings.folderPath : '', + seenCount: this._seenFiles.size + }; + } + + _onNewFile(filePath) { + const settings = this._settings; + if (!settings) return; + + // Extension filter + const ext = path.extname(filePath).replace(/^\./, '').toLowerCase(); + const rawExtensions = String(settings.extensions || '').trim(); + if (rawExtensions) { + const extList = rawExtensions.split(',').map(e => e.trim().toLowerCase().replace(/^\./, '')).filter(Boolean); + if (extList.length > 0) { + const matches = extList.includes(ext); + if (settings.filterMode === 'include' && !matches) return; + if (settings.filterMode === 'exclude' && matches) return; + } + } + + // Skip duplicates (session-based) + if (settings.skipDuplicates) { + const normalized = filePath.replace(/\\/g, '/').toLowerCase(); + if (this._seenFiles.has(normalized)) return; + this._seenFiles.add(normalized); + } + + // Batch: collect files over 200ms window then emit together + this._batchBuffer.push(filePath); + if (this._batchTimer) clearTimeout(this._batchTimer); + this._batchTimer = setTimeout(() => { + const files = this._batchBuffer.splice(0); + this._batchTimer = null; + if (files.length > 0) { + this.emit('new-files', files); + } + }, 200); + } +} + +module.exports = FolderMonitor; diff --git a/lib/hosters.js b/lib/hosters.js new file mode 100644 index 0000000..5bd7a5d --- /dev/null +++ b/lib/hosters.js @@ -0,0 +1,728 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request } = require('undici'); + +const UPLOAD_TIMEOUT = 1800000; // 30 minutes +const API_TIMEOUT = 45000; // 45 seconds +const SERVER_RETRY_ATTEMPTS = 6; +const SERVER_RETRY_DELAY_MS = 2500; +const LAST_UPLOAD_SERVERS = new Map(); + +function appendRawQuery(url, rawQuery) { + const parsed = new URL(url); + const cleanQuery = String(rawQuery || '').trim().replace(/^\?+/, ''); + if (!cleanQuery) return parsed.toString(); + + if (parsed.search && parsed.search.length > 1) { + parsed.search = `${parsed.search.slice(1)}&${cleanQuery}`; + } else { + parsed.search = cleanQuery; + } + + return parsed.toString(); +} + +function appendKeyParam(url, key) { + const parsed = new URL(url); + parsed.searchParams.set('key', key); + return parsed.toString(); +} + +// Hoster definitions - based on official API docs +const HOSTER_CONFIGS = { + 'doodstream.com': { + apiBase: 'https://doodapi.co', + serverEndpoints: ['/api/upload/server'], + // No hardcoded fallback node: that stale CDN host (tr1128ve.cloudatacdn.com) + // accepts the bytes but returns an empty result form with no filecode, so a + // failed server lookup must throw cleanly rather than upload ~1 GB into a + // dead end. (Same reasoning as the web-session path's fail-fast.) + buildUploadUrl: (url, key) => appendRawQuery(url, key), + formFields: (key) => ({ api_key: key }), + parseResult: parseDoodstreamResult + }, + 'voe.sx': { + apiBase: 'https://voe.sx', + serverEndpoints: ['/api/upload/server', '/api/v1/upload/server'], + buildUploadUrl: (url, key) => appendKeyParam(url, key), + formFields: () => ({}), + parseResult: parseVoeResult + }, + 'byse.sx': { + apiBase: 'https://api.byse.sx', + serverEndpoints: ['/upload/server'], + buildUploadUrl: (url, key) => appendKeyParam(url, key), + formFields: (key) => ({ key }), + parseResult: parseByseResult + } +}; + +function normalizeAbsoluteUrl(raw, apiBase) { + if (typeof raw !== 'string') return null; + const trimmed = raw.trim(); + if (!trimmed || /^\[object\s+Object\]$/i.test(trimmed)) return null; + + let candidate = trimmed; + if (candidate.startsWith('//')) { + candidate = `https:${candidate}`; + } else if (candidate.startsWith('/')) { + try { + candidate = new URL(candidate, apiBase).href; + } catch { + return null; + } + } else if (!/^[a-z][a-z\d+.-]*:\/\//i.test(candidate)) { + candidate = `https://${candidate.replace(/^\/+/, '')}`; + } + + try { + const parsed = new URL(candidate); + if (!['http:', 'https:'].includes(parsed.protocol)) return null; + return parsed.href; + } catch { + return null; + } +} + +function collectUploadUrlCandidates(value, out = []) { + if (typeof value === 'string') { + out.push(value); + return out; + } + + if (Array.isArray(value)) { + for (const entry of value) collectUploadUrlCandidates(entry, out); + return out; + } + + if (value && typeof value === 'object') { + const preferredKeys = ['upload_url', 'uploadUrl', 'url', 'server', 'srv', 'result']; + for (const key of preferredKeys) { + if (Object.prototype.hasOwnProperty.call(value, key)) { + collectUploadUrlCandidates(value[key], out); + } + } + + for (const nested of Object.values(value)) { + if (typeof nested === 'string') out.push(nested); + } + } + + return out; +} + +function extractUploadServerUrl(payload, apiBase) { + const source = payload && Object.prototype.hasOwnProperty.call(payload, 'result') + ? payload.result + : payload; + + const candidates = collectUploadUrlCandidates(source, []); + for (const candidate of candidates) { + const normalized = normalizeAbsoluteUrl(candidate, apiBase); + if (normalized) return normalized; + } + + return null; +} + +function shouldRetryServerLookup(message) { + const msg = String(message || '').toLowerCase(); + if (!msg) return true; + if (msg.includes('invalid') && msg.includes('key')) return false; + if (msg.includes('unauthorized') || msg.includes('forbidden')) return false; + if (msg.includes('no servers available')) return true; + if (msg.includes('temporar') || msg.includes('busy') || msg.includes('try again')) return true; + return true; +} + +function sleep(ms, signal) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (signal) signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + + function onAbort() { + clearTimeout(timer); + if (signal) signal.removeEventListener('abort', onAbort); + const err = new Error('Aborted'); + err.name = 'AbortError'; + reject(err); + } + + if (signal) { + if (signal.aborted) return onAbort(); + signal.addEventListener('abort', onAbort); + } + }); +} + +// --- Result parsers --- + +// Doodstream: { result: [{ download_url, protected_embed, filecode, protected_dl }] } +function parseDoodstreamResult(payload) { + let item = {}; + // Defensive: also handle direct callers that bypass uploadFile's payload + // normalisation (e.g. unit tests, future callers). + const result = payload && payload.result; + if (Array.isArray(result) && result.length > 0) { + item = result[0]; + } else if (result && typeof result === 'object') { + item = result; + } + + return { + download_url: item.download_url || item.protected_dl || null, + embed_url: item.protected_embed || null, + file_code: item.filecode || item.file_code || null + }; +} + +// VOE: { file: { file_code } } +function parseVoeResult(payload) { + const source = payload && typeof payload === 'object' && payload.result && typeof payload.result === 'object' + ? payload.result + : payload; + const file = source && typeof source.file === 'object' ? source.file : null; + const file_code = file?.file_code + || file?.filecode + || source?.file_code + || source?.filecode + || null; + + return { + download_url: file_code ? `https://voe.sx/${file_code}` : null, + embed_url: file_code ? `https://voe.sx/e/${file_code}` : null, + file_code + }; +} + +// Byse: { files: [{ filecode, filename, status }] } +function parseByseResult(payload) { + // Defensive: bypass-callers may pass null/non-object directly. + if (!payload || typeof payload !== 'object') payload = {}; + let file_code = null; + let perFileError = null; + + // Primary: files array (per official Byse API docs) + if (Array.isArray(payload.files) && payload.files.length > 0) { + const f = payload.files[0]; + file_code = f && (f.filecode || f.file_code) || null; + // Byse returns HTTP 200 + msg=OK even when a specific file was rejected + // ("Not video file format", "Duplicate", "File too small", ...). When + // filecode is empty and status carries a non-OK message, that IS the + // actual per-file error, not a server problem. + if (!file_code && f && f.status && !/^(ok|success|done)$/i.test(String(f.status))) { + perFileError = String(f.status).trim(); + } + } + // Fallback: result object + if (!file_code && payload.result) { + const result = payload.result; + if (Array.isArray(result) && result.length > 0) { + file_code = result[0].filecode || result[0].file_code; + } else if (typeof result === 'object') { + file_code = result.filecode || result.file_code; + } + } + + if (!file_code && perFileError) { + // Distinguish account-level from file-level failure. "not enough disk + // space", "quota exceeded", "storage full" etc. mean the ACCOUNT is + // exhausted — every further file on the same account will hit the same + // wall, so we must rotate. File-specific rejections (Duplicate, wrong + // format, too small/large) ARE per-file and rotation is pointless. + const accountLevel = /(not enough (disk )?(space|storage)|insufficient (disk )?space|disk (space )?full|storage (exhausted|full|voll|limit)|quota (exceeded|voll|überschritten)|account (full|voll|suspended|banned))/i.test(perFileError); + const err = new Error(`Byse lehnte Datei ab: ${perFileError}`); + if (accountLevel) { + err.accountError = true; + } else { + err.fileRejected = true; + // "Not video file format" is byse's known-misleading status: observed + // live (2026-06-09) ONLY on valid MKVs >2.7 GB while the same account + // accepted 1100+ smaller MKVs. Per-account size tiers produce it, and + // async registration can land the file anyway. Flag it suspect so the + // recovery poll still runs and the upload manager may try the file on + // the remaining accounts instead of failing it everywhere. + if (/not video file format/i.test(perFileError)) err.suspectReject = true; + } + throw err; + } + + return { + download_url: file_code ? `https://byse.sx/d/${file_code}` : null, + embed_url: file_code ? `https://byse.sx/e/${file_code}` : null, + file_code + }; +} + +// --- Multipart upload with progress --- + +function buildMultipart(filePath, formFields) { + const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex'); + const fileName = path.basename(filePath); + const fileSize = fs.statSync(filePath).size; + + let preamble = ''; + for (const [key, value] of Object.entries(formFields)) { + preamble += `--${boundary}\r\n`; + preamble += `Content-Disposition: form-data; name="${key}"\r\n\r\n`; + preamble += `${value}\r\n`; + } + preamble += `--${boundary}\r\n`; + const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + preamble += `Content-Disposition: form-data; name="file"; filename="${safeFileName}"\r\n`; + preamble += `Content-Type: application/octet-stream\r\n\r\n`; + + const epilogue = `\r\n--${boundary}--\r\n`; + + const preambleBuf = Buffer.from(preamble, 'utf-8'); + const epilogueBuf = Buffer.from(epilogue, 'utf-8'); + const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; + + return { boundary, preambleBuf, epilogueBuf, totalSize, fileSize }; +} + +function createUploadBody(filePath, formFields, onProgress, throttle, signal) { + const { boundary, preambleBuf, epilogueBuf, totalSize, fileSize } = buildMultipart(filePath, formFields); + + let bytesRead = 0; + const CHUNK_SIZE = 1024 * 1024; + + async function* generate() { + yield preambleBuf; + const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE }); + for await (const chunk of fileStream) { + if (signal && signal.aborted) throw new Error('Aborted'); + if (throttle) await throttle.consume(chunk.length, signal); + bytesRead += chunk.length; + yield chunk; + if (onProgress) onProgress(bytesRead, fileSize); + } + yield epilogueBuf; + } + + return { iterable: generate(), boundary, totalSize }; +} + +// --- API helper using built-in fetch (follows redirects automatically) --- + +async function apiGet(url, signal) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), API_TIMEOUT); + const onAbort = () => controller.abort(); + if (signal) signal.addEventListener('abort', onAbort); + + try { + const res = await fetch(url, { + method: 'GET', + signal: controller.signal, + redirect: 'follow' + }); + const text = await res.text(); + let data; + try { + data = JSON.parse(text); + } catch { + const err = new Error(`API-Antwort war kein JSON (HTTP ${res.status}): ${(text || '').slice(0, 200)}`); + if (res.status >= 500) err.transientNetwork = true; + throw err; + } + + if (data.status && [401, 403, 429, 500].includes(data.status)) { + const err = new Error(data.msg || data.message || JSON.stringify(data)); + if (data.status === 500) err.transientNetwork = true; + throw err; + } + return data; + } finally { + clearTimeout(timeout); + if (signal) signal.removeEventListener('abort', onAbort); + } +} + +// --- Main upload function --- + +async function getUploadServer(hosterName, hosterConfig, apiKey, signal) { + let lastMessage = ''; + let lastTransient = false; + + for (let attempt = 1; attempt <= SERVER_RETRY_ATTEMPTS; attempt++) { + for (const endpoint of hosterConfig.serverEndpoints) { + const url = `${hosterConfig.apiBase}${endpoint}?key=${encodeURIComponent(apiKey)}`; + try { + const data = await apiGet(url, signal); + const uploadUrl = extractUploadServerUrl(data, hosterConfig.apiBase); + if (uploadUrl) { + LAST_UPLOAD_SERVERS.set(hosterName, uploadUrl); + return uploadUrl; + } + + const apiMessage = data && (data.msg || data.message) + ? String(data.msg || data.message).trim() + : ''; + if (apiMessage) lastMessage = apiMessage; + } catch (err) { + if (err.name === 'AbortError') throw err; + if (err.message) lastMessage = err.message; + if (err.transientNetwork === true) lastTransient = true; + } + } + + if (attempt < SERVER_RETRY_ATTEMPTS && shouldRetryServerLookup(lastMessage)) { + await sleep(SERVER_RETRY_DELAY_MS, signal); + continue; + } + + break; + } + + const cachedServer = LAST_UPLOAD_SERVERS.get(hosterName); + if (cachedServer && shouldRetryServerLookup(lastMessage)) { + return cachedServer; + } + + if (shouldRetryServerLookup(lastMessage) && Array.isArray(hosterConfig.fallbackUploadServers)) { + for (const fallback of hosterConfig.fallbackUploadServers) { + const normalized = normalizeAbsoluteUrl(fallback, hosterConfig.apiBase); + if (normalized) { + LAST_UPLOAD_SERVERS.set(hosterName, normalized); + return normalized; + } + } + } + + if (lastMessage) { + const e = new Error(`Kein Upload-Server erhalten: ${lastMessage}`); + // "no servers available" / busy / try-again is a transient hoster-side + // condition, not an account fault — tag it so the account isn't blacklisted. + // Genuine auth failures (invalid key / unauthorized / forbidden) make + // shouldRetryServerLookup return false and stay classified as account errors. + if (shouldRetryServerLookup(lastMessage)) e.hosterTransient = true; + if (lastTransient) e.transientNetwork = true; + throw e; + } + throw new Error('Kein Upload-Server erhalten. API-Key pruefen.'); +} + +async function _fetchByseFileList(apiKey, signal) { + // Byse's file-list endpoint. Returns up to 100 most-recent files — enough + // to match the upload we just did against what the server has. The API + // shape is typical XFS: { status, msg, result: { files: [...] } } or + // { status, msg, files: [...] }. + const url = `https://api.byse.sx/file/list?key=${encodeURIComponent(apiKey)}&per_page=100&sort=date&order=desc`; + try { + const { body, statusCode } = await request(url, { + method: 'GET', signal, + headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' }, + headersTimeout: 30_000, bodyTimeout: 30_000 + }); + const text = await body.text(); + if (statusCode < 200 || statusCode >= 300) return []; + const data = JSON.parse(text); + const src = Array.isArray(data.files) ? data.files + : (data.result && Array.isArray(data.result.files) ? data.result.files + : (Array.isArray(data.result) ? data.result : [])); + return src.map(f => ({ + file_code: String(f.file_code || f.filecode || '').trim(), + file_name: String(f.title || f.name || f.file_name || '').trim() + })).filter(f => f.file_code); + } catch { + return []; + } +} + +function _normalizeFileTitle(s) { + return String(s || '').toLowerCase().replace(/\.[a-z0-9]+$/i, '').replace(/[^a-z0-9]+/g, ''); +} + +async function _resolveByseUploadByName(apiKey, fileName, baselineCodes, signal) { + const expected = _normalizeFileTitle(fileName); + const POLL_ATTEMPTS = 15; + const POLL_DELAY_MS = 2000; + for (let i = 0; i < POLL_ATTEMPTS; i++) { + if (signal && signal.aborted) return null; + const list = await _fetchByseFileList(apiKey, signal); + const newFiles = list.filter(f => !baselineCodes.has(f.file_code)); + // Exact-normalized filename match ONLY. The old fallback ("only one new + // file → take it") was unsafe during parallel byse uploads: job A's + // poller could claim job B's newly appeared file and return the wrong + // URL. At the cost of a few false-negatives when byse mangles the + // filename beyond our normalizer, correctness for parallel uploads wins. + const match = newFiles.find(f => _normalizeFileTitle(f.file_name) === expected); + if (match) { + return { + download_url: `https://byse.sx/d/${match.file_code}`, + embed_url: `https://byse.sx/e/${match.file_code}`, + file_code: match.file_code + }; + } + if (i < POLL_ATTEMPTS - 1) { + try { + await sleep(POLL_DELAY_MS, signal); + } catch { + return null; + } + } + } + return null; +} + +async function _fetchDoodstreamFileList(apiKey, signal) { + // doodapi.co file list: { msg, status:200, result: { files: [{ file_code, title, uploaded, ... }] } } + // sort=created&order=desc forces newest-first — VERIFIED against a real 90k-file + // account, where a single page without it could miss a just-uploaded file. The + // recovery only needs the most recent uploads, so page 1 newest-first suffices. + const url = `https://doodapi.co/api/file/list?key=${encodeURIComponent(apiKey)}&per_page=200&sort=created&order=desc`; + try { + const { body, statusCode } = await request(url, { + method: 'GET', signal, + headers: { 'Accept': 'application/json', 'User-Agent': 'multi-hoster-uploader/1.1' }, + headersTimeout: 30_000, bodyTimeout: 30_000 + }); + const text = await body.text(); + if (statusCode < 200 || statusCode >= 300) return []; + const data = JSON.parse(text); + const files = data && data.result && Array.isArray(data.result.files) ? data.result.files : []; + return files.map(f => ({ + file_code: String(f.file_code || f.filecode || '').trim(), + file_name: String(f.title || f.file_name || f.name || '').trim() + })).filter(f => f.file_code); + } catch { + return []; + } +} + +const DOODSTREAM_POLL = { attempts: 12, delayMs: 2500 }; // test-tunable via __test + +async function _resolveDoodstreamUploadByName(apiKey, fileName, baselineCodes, signal) { + // Same recovery byse uses: the upload POST returned no filecode, but the file + // may register in the account a little later. Poll the list for a NEW file + // whose normalized title matches what we uploaded. Exact-name match only + // (never "take the only new one") so parallel doodstream uploads can't claim + // each other's files. + const expected = _normalizeFileTitle(fileName); + const POLL_ATTEMPTS = DOODSTREAM_POLL.attempts; + const POLL_DELAY_MS = DOODSTREAM_POLL.delayMs; + for (let i = 0; i < POLL_ATTEMPTS; i++) { + if (signal && signal.aborted) return null; + const list = await _fetchDoodstreamFileList(apiKey, signal); + const fresh = list.filter(f => !baselineCodes.has(f.file_code)); + const match = fresh.find(f => _normalizeFileTitle(f.file_name) === expected); + if (match) { + return { + download_url: `https://doodstream.com/d/${match.file_code}`, + embed_url: `https://doodstream.com/e/${match.file_code}`, + file_code: match.file_code + }; + } + if (i < POLL_ATTEMPTS - 1) { + try { + await sleep(POLL_DELAY_MS, signal); + } catch { + return null; + } + } + } + return null; +} + +async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, throttle, opts) { + const config = HOSTER_CONFIGS[hosterName]; + if (!config) throw new Error(`Unbekannter Hoster: ${hosterName}`); + + let byseBaseline = null; + if (hosterName === 'byse.sx') { + if (opts && opts.byseBaseline instanceof Set) { + byseBaseline = opts.byseBaseline; + } else { + const baseline = await _fetchByseFileList(apiKey, signal); + byseBaseline = new Set(baseline.map(f => f.file_code)); + } + } + let doodBaseline = null; + if (hosterName === 'doodstream.com') { + if (opts && opts.doodBaseline instanceof Set) { + doodBaseline = opts.doodBaseline; + } else { + const baseline = await _fetchDoodstreamFileList(apiKey, signal); + doodBaseline = new Set(baseline.map(f => f.file_code)); + } + } + + // Step 1: Get upload server + const uploadUrl = await getUploadServer(hosterName, config, apiKey, signal); + + // Step 2: Upload file with progress + const targetUrl = config.buildUploadUrl(uploadUrl, apiKey); + const formFields = config.formFields(apiKey); + + const { iterable, boundary, totalSize } = createUploadBody(filePath, formFields, onProgress, throttle, signal); + + const { body, statusCode, headers } = await request(targetUrl, { + method: 'POST', + body: iterable, + signal, + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize), + 'Accept': 'application/json, text/plain;q=0.9, */*;q=0.8', + 'User-Agent': 'multi-hoster-uploader/1.1' + }, + headersTimeout: UPLOAD_TIMEOUT, + bodyTimeout: UPLOAD_TIMEOUT + }); + + const rawBody = await body.text(); + let payload = null; + try { + payload = rawBody ? JSON.parse(rawBody) : {}; + } catch { + const snippet = rawBody ? rawBody.slice(0, 240).replace(/\s+/g, ' ').trim() : ''; + const err = new Error( + `Upload-Antwort von ${hosterName} war kein JSON (HTTP ${statusCode}${snippet ? `): ${snippet}` : ')'}` + ); + if (statusCode >= 500) err.transientNetwork = true; + throw err; + } + // Normalize valid-but-not-object JSON (JSON.parse('null') → null; + // JSON.parse('"foo"') → string; JSON.parse('[1]') → array). Without this + // the downstream `payload.msg` / `payload.status` / parseResult(payload) + // calls crash with a confusing TypeError instead of letting the existing + // fallback defaults kick in. Arrays from servers that return a top-level + // list (rare but seen in the wild) are kept addressable as `payload.X` + // → undefined, which the parsers already handle. + if (payload === null || typeof payload !== 'object') { + payload = {}; + } + + if (statusCode < 200 || statusCode >= 300) { + const err = new Error( + payload.msg + || payload.message + || `Upload fehlgeschlagen (HTTP ${statusCode}${headers?.['content-type'] ? `, ${headers['content-type']}` : ''})` + ); + if (statusCode >= 500) err.transientNetwork = true; + throw err; + } + + if (payload.status && [401, 403, 429, 500].includes(payload.status)) { + const err = new Error(payload.msg || payload.message || JSON.stringify(payload)); + if (payload.status === 500) err.transientNetwork = true; + throw err; + } + + let result = null; + let parseErr = null; + try { + result = config.parseResult(payload); + } catch (err) { + if (err && typeof err === 'object' && !err.diagnostic) { + try { + err.diagnostic = { + hoster: hosterName, + http: statusCode, + contentType: (headers && headers['content-type']) || null, + payloadSnippet: JSON.stringify(payload).slice(0, 1000), + uploadUrl: targetUrl + }; + } catch { /* JSON cycle — skip diagnostic */ } + } + parseErr = err; + } + if (result && (result.file_code || result.download_url || result.embed_url)) { + return result; + } + + // Explicit rejections skip the recovery poll — EXCEPT suspect ones + // (byse "Not video file format", see parseByseResult): for those the file + // may have registered asynchronously despite the rejection-looking status, + // so the poll must still run. Without this exception the rescue below is + // dead for the very case it documents (regression shipped in 3.3.5x). + // When the caller's file probe positively says the upload is NOT a video + // (opts.probeIsVideoLike === false), the rejection is genuine — skip the + // 30s poll for it like any other explicit rejection. + const suspectBypass = parseErr + && parseErr.suspectReject === true + && !(opts && opts.probeIsVideoLike === false); + const explicitlyRejected = parseErr + && (parseErr.fileRejected === true || parseErr.accountError === true) + && !suspectBypass; + + // Byse-specific async handling: server accepts the file but responds with + // filecode="" + misleading status ("Not video file format"). The file shows + // up in the account shortly after — poll the list to claim it. User observed + // this with 2+ GB MKV uploads that appeared as "OK" on the byse dashboard + // even after our uploader gave up. + if (hosterName === 'byse.sx' && byseBaseline && !explicitlyRejected) { + const fileName = path.basename(filePath); + const polled = await _resolveByseUploadByName(apiKey, fileName, byseBaseline, signal); + if (polled) return polled; + } + + // Doodstream: the doodapi upload POST returned no filecode (the same backend + // hiccup that empties the web form). Poll the account file list by name — if + // the file did register, claim its code instead of failing the upload. + if (hosterName === 'doodstream.com' && doodBaseline && !explicitlyRejected) { + const fileName = path.basename(filePath); + const polled = await _resolveDoodstreamUploadByName(apiKey, fileName, doodBaseline, signal); + if (polled) return polled; + } + + if (parseErr) throw parseErr; + + if (payload.success === false) { + throw new Error(payload.msg || payload.message || `Upload zu ${hosterName} wurde vom Server abgelehnt.`); + } + + // Avoid throwing a bare "OK" / "SUCCESS" as the error message — that happens + // when the server says "msg: OK" but ships no file_code anywhere we know + // about, typically an API change. Surface the full (trimmed) payload so + // future logs actually show what the server returned. + const msg = String(payload.msg || payload.message || '').trim(); + const isOkishNoPayload = /^(ok|success|done|accepted)$/i.test(msg); + if (isOkishNoPayload || !msg) { + const snippet = JSON.stringify(payload).slice(0, 400); + // 2xx with no filecode: the hoster accepted the upload (bytes sent, status + // OK) but returned no usable link. For doodstream this is the API-path + // analog of the web empty-form — the backend file-registration timing out + // under large-file load. It's a hoster-side flake, NOT an account problem, + // so tag it hosterTransient: the upload-manager then fails this file WITHOUT + // blacklisting the account (same protection the web path got in 3.3.29) and + // the account stays usable for the next retry/batch. + const err = new Error( + `Upload zu ${hosterName} lieferte keine file_code-Antwort (Payload: ${snippet})` + ); + err.hosterTransient = true; + throw err; + } + throw new Error(msg); +} + +async function prefetchBaseline(hosterName, apiKey, signal) { + try { + if (hosterName === 'byse.sx') { + const baseline = await _fetchByseFileList(apiKey, signal); + return new Set(baseline.map(f => f.file_code)); + } + if (hosterName === 'doodstream.com') { + const baseline = await _fetchDoodstreamFileList(apiKey, signal); + return new Set(baseline.map(f => f.file_code)); + } + } catch { /* leave caller to fall back to per-job fetch */ } + return null; +} + +module.exports = { + uploadFile, + prefetchBaseline, + HOSTER_CONFIGS, + __test: { + extractUploadServerUrl, + parseVoeResult, + parseDoodstreamResult, + parseByseResult, + DOODSTREAM_POLL + } +}; diff --git a/lib/ip-allowlist.js b/lib/ip-allowlist.js new file mode 100644 index 0000000..50ec344 --- /dev/null +++ b/lib/ip-allowlist.js @@ -0,0 +1,50 @@ +function normalizeIp(ip) { + return String(ip || '').trim().replace(/^::ffff:/i, '').toLowerCase(); +} + +function isLoopbackIp(ip) { + const c = normalizeIp(ip); + return c === '' || c === '::1' || c === 'localhost' || /^127\./.test(c); +} + +function ipv4ToInt(ip) { + const parts = String(ip).split('.'); + if (parts.length !== 4) return null; + let n = 0; + for (const p of parts) { + if (!/^\d{1,3}$/.test(p)) return null; + const v = Number(p); + if (v < 0 || v > 255) return null; + n = (n << 8) + v; + } + return n >>> 0; +} + +function matchIpRule(clientIp, rule) { + const client = normalizeIp(clientIp); + const r = String(rule || '').trim().toLowerCase(); + if (!r) return false; + if (r === '*' || r === '0.0.0.0/0') return true; + if (r === client) return true; + const slash = r.indexOf('/'); + if (slash > 0) { + const baseInt = ipv4ToInt(r.slice(0, slash)); + const clientInt = ipv4ToInt(client); + const bits = Number(r.slice(slash + 1)); + if (baseInt === null || clientInt === null || !Number.isInteger(bits) || bits < 0 || bits > 32) return false; + if (bits === 0) return true; + const mask = bits === 32 ? 0xffffffff : (~((1 << (32 - bits)) - 1)) >>> 0; + return (clientInt & mask) === (baseInt & mask); + } + return false; +} + +function evaluateClientAllowed(clientIp, rules) { + const client = normalizeIp(clientIp); + if (isLoopbackIp(client)) return true; + const list = Array.isArray(rules) ? rules : []; + if (list.length === 0) return false; + return list.some((rule) => matchIpRule(client, rule)); +} + +module.exports = { normalizeIp, isLoopbackIp, ipv4ToInt, matchIpRule, evaluateClientAllowed }; diff --git a/lib/log-mode.js b/lib/log-mode.js new file mode 100644 index 0000000..2a945ce --- /dev/null +++ b/lib/log-mode.js @@ -0,0 +1,109 @@ +// Log-file mode resolution for fileuploader.log: +// - "single" → one file: fileuploader.log +// - "daily" → per-day: fileuploader-YYYY-MM-DD.log +// - "session" → per-launch: DD-MM-YYYY-mdu-session-HH-MM-NNNNNN.log +// +// Pure functions only — no fs, no Date.now() at call time — so they unit-test +// cleanly and the main.js call sites pass in `new Date()` + the session stamp. +// +// MIGRATION TRAP this lib protects against: the legacy boolean was named +// `sessionLog` but actually toggled *daily* mode. A naive rename would silently +// flip every per-day user onto per-session. normalizeLogMode below maps the +// legacy `sessionLog: true` to "daily", NOT "session". Read logMode everywhere +// downstream; do not derive from sessionLog at call sites. +// +// Loaded both as CommonJS (main.js, tests) and as a browser global +// (renderer/app.js via index.html script tag) so a single implementation backs +// runtime and tests — same pattern as queue-prune.js / queue-dedup.js. + +(function (root) { + 'use strict'; + + const VALID_MODES = new Set(['single', 'daily', 'session']); + + function normalizeLogMode(globalSettings) { + const gs = globalSettings && typeof globalSettings === 'object' ? globalSettings : {}; + if (typeof gs.logMode === 'string' && VALID_MODES.has(gs.logMode)) { + return gs.logMode; + } + // Legacy boolean migration: sessionLog *named* like "session" but actually + // implemented "daily" — preserve daily users on the migration path. + if (gs.sessionLog === true) return 'daily'; + return 'single'; + } + + function _two(n) { return String(n).padStart(2, '0'); } + + function formatDateStamp(date) { + return `${date.getFullYear()}-${_two(date.getMonth() + 1)}-${_two(date.getDate())}`; + } + + function formatSessionStamp(date, rand) { + const d = `${_two(date.getDate())}-${_two(date.getMonth() + 1)}-${date.getFullYear()}`; + const t = `${_two(date.getHours())}-${_two(date.getMinutes())}`; + const r = (rand !== undefined && rand !== null && String(rand).trim()) ? `-${String(rand).trim()}` : ''; + return `${d}-mdu-session-${t}${r}`; + } + + /** + * Compute the log filename for the given mode + clock. + * @param {Object} args + * @param {string} args.baseName e.g. "fileuploader" + * @param {string} args.ext e.g. ".log" + * @param {string} args.mode "single" | "daily" | "session" + * @param {Date} args.date current timestamp + * @param {string} [args.sessionId] required when mode === "session" + * @returns {string} the bare filename (no directory) + */ + function resolveLogFileName(args) { + const a = args || {}; + const base = String(a.baseName || 'fileuploader'); + const ext = String(a.ext || '.log'); + const mode = VALID_MODES.has(a.mode) ? a.mode : 'single'; + if (mode === 'single') return `${base}${ext}`; + if (mode === 'daily') { + const date = a.date instanceof Date ? a.date : new Date(); + return `${base}-${formatDateStamp(date)}${ext}`; + } + // session — the stamp is the full app-defined stem (DD-MM-YYYY-mdu-session-HH-MM), + // independent of baseName. + const sid = a.sessionId && String(a.sessionId).trim(); + if (sid) return `${sid}${ext}`; + // Defensive: if a session-id wasn't passed, fall back to single rather + // than emit a malformed name. main.js always supplies one. + return `${base}${ext}`; + } + + /** + * Reverse of resolveLogFileName: given a full filename like + * "fileuploader-2026-06-03.log" or + * "fileuploader-session-2026-06-03_18-16-20-8132.log", strip the mode-stamp + * so the bare base ("fileuploader.log") remains. Used when persisting an + * auto-resolved fallback path back into config — otherwise the saved path + * would keep growing a new stamp on every reload. + */ + function stripModeStampFromFileName(fileName) { + if (!fileName || typeof fileName !== 'string') return fileName; + const newSessionRe = /^\d{2}-\d{2}-\d{4}-mdu-session-\d{2}-\d{2}(?:-\d+)?(\.[^.]+)?$/; + const mNew = fileName.match(newSessionRe); + if (mNew) return `fileuploader${mNew[1] || ''}`; + // Order matters: session first (longer, more specific) before daily. + // Both regexes are anchored to $ with no nested/ambiguous quantifiers, so + // matching is linear — the eslint security warning is precautionary. + // eslint-disable-next-line security/detect-unsafe-regex + const sessionRe = /-session-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?:-\d+)?(\.[^.]+)?$/; + // eslint-disable-next-line security/detect-unsafe-regex + const dailyRe = /-\d{4}-\d{2}-\d{2}(\.[^.]+)?$/; + let out = fileName.replace(sessionRe, (m, ext) => ext || ''); + out = out.replace(dailyRe, (m, ext) => ext || ''); + return out; + } + + const api = { normalizeLogMode, resolveLogFileName, formatDateStamp, formatSessionStamp, stripModeStampFromFileName, VALID_MODES }; + + if (typeof module !== 'undefined' && module.exports) { + module.exports = api; + } else if (root) { + root.LogMode = api; + } +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/log-policy.js b/lib/log-policy.js new file mode 100644 index 0000000..3bef9f8 --- /dev/null +++ b/lib/log-policy.js @@ -0,0 +1,17 @@ +// Per-hoster upload-log policy. Decides whether a hoster's successful upload +// links get written to fileuploader.log. Pure + dependency-free so it's +// trivially unit-testable and shared between the runtime decision and tests. +// +// Contract: logging is ON unless the hoster's settings explicitly set +// logToFile === false. Missing settings / missing hoster / malformed input +// all default to ON, so the feature is strictly opt-out and never silently +// drops links because a config key wasn't present. + +function hosterLogToFileEnabled(hosterSettings, hoster) { + if (!hosterSettings || typeof hosterSettings !== 'object') return true; + const hs = hosterSettings[hoster]; + if (!hs || typeof hs !== 'object') return true; + return hs.logToFile !== false; +} + +module.exports = { hosterLogToFileEnabled }; diff --git a/lib/log-rotation.js b/lib/log-rotation.js new file mode 100644 index 0000000..0242848 --- /dev/null +++ b/lib/log-rotation.js @@ -0,0 +1,52 @@ +// Generic numbered-backup log rotation. Used by the upload log + can be +// reused by other long-lived log files (debug log, account-rotation log). +// +// Behaviour: +// - File missing → no-op, returns false. +// - File ≤ maxBytes → no-op, returns false. +// - File > maxBytes → drop oldest .N backup, shift .K → .K+1, rename live +// file to .1, return true. Caller (or the next append) creates a fresh +// primary on demand. +// +// Errors are reported via `log` (e.g. debugLog) but never thrown — rotation +// is best-effort; the caller's append happens anyway. + +const fs = require('fs'); +const path = require('path'); + +function maybeRotateLogFile(filePath, maxBytes, maxBackups = 3, log = () => {}) { + if (!filePath || !Number.isFinite(maxBytes) || maxBytes <= 0) return false; + let size = 0; + try { + const st = fs.statSync(filePath); + size = st.size; + } catch (err) { + // ENOENT is normal — nothing to rotate yet. + if (err && err.code !== 'ENOENT') { + log(`logRotation: stat ${filePath} failed: ${err.message}`); + } + return false; + } + if (size <= maxBytes) return false; + + const ext = path.extname(filePath); + const base = filePath.slice(0, filePath.length - ext.length); + + // Drop the oldest backup if it exists, then shift each numbered backup up + // one slot. Errors are ignored: missing intermediate backups are normal, + // failed renames just mean we'll rotate again next time. + try { fs.unlinkSync(`${base}.${maxBackups}${ext}`); } catch {} + for (let i = maxBackups - 1; i >= 1; i--) { + try { fs.renameSync(`${base}.${i}${ext}`, `${base}.${i + 1}${ext}`); } catch {} + } + try { + fs.renameSync(filePath, `${base}.1${ext}`); + log(`logRotation: rotated ${filePath} (${(size / 1024 / 1024).toFixed(1)} MB) → ${base}.1${ext}`); + return true; + } catch (err) { + log(`logRotation: rename ${filePath} → ${base}.1${ext} failed: ${err.message}`); + return false; + } +} + +module.exports = { maybeRotateLogFile }; diff --git a/lib/orphan-tmp.js b/lib/orphan-tmp.js new file mode 100644 index 0000000..eb66d51 --- /dev/null +++ b/lib/orphan-tmp.js @@ -0,0 +1,29 @@ +(function (root) { + 'use strict'; + + function selectOrphanTmps(fileNames, opts) { + const o = opts || {}; + const baseName = String(o.baseName || ''); + const currentPid = o.currentPid; + const isAlive = typeof o.isAlive === 'function' ? o.isAlive : () => false; + const out = []; + if (!baseName || !Array.isArray(fileNames)) return out; + const prefix = baseName + '.'; + const suffix = '.tmp'; + for (const file of fileNames) { + if (typeof file !== 'string') continue; + if (!file.startsWith(prefix) || !file.endsWith(suffix)) continue; + const mid = file.slice(prefix.length, file.length - suffix.length); + if (!/^\d+$/.test(mid)) continue; + const pid = Number(mid); + if (pid === currentPid) continue; + if (isAlive(pid)) continue; + out.push(file); + } + return out; + } + + const api = { selectOrphanTmps }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.OrphanTmp = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/queue-dedup.js b/lib/queue-dedup.js new file mode 100644 index 0000000..4b97799 --- /dev/null +++ b/lib/queue-dedup.js @@ -0,0 +1,113 @@ +// Startup queue auto-dedup logic. Extracted from renderer/app.js +// _autoDeduplicateFromLog so the decision can be unit-tested without a DOM or +// the renderer's module-level state. +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag) so a single implementation backs +// runtime and tests — no drift. +// +// Behaviour: on launch the restored queue is compared against the lifetime +// upload log. Two rules drop a job: +// 1) a 'done' job whose fileName|hoster appears in the log (declutter of +// already-finished work), and +// 2) ANY job (incl. preview) whose newest matching log entry is timestamped +// at/after the snapshot's savedAt — it provably completed AFTER the queue +// was last persisted, so a restored 'preview' row for it is a stale ghost. +// +// Rule 2 only fires when a savedAt is passed AND the log carries timestamps; +// without them this falls back to rule 1 alone. That fallback is the invariant +// the canary tests pin: a pending job matching an OLDER log line (ts < savedAt, +// or no ts at all) is KEPT — it's an intentional re-upload of a file uploaded +// before, not a ghost. The old code filtered on log-presence alone, regardless +// of status, so the ENTIRE restored queue vanished on the next restart/update +// whenever the files had been uploaded previously. Manual log import +// (importUploadLog) stays separate and explicit for bulk dedup. + +(function (root) { + 'use strict'; + + function _key(fileName, hoster) { + return `${String(fileName).toLowerCase()}|${String(hoster).toLowerCase()}`; + } + + /** + * Partition restored queue jobs into kept vs removed, given lifetime log + * entries. Removes only 'done' jobs whose fileName|hoster is in the log. + * @param {Array<{status:string,fileName:string,hoster:string}>} jobs + * @param {Array<{fileName:string,hoster:string}>} logEntries + * @returns {{ kept: Array, removed: Array }} + */ + function partitionRestoredJobsByLog(jobs, logEntries, savedAt) { + const kept = []; + const removed = []; + if (!Array.isArray(jobs) || jobs.length === 0) return { kept, removed }; + + const logKeys = new Set(); + const logMaxTs = new Map(); + for (const e of (Array.isArray(logEntries) ? logEntries : [])) { + if (e && e.fileName && e.hoster) { + const k = _key(e.fileName, e.hoster); + logKeys.add(k); + if (typeof e.ts === 'number' && isFinite(e.ts)) { + const prev = logMaxTs.get(k); + if (prev === undefined || e.ts > prev) logMaxTs.set(k, e.ts); + } + } + } + + const savedAtFloor = (typeof savedAt === 'number' && isFinite(savedAt)) + ? Math.floor(savedAt / 1000) * 1000 + : null; + + const filesPerKey = new Map(); + for (const job of jobs) { + if (job && job.fileName && job.hoster) { + const jk = _key(job.fileName, job.hoster); + let set = filesPerKey.get(jk); + if (!set) { set = new Set(); filesPerKey.set(jk, set); } + set.add(job.file || ''); + } + } + + for (const job of jobs) { + const hasIds = job && job.fileName && job.hoster; + const k = hasIds ? _key(job.fileName, job.hoster) : null; + const doneInLog = job && job.status === 'done' && hasIds && logKeys.has(k); + const keyUnambiguous = k !== null && filesPerKey.get(k).size <= 1; + const uploadedAfterSnapshot = savedAtFloor !== null && k !== null && keyUnambiguous + && logMaxTs.has(k) && logMaxTs.get(k) >= savedAtFloor; + if (doneInLog || uploadedAfterSnapshot) { + removed.push(job); + } else { + kept.push(job); + } + } + return { kept, removed }; + } + + function completedSelectionKeys(selectedFiles, hosters, logEntries, savedAt) { + const out = []; + if (!Array.isArray(selectedFiles) || !Array.isArray(hosters)) return out; + if (!(typeof savedAt === 'number' && isFinite(savedAt))) return out; + const synthetic = []; + for (const f of selectedFiles) { + if (!f || !f.path) continue; + const name = f.name || String(f.path).split(/[\\/]/).pop(); + for (const h of hosters) { + if (h) synthetic.push({ fileName: name, hoster: h, file: f.path, status: 'preview' }); + } + } + if (synthetic.length === 0) return out; + const { removed } = partitionRestoredJobsByLog(synthetic, logEntries, savedAt); + for (const job of removed) out.push(`${job.file}|${job.hoster}`); + return out; + } + + const api = { partitionRestoredJobsByLog, completedSelectionKeys }; + + if (typeof module !== 'undefined' && module.exports) { + module.exports = api; + } else if (root) { + root.QueueDedup = api; + } +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/queue-prune.js b/lib/queue-prune.js new file mode 100644 index 0000000..81352f4 --- /dev/null +++ b/lib/queue-prune.js @@ -0,0 +1,59 @@ +// Queue auto-prune logic. Extracted from renderer/app.js handleBatchDone so +// the algorithm can be unit-tested without needing a DOM or the renderer's +// module-level state (queueJobs, _jobIndexById). +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag) so the same single +// implementation backs both runtime and tests — no drift between them. +// +// Behaviour: when the number of terminal-status jobs (done / skipped / +// error / aborted) in the queue exceeds `limit`, drop the oldest terminal +// jobs (insertion order) until we're back at the limit. Non-terminal jobs +// (queued / preview / uploading / retrying / getting-server) are always +// kept — those are work the user can still act on. Without this cap a +// long session accumulates thousands of done rows and every render becomes +// O(N) on a perpetually-growing N. + +(function (root) { + 'use strict'; + + const TERMINAL_STATUSES = new Set(['done', 'skipped', 'error', 'aborted']); + + /** + * Compute which jobs to keep vs drop, given a queue and a terminal-jobs cap. + * @param {Array<{id: string, status: string}>} jobs the current queue + * @param {number} limit max terminal jobs to keep + * @returns {null | { kept: Array, dropped: Array }} null when nothing changed + */ + function pruneOldestTerminalJobs(jobs, limit) { + if (!Array.isArray(jobs) || jobs.length === 0) return null; + if (!Number.isFinite(limit) || limit < 0) return null; + + // Walk once, record indices of terminal jobs in insertion order. + const terminalIdxs = []; + for (let i = 0; i < jobs.length; i++) { + const j = jobs[i]; + if (j && TERMINAL_STATUSES.has(j.status)) terminalIdxs.push(i); + } + if (terminalIdxs.length <= limit) return null; + + const dropCount = terminalIdxs.length - limit; + const dropSet = new Set(terminalIdxs.slice(0, dropCount)); + + const kept = []; + const dropped = []; + for (let i = 0; i < jobs.length; i++) { + if (dropSet.has(i)) dropped.push(jobs[i]); + else kept.push(jobs[i]); + } + return { kept, dropped }; + } + + const api = { pruneOldestTerminalJobs, TERMINAL_STATUSES }; + + if (typeof module !== 'undefined' && module.exports) { + module.exports = api; + } else if (root) { + root.QueuePrune = api; + } +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/remote-capture-preload.js b/lib/remote-capture-preload.js new file mode 100644 index 0000000..c32fa96 --- /dev/null +++ b/lib/remote-capture-preload.js @@ -0,0 +1,23 @@ +const { contextBridge, ipcRenderer } = require('electron'); + +contextBridge.exposeInMainWorld('capture', { + // Get capture source ID from main process (desktopCapturer runs in main) + getSourceId: () => ipcRenderer.invoke('remote:get-capture-source-id'), + + // Signaling: receive offer/ICE from main process (relayed from dashboard) + onSignaling: (callback) => { + ipcRenderer.on('remote:signaling-to-capture', (_event, data) => callback(data)); + }, + + // Signaling: send answer/ICE back to main process (relayed to dashboard) + sendSignaling: (data) => ipcRenderer.send('remote:signaling-from-capture', data), + + // Input: forward input events from DataChannel to main process + sendInput: (data) => ipcRenderer.send('remote:input-event', data), + + // Notify main process of client connection/disconnection + notifyClientCount: (count) => ipcRenderer.send('remote:client-count', count), + + // Debug logging to main process + log: (...args) => ipcRenderer.send('remote:capture-log', args.join(' ')) +}); diff --git a/lib/remote-capture.html b/lib/remote-capture.html new file mode 100644 index 0000000..64d3699 --- /dev/null +++ b/lib/remote-capture.html @@ -0,0 +1,150 @@ + + +Remote Capture + + + + diff --git a/lib/remote-server.js b/lib/remote-server.js new file mode 100644 index 0000000..486c5e3 --- /dev/null +++ b/lib/remote-server.js @@ -0,0 +1,216 @@ +const { WebSocketServer } = require('ws'); +const crypto = require('crypto'); +const { evaluateClientAllowed } = require('./ip-allowlist'); + +function timingSafeEqualStr(a, b) { + const x = Buffer.from(String(a == null ? '' : a)); + const y = Buffer.from(String(b == null ? '' : b)); + return x.length === y.length && crypto.timingSafeEqual(x, y); +} + +class RemoteServer { + constructor() { + this._wss = null; + this._clients = new Map(); // ws -> { id, role, authenticated } + this._config = null; + this._failedAttempts = new Map(); // ip -> { count, blockedUntil } + this._lastAccess = null; + } + + start(opts) { + return new Promise((resolve, reject) => { + this._config = opts; + + const wssOpts = { port: opts.port, maxPayload: 256 * 1024 }; + if (opts.host) wssOpts.host = opts.host; + this._wss = new WebSocketServer(wssOpts, () => { + resolve(); + }); + + this._wss.on('error', (err) => { + reject(err); + }); + + this._wss.on('connection', (ws, req) => { + this._handleConnection(ws, req); + }); + }); + } + + stop() { + if (this._wss) { + for (const [ws] of this._clients) { + ws.close(1000, 'Server shutting down'); + } + this._clients.clear(); + this._wss.close(); + this._wss = null; + } + } + + getClientCount() { + let count = 0; + for (const [, client] of this._clients) { + if (client.authenticated) count++; + } + return count; + } + + getPort() { + if (this._wss && this._wss.address()) { + return this._wss.address().port; + } + return null; + } + + _handleConnection(ws, req) { + const ip = req.socket.remoteAddress || 'unknown'; + + if (this._isBlocked(ip)) { + ws.close(4003, 'Too many failed attempts'); + return; + } + + if (Array.isArray(this._config.allowlist) && !evaluateClientAllowed(ip, this._config.allowlist)) { + ws.close(4005, 'Client IP not allowed'); + return; + } + + const clientId = crypto.randomUUID(); + this._clients.set(ws, { id: clientId, role: null, authenticated: false }); + + let authReceived = false; + const authTimeout = setTimeout(() => { + if (!authReceived) { + ws.close(4001, 'Auth timeout'); + this._clients.delete(ws); + } + }, 5000); + + ws.on('message', (raw) => { + let msg; + try { msg = JSON.parse(raw); } catch { return; } + + const client = this._clients.get(ws); + if (!client) return; + + if (!client.authenticated) { + authReceived = true; + clearTimeout(authTimeout); + + if (msg.type === 'auth' && timingSafeEqualStr(msg.token, this._config.token)) { + client.authenticated = true; + client.role = this._config.diagnosticMode ? 'diagnostic' : (msg.role || 'viewer'); + this._lastAccess = Date.now(); + ws.send(JSON.stringify({ type: 'auth-ok', clientId })); + + if (!this._config.diagnosticMode && this.getClientCount() === 1) { + this._config.onCreateCaptureWindow(); + } + } else { + this._recordFailedAttempt(ip); + ws.close(4002, 'Invalid token'); + this._clients.delete(ws); + } + return; + } + + if (this._config.diagnosticMode) { + if (msg.type === 'diag-request' && typeof this._config.onDiagnosticRequest === 'function') { + this._lastAccess = Date.now(); + this._config.onDiagnosticRequest(msg, client, (payload) => { + this.sendToClient(client.id, { type: 'diag-response', reqId: msg.reqId, ...payload }); + }); + } + return; + } + + if (msg.type === 'offer' || msg.type === 'ice-candidate') { + msg.clientId = client.id; + msg.role = client.role; + this._config.onSignalingToCapture(msg); + } + }); + + ws.on('close', () => { + clearTimeout(authTimeout); + const client = this._clients.get(ws); + const wasAuthenticated = client && client.authenticated; + this._clients.delete(ws); + + if (wasAuthenticated && !this._config.diagnosticMode) { + this._config.onSignalingToCapture({ + type: 'client-disconnected', + clientId: client.id + }); + + if (this.getClientCount() === 0) { + this._config.onDestroyCaptureWindow(); + } + } + }); + + ws.on('error', () => { + clearTimeout(authTimeout); + const client = this._clients.get(ws); + const wasAuthenticated = client && client.authenticated; + this._clients.delete(ws); + + if (wasAuthenticated && !this._config.diagnosticMode) { + this._config.onSignalingToCapture({ + type: 'client-disconnected', + clientId: client.id + }); + if (this.getClientCount() === 0) { + this._config.onDestroyCaptureWindow(); + } + } + }); + } + + getLastAccess() { + return this._lastAccess; + } + + sendToClient(clientId, data) { + for (const [ws, client] of this._clients) { + if (client.id === clientId && client.authenticated) { + if (ws.readyState === 1) { + try { ws.send(JSON.stringify(data)); } catch {} + } + break; + } + } + } + + broadcast(data) { + const msg = JSON.stringify(data); + for (const [ws, client] of this._clients) { + if (client.authenticated && ws.readyState === 1) { + ws.send(msg); + } + } + } + + _isBlocked(ip) { + const entry = this._failedAttempts.get(ip); + if (!entry) return false; + if (entry.blockedUntil && Date.now() < entry.blockedUntil) return true; + if (entry.blockedUntil && Date.now() >= entry.blockedUntil) { + this._failedAttempts.delete(ip); + return false; + } + return false; + } + + _recordFailedAttempt(ip) { + const entry = this._failedAttempts.get(ip) || { count: 0, blockedUntil: null }; + entry.count++; + if (entry.count >= 5) { + entry.blockedUntil = Date.now() + 60000; + } + this._failedAttempts.set(ip, entry); + } +} + +module.exports = RemoteServer; diff --git a/lib/secret-store.js b/lib/secret-store.js new file mode 100644 index 0000000..51412e9 --- /dev/null +++ b/lib/secret-store.js @@ -0,0 +1,75 @@ +// Wraps Electron's safeStorage (OS-level credential encryption: DPAPI on +// Windows, Keychain on macOS, libsecret on Linux) to keep hoster passwords and +// API keys out of the plaintext electron-config.json. +// +// On Windows the DPAPI key is tied to the current user profile, so credentials +// encrypted here are only readable by the same Windows user. For backups we +// export to plaintext (the .mhu envelope has its own AES-GCM layer) so moving +// between machines/users works transparently. + +const SENTINEL = 'enc:v1:'; +const CRED_FIELDS = ['password', 'apiKey']; + +let _safeStorageCache = undefined; +function getSafeStorage() { + if (_safeStorageCache !== undefined) return _safeStorageCache; + try { + const { safeStorage } = require('electron'); + if (safeStorage && typeof safeStorage.isEncryptionAvailable === 'function' + && safeStorage.isEncryptionAvailable()) { + _safeStorageCache = safeStorage; + return _safeStorageCache; + } + } catch {} + _safeStorageCache = null; + return null; +} + +function isEncrypted(value) { + return typeof value === 'string' && value.startsWith(SENTINEL); +} + +function encryptField(value) { + if (!value || typeof value !== 'string') return value; + if (isEncrypted(value)) return value; + const ss = getSafeStorage(); + if (!ss) return value; + try { + const buf = ss.encryptString(value); + return SENTINEL + buf.toString('base64'); + } catch { + return value; + } +} + +function decryptField(value) { + if (!value || typeof value !== 'string') return value; + if (!isEncrypted(value)) return value; + const ss = getSafeStorage(); + if (!ss) return ''; + try { + const buf = Buffer.from(value.slice(SENTINEL.length), 'base64'); + return ss.decryptString(buf); + } catch { + return ''; + } +} + +function mapHosterAccounts(config, fn) { + if (!config || !config.hosters || typeof config.hosters !== 'object') return config; + for (const accounts of Object.values(config.hosters)) { + if (!Array.isArray(accounts)) continue; + for (const acc of accounts) { + if (!acc || typeof acc !== 'object') continue; + for (const f of CRED_FIELDS) { + if (acc[f]) acc[f] = fn(acc[f]); + } + } + } + return config; +} + +function encryptCredentials(config) { return mapHosterAccounts(config, encryptField); } +function decryptCredentials(config) { return mapHosterAccounts(config, decryptField); } + +module.exports = { encryptField, decryptField, encryptCredentials, decryptCredentials, isEncrypted }; diff --git a/lib/semaphore.js b/lib/semaphore.js new file mode 100644 index 0000000..0c0a179 --- /dev/null +++ b/lib/semaphore.js @@ -0,0 +1,73 @@ +/** + * FIFO Semaphore for per-hoster concurrency control. + * acquire(signal?) blocks until a slot is available or the signal aborts. + * release() frees a slot. + */ +class Semaphore { + constructor(limit) { + this.limit = Math.max(1, limit || 1); + this.active = 0; + this.queue = []; // { resolve, reject, signal?, onAbort? } + } + + acquire(signal) { + return new Promise((resolve, reject) => { + if (signal && signal.aborted) { + reject(new Error('Aborted')); + return; + } + + if (this.active < this.limit) { + this.active++; + resolve(); + return; + } + + const entry = { resolve, reject }; + + if (signal) { + entry.signal = signal; + entry.onAbort = () => { + const idx = this.queue.indexOf(entry); + if (idx !== -1) this.queue.splice(idx, 1); + reject(new Error('Aborted')); + }; + signal.addEventListener('abort', entry.onAbort, { once: true }); + } + + this.queue.push(entry); + }); + } + + _cleanupEntry(entry) { + if (entry.signal && entry.onAbort) { + entry.signal.removeEventListener('abort', entry.onAbort); + } + } + + release() { + if (this.queue.length > 0) { + const entry = this.queue.shift(); + this._cleanupEntry(entry); + entry.resolve(); + } else { + this.active = Math.max(0, this.active - 1); + } + } + + updateLimit(newLimit) { + this.limit = Math.max(1, newLimit || 1); + while (this.active < this.limit && this.queue.length > 0) { + this.active++; + const entry = this.queue.shift(); + this._cleanupEntry(entry); + entry.resolve(); + } + } + + get pending() { + return this.queue.length; + } +} + +module.exports = Semaphore; diff --git a/lib/stats.js b/lib/stats.js new file mode 100644 index 0000000..277c21e --- /dev/null +++ b/lib/stats.js @@ -0,0 +1,142 @@ +(function (root) { + function summarizePerHoster(history, opts) { + const out = {}; + if (!Array.isArray(history)) return out; + const cutoff = opts && Number.isFinite(opts.sinceMs) ? opts.sinceMs : null; + const limitBatches = opts && Number.isFinite(opts.lastNBatches) && opts.lastNBatches > 0 ? opts.lastNBatches : null; + + const entries = [...history]; + entries.sort((a, b) => { + const ta = a && a.timestamp ? Date.parse(a.timestamp) : 0; + const tb = b && b.timestamp ? Date.parse(b.timestamp) : 0; + return tb - ta; + }); + const sliced = limitBatches ? entries.slice(0, limitBatches) : entries; + + for (const batch of sliced) { + if (!batch || !Array.isArray(batch.files)) continue; + if (cutoff !== null) { + const ts = batch.timestamp ? Date.parse(batch.timestamp) : 0; + if (!ts || ts < cutoff) continue; + } + for (const file of batch.files) { + if (!file || !Array.isArray(file.results)) continue; + for (const r of file.results) { + if (!r || !r.hoster) continue; + const bucket = out[r.hoster] || (out[r.hoster] = { ok: 0, fail: 0, total: 0 }); + bucket.total++; + if (r.status === 'done') bucket.ok++; + else bucket.fail++; + } + } + } + for (const h of Object.keys(out)) { + const b = out[h]; + b.rate = b.total > 0 ? b.ok / b.total : null; + } + return out; + } + + function classifyErrorCategory(err) { + if (!err || typeof err !== 'string') return 'unknown'; + const s = err.toLowerCase(); + if (/abgebrochen|aborted|cancel/.test(s)) return 'aborted'; + if (/not video file format|kein videoformat|invalid file|wrong format|duplicate|already exists|file too (small|big|large)|datei zu (gro|klein)/.test(s)) return 'file-rejected'; + if (/quota|storage (full|exhausted|voll)|account (full|banned|suspended)|disk (space )?full|insufficient (disk )?space|not enough (disk )?(space|storage)/.test(s)) return 'account-error'; + if (/csrf|kein upload-server|server.*?(busy|unavailable|try again)|no servers available|filecode|kein filecode|empty.*?(form|response)/.test(s)) return 'hoster-transient'; + if (/timeout|econnreset|enotfound|fetch failed|network|socket hang up|abort/.test(s)) return 'network'; + return 'unknown'; + } + + function summarizeBatchErrors(batchSummary) { + const buckets = { + 'file-rejected': [], + 'account-error': [], + 'hoster-transient': [], + 'network': [], + 'unknown': [], + 'aborted': [] + }; + if (!batchSummary || !Array.isArray(batchSummary.files)) return buckets; + for (const f of batchSummary.files) { + if (!f || !Array.isArray(f.results)) continue; + for (const r of f.results) { + if (!r || r.status === 'done') continue; + const cat = classifyErrorCategory(r.error); + buckets[cat].push({ + fileName: f.name || f.fileName || '', + hoster: r.hoster || '', + error: r.error || '', + jobId: r.jobId || null + }); + } + } + return buckets; + } + + const RETRYABLE_CATEGORIES = new Set(['hoster-transient', 'network', 'unknown']); + function isRetryableCategory(cat) { + return RETRYABLE_CATEGORIES.has(cat); + } + + const CATEGORY_LABELS = { + 'file-rejected': 'Datei abgelehnt', + 'account-error': 'Account-Problem', + 'hoster-transient': 'Hoster-Flake', + 'network': 'Netzwerk', + 'unknown': 'Unbekannt', + 'aborted': 'Abgebrochen' + }; + + function formatLinks(rows, format) { + if (!Array.isArray(rows)) return ''; + const safe = rows.filter(r => r && r.url); + if (safe.length === 0) return ''; + switch (format) { + case 'plain': + return safe.map(r => r.url).join('\n'); + case 'bbcode': + return safe.map(r => { + const label = r.fileName || r.hoster || r.url; + return `[url=${r.url}]${label}[/url]`; + }).join('\n'); + case 'markdown': + return safe.map(r => { + const label = r.fileName || r.hoster || r.url; + return `- [${label}](${r.url})`; + }).join('\n'); + case 'html': + return safe.map(r => { + const label = r.fileName || r.hoster || r.url; + return `${label}`; + }).join('\n'); + case 'csv': { + const head = 'fileName,hoster,url\n'; + return head + safe.map(r => { + const esc = (v) => `"${String(v || '').replace(/"/g, '""')}"`; + return [esc(r.fileName), esc(r.hoster), esc(r.url)].join(','); + }).join('\n'); + } + case 'json': + return JSON.stringify(safe.map(r => ({ fileName: r.fileName || '', hoster: r.hoster || '', url: r.url })), null, 2); + default: + return safe.map(r => r.url).join('\n'); + } + } + + const api = { + summarizePerHoster, + classifyErrorCategory, + summarizeBatchErrors, + isRetryableCategory, + RETRYABLE_CATEGORIES, + CATEGORY_LABELS, + formatLinks + }; + + if (typeof module !== 'undefined' && module.exports) { + module.exports = api; + } else if (root) { + root.Stats = api; + } +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/support-bundle.js b/lib/support-bundle.js new file mode 100644 index 0000000..5ba0dc0 --- /dev/null +++ b/lib/support-bundle.js @@ -0,0 +1,112 @@ +const fs = require('fs'); + +const CRED_KEYS = new Set(['password', 'apiKey', 'token', 'cookie', 'sessionId', 'webhookUrl', 'diagToken']); +const REDACTED = ''; + +function sanitizeConfig(config) { + if (!config || typeof config !== 'object') return config; + const clone = JSON.parse(JSON.stringify(config)); + (function walk(o) { + if (!o) return; + if (Array.isArray(o)) { for (const e of o) walk(e); return; } + if (typeof o !== 'object') return; + for (const k of Object.keys(o)) { + if (CRED_KEYS.has(k) && typeof o[k] === 'string' && o[k]) o[k] = REDACTED; + else walk(o[k]); + } + })(clone); + return clone; +} + +function collectSecretValues(config) { + const out = new Set(); + (function walk(o) { + if (!o) return; + if (Array.isArray(o)) { for (const e of o) walk(e); return; } + if (typeof o !== 'object') return; + for (const k of Object.keys(o)) { + const v = o[k]; + if (CRED_KEYS.has(k) && typeof v === 'string' && v.length >= 6) out.add(v); + else walk(v); + } + })(config); + return Array.from(out); +} + +function redactLogText(text, secrets) { + if (typeof text !== 'string' || !text) return text; + let out = text; + if (Array.isArray(secrets)) { + for (const s of secrets) { + if (typeof s === 'string' && s.length >= 6) out = out.split(s).join(REDACTED); + } + } + out = out + .replace(/https?:\/\/(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/[\w-]+/gi, 'https://discord.com/api/webhooks/' + REDACTED) + .replace(/(\/\/[^\s/:@]+:)[^\s/@]+(@)/g, '$1' + REDACTED + '$2') + .replace(/(authorization:\s*(?:bearer|basic)\s+)\S+/gi, '$1' + REDACTED) + .replace(/\bbearer\s+[A-Za-z0-9._\-/+]{16,}/gi, 'bearer ' + REDACTED) + .replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{6,}/g, REDACTED) + .replace(/([?&](?:api_?key|key|token|access_token|password|pass)=)[^\s&"'`]+/gi, '$1' + REDACTED) + .replace(/("?\b(?:api[_-]?key|apikey|password|passwd|secret|(?:access|refresh|auth|session)[_-]?token|token|sessionid|session)"?\s*[:=]\s*"?)[A-Za-z0-9._\-/+]{8,}/gi, '$1' + REDACTED) + .replace(/(\bset-cookie:|\bcookie:)\s*\S[^\n]*/gi, '$1 ' + REDACTED) + .replace(/(\bsess(?:_?id)?\b["'=:\s]+)[A-Za-z0-9._\-]{8,}/gi, '$1' + REDACTED); + return out; +} + +function valueScrub(value, secrets) { + if (value == null) return value; + const json = JSON.stringify(value); + let scrubbed = json; + if (Array.isArray(secrets)) { + for (const s of secrets) { + if (typeof s === 'string' && s.length >= 6) scrubbed = scrubbed.split(s).join(REDACTED); + } + } + return JSON.parse(scrubbed); +} + +function collectFile(filePath, label, maxBytes) { + if (!filePath) return `=== ${label} ===\n\n\n`; + let stat; + try { stat = fs.statSync(filePath); } + catch (err) { + if (err && err.code === 'ENOENT') return `=== ${label} (${filePath}) ===\n\n\n`; + return `=== ${label} (${filePath}) ===\n\n\n`; + } + const cap = Number.isFinite(maxBytes) && maxBytes > 0 ? maxBytes : 5 * 1024 * 1024; + let content; + try { + if (stat.size > cap) { + const fd = fs.openSync(filePath, 'r'); + const buf = Buffer.alloc(cap); + fs.readSync(fd, buf, 0, cap, stat.size - cap); + fs.closeSync(fd); + const skipped = stat.size - cap; + content = `\n` + buf.toString('utf-8'); + } else { + content = fs.readFileSync(filePath, 'utf-8'); + } + } catch (err) { + content = ``; + } + return `=== ${label} (${filePath}, size=${stat.size} bytes) ===\n${content}\n\n`; +} + +function buildSupportBundleText({ header, sanitizedConfig, files }) { + const parts = []; + parts.push('=== Multi-Hoster-Upload Support Bundle ===\n'); + if (header && typeof header === 'object') { + for (const [k, v] of Object.entries(header)) parts.push(`${k}: ${v}\n`); + } + parts.push('\n'); + parts.push('=== Config (sanitized — password/apiKey/token/cookie/sessionId redacted) ===\n'); + parts.push(JSON.stringify(sanitizedConfig, null, 2)); + parts.push('\n\n'); + for (const f of (files || [])) { + parts.push(collectFile(f.path, f.label || f.path, f.maxBytes)); + } + return parts.join(''); +} + +module.exports = { sanitizeConfig, collectSecretValues, redactLogText, valueScrub, collectFile, buildSupportBundleText, CRED_KEYS, REDACTED }; diff --git a/lib/throttle-timer.js b/lib/throttle-timer.js new file mode 100644 index 0000000..c4a9d52 --- /dev/null +++ b/lib/throttle-timer.js @@ -0,0 +1,59 @@ +(function (root) { + 'use strict'; + + function makeThrottleTimer(opts) { + const o = opts || {}; + const now = typeof o.now === 'function' ? o.now : (() => Date.now()); + const schedule = typeof o.schedule === 'function' + ? o.schedule + : ((cb, ms) => setTimeout(cb, ms)); + const clear = typeof o.clear === 'function' ? o.clear : ((h) => clearTimeout(h)); + + let handle = null; + let burstStart = null; + let pendingFn = null; + + function fire() { + handle = null; + burstStart = null; + const fn = pendingFn; + pendingFn = null; + if (typeof fn === 'function') fn(); + } + + function request(fn, delay, maxWait) { + if (typeof fn === 'function') pendingFn = fn; + const t = now(); + if (burstStart === null) burstStart = t; + let wait = typeof delay === 'number' && delay >= 0 ? delay : 0; + if (typeof maxWait === 'number' && maxWait >= 0) { + const remaining = maxWait - (t - burstStart); + wait = Math.min(wait, remaining < 0 ? 0 : remaining); + } + if (handle !== null) clear(handle); + handle = schedule(fire, wait); + } + + function flushSync() { + if (handle !== null) { clear(handle); handle = null; } + burstStart = null; + const fn = pendingFn; + pendingFn = null; + if (typeof fn === 'function') fn(); + } + + function cancel() { + if (handle !== null) { clear(handle); handle = null; } + burstStart = null; + pendingFn = null; + } + + function isPending() { return handle !== null; } + + return { request, flushSync, cancel, isPending }; + } + + const api = { makeThrottleTimer }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.ThrottleTimer = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/throttle.js b/lib/throttle.js new file mode 100644 index 0000000..abb1ad0 --- /dev/null +++ b/lib/throttle.js @@ -0,0 +1,43 @@ +/** + * Token-bucket speed limiter for bandwidth throttling. + * maxBytesPerSec = 0 means unlimited (passthrough). + */ +class Throttle { + constructor(maxBytesPerSec) { + this.maxBps = maxBytesPerSec || 0; + this.tokens = this.maxBps; + this.lastRefill = Date.now(); + } + + async consume(bytes, signal) { + if (this.maxBps <= 0) return; // unlimited + + while (bytes > 0) { + if (signal && signal.aborted) return; + this._refill(); + const available = Math.min(bytes, Math.floor(this.tokens)); + if (available > 0) { + this.tokens -= available; + bytes -= available; + } + if (bytes > 0) { + if (signal && signal.aborted) return; + // Wait 50ms for tokens to refill + await new Promise((r) => setTimeout(r, 50)); + } + } + } + + _refill() { + const now = Date.now(); + const elapsed = (now - this.lastRefill) / 1000; + this.tokens = Math.min(this.maxBps, this.tokens + elapsed * this.maxBps); + this.lastRefill = now; + } + + updateRate(maxBytesPerSec) { + this.maxBps = maxBytesPerSec || 0; + } +} + +module.exports = Throttle; diff --git a/lib/throttled-cache.js b/lib/throttled-cache.js new file mode 100644 index 0000000..c6376cf --- /dev/null +++ b/lib/throttled-cache.js @@ -0,0 +1,51 @@ +// Time-windowed memoization. Reuses a previously-computed value if the +// signature + input identity match AND the cached entry is younger than +// `refreshMs`. Used by the renderer's dynamic-key sort throttle (every +// progress tick re-sorts a 5000-row queue → reuse for 200 ms, the user +// can't perceive sub-200 ms reorder lag). +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag) — same single implementation +// across runtime and tests. + +(function (root) { + 'use strict'; + + /** + * Build a throttled cache. The clock is injected so tests don't have to + * sleep — pass `() => fakeClock.value` from tests. + * + * @param {number} refreshMs cache TTL in milliseconds + * @param {() => number} [now] clock source, defaults to Date.now + */ + function makeThrottledCache(refreshMs, now) { + if (!Number.isFinite(refreshMs) || refreshMs < 0) { + throw new TypeError('refreshMs must be a non-negative finite number'); + } + const clock = typeof now === 'function' ? now : () => Date.now(); + let entry = null; + return { + get(sig, input) { + if (!entry) return undefined; + if (entry.sig !== sig) return undefined; + if (entry.input !== input) return undefined; + if (clock() - entry.ts >= refreshMs) return undefined; + return entry.value; + }, + set(sig, input, value) { + entry = { sig, input, value, ts: clock() }; + return value; + }, + clear() { entry = null; }, + // Introspection (mainly for tests/debug). Returns null when empty. + peek() { + if (!entry) return null; + return { sig: entry.sig, ts: entry.ts, age: clock() - entry.ts }; + } + }; + } + + const api = { makeThrottledCache }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.ThrottledCache = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/updater.js b/lib/updater.js new file mode 100644 index 0000000..ac38810 --- /dev/null +++ b/lib/updater.js @@ -0,0 +1,283 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { app } = require('electron'); + +const UPDATE_REPO = 'Administrator/Multi-Hoster-Upload'; +const GITEA_BASE = 'https://git.24-music.de'; +const API_URL = `${GITEA_BASE}/api/v1/repos/${UPDATE_REPO}/releases?limit=1`; + +const CHECK_TIMEOUT = 15000; + +let cachedCheck = null; +let cachedCheckTs = 0; +const CACHE_TTL = 10 * 60 * 1000; // 10 min + +let activeAbort = null; + +function getCurrentVersion() { + return app.getVersion(); +} + +function parseVersion(str) { + const clean = String(str || '').replace(/^v/i, '').trim(); + const parts = clean.split('.').map(Number); + return { + major: parts[0] || 0, + minor: parts[1] || 0, + patch: parts[2] || 0 + }; +} + +function isNewer(remote, current) { + const r = parseVersion(remote); + const c = parseVersion(current); + if (r.major !== c.major) return r.major > c.major; + if (r.minor !== c.minor) return r.minor > c.minor; + return r.patch > c.patch; +} + +function pickSetupAsset(assets) { + if (!Array.isArray(assets)) return null; + // Prefer asset with "setup" in the name (case-insensitive) + const setup = assets.find(a => + /setup/i.test(a.name) && /\.exe$/i.test(a.name) + ); + if (setup) return setup; + // Fallback: any .exe + return assets.find(a => /\.exe$/i.test(a.name)) || null; +} + +function findLatestYml(assets) { + if (!Array.isArray(assets)) return null; + return assets.find(a => /^latest\.yml$/i.test(a.name)) || null; +} + +async function fetchJson(url, signal) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), CHECK_TIMEOUT); + const onAbort = () => controller.abort(); + if (signal) signal.addEventListener('abort', onAbort); + + try { + const res = await fetch(url, { + method: 'GET', + signal: controller.signal, + redirect: 'follow' + }); + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + throw new Error(`Update-Server Antwort war kein JSON (HTTP ${res.status}): ${text.slice(0, 200)}`); + } + } finally { + clearTimeout(timeout); + if (signal) signal.removeEventListener('abort', onAbort); + } +} + +async function checkForUpdate() { + // Return cached result if fresh + if (cachedCheck && (Date.now() - cachedCheckTs) < CACHE_TTL) { + return cachedCheck; + } + + const releases = await fetchJson(API_URL); + + if (!Array.isArray(releases) || releases.length === 0) { + return { available: false }; + } + + const release = releases[0]; + const remoteVersion = release.tag_name || release.name || ''; + const currentVersion = getCurrentVersion(); + + if (!isNewer(remoteVersion, currentVersion)) { + cachedCheck = { available: false, currentVersion, remoteVersion }; + cachedCheckTs = Date.now(); + return cachedCheck; + } + + const setupAsset = pickSetupAsset(release.assets); + const latestYml = findLatestYml(release.assets); + + if (!setupAsset) { + return { available: false, reason: 'Kein Setup-Asset im Release gefunden' }; + } + + cachedCheck = { + available: true, + currentVersion, + remoteVersion: remoteVersion.replace(/^v/i, ''), + releaseUrl: release.html_url, + assetUrl: setupAsset.browser_download_url, + assetSize: setupAsset.size, + assetName: setupAsset.name, + latestYmlUrl: latestYml ? latestYml.browser_download_url : null, + releaseNotes: release.body || '' + }; + cachedCheckTs = Date.now(); + return cachedCheck; +} + +async function parseLatestYml(url) { + if (!url) return null; + try { + const res = await fetch(url, { redirect: 'follow' }); + const text = await res.text(); + // Extract sha512 from latest.yml + const match = text.match(/sha512:\s*([A-Za-z0-9+/=]+)/); + return match ? match[1] : null; + } catch { + return null; + } +} + +function verifyExeHeader(buf) { + // Check MZ header + if (buf.length < 128 * 1024) return false; + return buf[0] === 0x4D && buf[1] === 0x5A; // 'MZ' +} + +async function installUpdate(onProgress) { + if (activeAbort) activeAbort.abort(); + activeAbort = new AbortController(); + const signal = activeAbort.signal; + + try { + // Stage: starting + if (onProgress) onProgress({ stage: 'starting', percent: 0 }); + + // Check or use cached + let check = cachedCheck; + if (!check || !check.available) { + check = await checkForUpdate(); + } + if (!check || !check.available) { + throw new Error('Kein Update verfuegbar'); + } + if (!check.assetUrl || !check.assetName) { + throw new Error('Update-Asset unvollstaendig (URL oder Name fehlt)'); + } + + // Stage: downloading + const tmpDir = app.getPath('temp'); + const installerPath = path.join(tmpDir, check.assetName); + + const res = await fetch(check.assetUrl, { + method: 'GET', + signal, + redirect: 'follow' + }); + + if (!res.ok) { + throw new Error(`Download fehlgeschlagen: HTTP ${res.status}`); + } + + const totalBytes = check.assetSize || 0; + let downloadedBytes = 0; + const chunks = []; + + const DOWNLOAD_STALL_MS = 45000; + let stallTimer = null; + const reader = res.body.getReader(); + while (true) { + if (signal.aborted) throw new Error('Abgebrochen'); + let chunk; + try { + chunk = await Promise.race([ + reader.read(), + new Promise((_, reject) => { stallTimer = setTimeout(() => reject(new Error('__STALL__')), DOWNLOAD_STALL_MS); }) + ]); + } catch (e) { + if (e && e.message === '__STALL__') { + try { activeAbort.abort(); } catch {} + throw new Error('Download hängt — seit 45 s keine Daten (Netzwerk/Server überlastet). Bitte laufende Uploads stoppen und erneut versuchen.'); + } + throw e; + } finally { + if (stallTimer) { clearTimeout(stallTimer); stallTimer = null; } + } + const { done, value } = chunk; + if (done) break; + chunks.push(value); + downloadedBytes += value.length; + if (onProgress) { + onProgress({ + stage: 'downloading', + percent: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0, + bytesDownloaded: downloadedBytes, + bytesTotal: totalBytes + }); + } + } + + const fileBuffer = Buffer.concat(chunks); + + // Stage: verifying + if (onProgress) onProgress({ stage: 'verifying', percent: 0 }); + + if (!verifyExeHeader(fileBuffer)) { + throw new Error('Heruntergeladene Datei ist keine gueltige EXE'); + } + + // Optional SHA-512 verification from latest.yml + const expectedSha = await parseLatestYml(check.latestYmlUrl); + if (expectedSha) { + const actualSha = crypto.createHash('sha512').update(fileBuffer).digest('base64'); + if (actualSha !== expectedSha) { + // Try hex comparison + const actualHex = crypto.createHash('sha512').update(fileBuffer).digest('hex'); + if (actualHex !== expectedSha.toLowerCase()) { + throw new Error('SHA-512 Pruefung fehlgeschlagen'); + } + } + } + + // Write to disk + fs.writeFileSync(installerPath, fileBuffer); + + // Stage: launching + if (onProgress) onProgress({ stage: 'launching', percent: 100 }); + + const { spawn } = require('child_process'); + spawn(installerPath, ['/S', '--updated', '--force-run'], { + detached: true, + stdio: 'ignore' + }).unref(); + + // Stage: done + if (onProgress) onProgress({ stage: 'done', percent: 100 }); + + const _doQuit = () => setTimeout(() => app.quit(), 900); + const _getActive = () => { + try { return globalThis._mhuUploadManagerRef && globalThis._mhuUploadManagerRef.getActiveJobCount ? globalThis._mhuUploadManagerRef.getActiveJobCount() : 0; } + catch { return 0; } + }; + if (_getActive() > 0) { + const POLL_MS = 3000; + const poller = setInterval(() => { + if (_getActive() === 0) { clearInterval(poller); _doQuit(); } + }, POLL_MS); + setTimeout(() => { try { clearInterval(poller); } catch {} _doQuit(); }, 30 * 60 * 1000); + } else { + _doQuit(); + } + + } catch (err) { + if (onProgress) onProgress({ stage: 'error', error: err.message }); + throw err; + } finally { + activeAbort = null; + } +} + +function abortUpdate() { + if (activeAbort) { + activeAbort.abort(); + activeAbort = null; + } +} + +module.exports = { checkForUpdate, installUpdate, abortUpdate }; diff --git a/lib/upload-log.js b/lib/upload-log.js new file mode 100644 index 0000000..6025791 --- /dev/null +++ b/lib/upload-log.js @@ -0,0 +1,34 @@ +(function (root) { + 'use strict'; + + function _pad(n) { return String(n).padStart(2, '0'); } + + function formatUploadLogLine(date, hoster, link, fileName) { + const d = date instanceof Date ? date : new Date(); + const dateStr = `${d.getFullYear()}-${_pad(d.getMonth() + 1)}-${_pad(d.getDate())} ` + + `${_pad(d.getHours())}:${_pad(d.getMinutes())}:${_pad(d.getSeconds())}`; + return `${dateStr}|${hoster}|${link}||${fileName}|\n`; + } + + function parseUploadLogLine(line) { + if (typeof line !== 'string') return null; + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) return null; + const parts = trimmed.split('|'); + if (parts.length < 5) return null; + const hoster = (parts[1] || '').trim(); + let fileName = ''; + for (let i = parts.length - 1; i >= 4; i--) { + if (parts[i].trim() !== '') { fileName = parts[i]; break; } + } + if (!hoster || !fileName) return null; + const tsStr = (parts[0] || '').trim(); + const tsParsed = tsStr ? Date.parse(tsStr.replace(' ', 'T')) : NaN; + const ts = isNaN(tsParsed) ? undefined : tsParsed; + return { hoster, fileName, ts }; + } + + const api = { formatUploadLogLine, parseUploadLogLine }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.UploadLog = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/upload-manager.js b/lib/upload-manager.js new file mode 100644 index 0000000..c95f1aa --- /dev/null +++ b/lib/upload-manager.js @@ -0,0 +1,1423 @@ +const { EventEmitter } = require('events'); +const path = require('path'); +const fs = require('fs'); +const crypto = require('crypto'); +const { uploadFile, prefetchBaseline } = require('./hosters'); +const VidmolyUploader = require('./vidmoly-upload'); +const VoeUploader = require('./voe-upload'); +const DoodstreamUploader = require('./doodstream-upload'); +const ClouddropUploader = require('./clouddrop-upload'); +const Semaphore = require('./semaphore'); +const Throttle = require('./throttle'); +const { probeFileHead } = require('./file-probe'); + +const DEFAULT_SETTINGS = { + retries: 3, + maxSpeedKbs: 0, + parallelCount: 2, + restartBelowKbs: 0, + timeIntervalSec: 0, + maxSizeMb: 0, + sizeMemoEnabled: true +}; + +class UploadManager extends EventEmitter { + constructor(hosterSettings, globalSettings, accountPools) { + super(); + this.hosterSettings = hosterSettings || {}; + this.globalSettings = globalSettings || {}; + this.accountPools = accountPools || {}; + this.semaphores = {}; + this.globalSemaphore = null; + this.abortController = new AbortController(); + this.running = false; + this.stopAfterActive = false; + this.statsInterval = null; + this.startTime = 0; + this.activeJobs = new Map(); // uploadId -> { jobId, speedKbs, bytesUploaded, hoster } + this.jobAbortControllers = new Map(); // jobId -> AbortController + this.cancelledJobIds = new Set(); + this.sessionBytes = 0; + this._transientErrorTotal = 0; + this.lastStartTime = {}; // hoster -> timestamp of last upload start + this.intervalLocks = {}; // hoster -> Promise chain for serialized interval waits + this.globalThrottle = null; + this._failedAccounts = new Map(); // hoster -> Set of failed accountIds + this._accountOverrides = new Map(); // hoster -> fallback account object + this._suspectSizeMemo = new Map(); // 'hoster:accountId' -> { size: smallest suspect-rejected fileSize, count: confirmed rejections }; blocks only after 2nd rejection + this._suspectGoodAccounts = new Map(); // hoster -> accountId that accepted a suspect-class file + this._doodApiKeyCache = new Map(); // accountId/username -> derived doodstream API key ('' = tried, none) + this._baselineCache = new Map(); // hoster:apiKey -> Promise> (one fetch shared across all jobs in batch) + } + + updateAccountPools(accountPools) { + if (accountPools && typeof accountPools === 'object') { + this.accountPools = accountPools; + } + } + + switchAccount(hoster, fallbackAccount) { + const prev = this._accountOverrides.get(hoster); + this._accountOverrides.set(hoster, fallbackAccount); + this._rotLog('switchAccount', { + hoster, + prevOverrideId: prev ? prev.id : null, + toAccountId: fallbackAccount ? fallbackAccount.id : null + }); + } + + // Introspection helpers used by main.js to re-resolve fallbacks when the + // config changes mid-batch (e.g. user adds a new account after their only + // one ran out of space). Without this, an account that got marked failed + // before a fallback existed stays stuck until the app restarts. + getFailedAccountKeys() { + return Array.from(this._failedAccounts.keys()); + } + + getOverride(hoster) { + return this._accountOverrides.get(hoster) || null; + } + + getActiveJobCount() { + return this.activeJobs.size; + } + + getDiagnostics() { + const activeByHoster = {}; + for (const v of this.activeJobs.values()) { + const h = v && v.hoster ? v.hoster : 'unknown'; + activeByHoster[h] = (activeByHoster[h] || 0) + 1; + } + let pending = 0; + for (const sem of Object.values(this.semaphores)) pending += (sem && sem.pending) || 0; + return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size }; + } + + clearFailedAccount(hoster, accountId) { + return this._failedAccounts.delete(`${hoster}:${accountId}`); + } + + clearAllFailedAccounts() { + const n = this._failedAccounts.size; + this._failedAccounts.clear(); + return n; + } + + // True if the hoster has a usable override stored that differs from the + // account currently in the task and isn't itself already marked failed. + // Used by the retry loop to decide "retry on same account vs break to + // rotation" — skipping wasted attempts on a likely-bad primary when a + // pre-resolved fallback is ready to try. + _hasPendingOverride(hoster, currentAccountId) { + const override = this._accountOverrides.get(hoster); + if (!override) return false; + if (override.id === currentAccountId) return false; + if (this._failedAccounts.has(hoster + ':' + override.id)) return false; + return true; + } + + _rotLog(event, data) { + this.emit('rot-log', { ts: Date.now(), event, ...data }); + } + + _noteSuspectReject(hoster, accountId, fileSize) { + if (!accountId || !Number.isFinite(fileSize) || fileSize <= 0) return; + const key = hoster + ':' + accountId; + const prev = this._suspectSizeMemo.get(key); + if (prev === undefined) this._suspectSizeMemo.set(key, { size: fileSize, count: 1 }); + else this._suspectSizeMemo.set(key, { size: Math.min(prev.size, fileSize), count: prev.count + 1 }); + } + + _suspectMemoBlocks(hoster, accountId, fileSize) { + if (this.hosterSettings[hoster] && this.hosterSettings[hoster].sizeMemoEnabled === false) return false; + const memo = this._suspectSizeMemo.get(hoster + ':' + accountId); + return !!memo && memo.count >= 2 && fileSize > memo.size; + } + + // File-specific rejections from the hoster: the same file will get rejected + // on any account, so rotation is pointless. Matches the `err.fileRejected` + // flag set by parsers plus known rejection phrases. + // NOTE: We deliberately do NOT match the generic "lehnte Datei ab" prefix + // here — that phrase is used by the Byse parser for both file- AND + // account-level errors. Account-level ones set err.accountError instead, + // which takes priority in _shouldSkipRetryOnAccountError. + _isFileRejectedError(err) { + if (!err) return false; + if (err.transientNetwork === true) return false; + if (err.accountError === true) return false; // explicit account-level wins + if (err.fileRejected === true) return true; + if (!err.message) return false; + const m = String(err.message); + return /(Not video file format|Duplicate|Datei zu (klein|gross|groß)|File too (small|large)|Invalid file|Unsupported format)/i.test(m); + } + + // Hoster-side transient flake — the hoster's backend accepted the upload but + // returned a malformed/empty result (e.g. doodstream CDN form with no fn/no + // st). Same account + same file works on a later attempt; this is NOT an + // account problem. Treated exactly like a transient network error: skip + // remaining in-batch retries (the flake won't clear in 3s and a re-upload of + // 95 MB is expensive), don't blacklist the account, fail this file cleanly. + // The user's next manual retry — or a later batch — can use the same account. + _isHosterTransientError(err) { + if (!err) return false; + if (err.hosterTransient === true) return true; // explicit flag — primary + if (!err.message) return false; + // Defensive fallback: catch the same class of error if it bubbles up + // wrapped (e.g. through a different code path) without the flag set. + return /Server gab leeren Link zurueck|kein Filecode/i.test(String(err.message)); + } + + // Transient network errors — the account is fine, the network or the + // hoster's own backend hiccuped. Retrying on the SAME account is the right + // move; marking it failed would wrongly poison the fallback chain. If all + // retries on the current account still hit this class of error, we bail + // out for this file without blacklisting the account, so other jobs in the + // batch still get a fresh chance on it. + _isTransientNetworkError(err) { + if (!err) return false; + if (err.transientNetwork === true) return true; + if (!err.message) return false; + const m = String(err.message); + const TRANSIENT = [ + /ENOTFOUND/i, + /ECONNRESET/i, + /ECONNREFUSED/i, + /ETIMEDOUT/i, + /EAI_AGAIN/i, + /EHOSTUNREACH/i, + /ENETUNREACH/i, + /EPIPE/i, + /socket hang up/i, + /network (error|failure|problem)/i, + /dns (lookup|error|failed)/i, + /getaddrinfo/i, + /fetch failed/i, + /\bconnect (ETIMEDOUT|ECONN)/i, + /HTTP 5\d\d\b/i, + /Bad Gateway/i, + /Service Unavailable/i, + /Gateway Time-?out/i + ]; + return TRANSIENT.some(p => p.test(m)); + } + + // Error classes that mean "this account is the problem, retrying on it won't + // help" — we skip the remaining retries and go straight to the fallback + // account. Keeps single runs fast when an account is rate-limited, banned, + // or out of quota. + _shouldSkipRetryOnAccountError(err) { + if (!err) return false; + if (err.transientNetwork === true) return false; + // Explicit account-level flag from hoster parsers — highest priority. + if (err.accountError === true) return true; + if (!err.message) return false; + const m = String(err.message); + const PATTERNS = [ + /Kein Upload-Server/i, + /No upload server/i, + /kein server/i, + /quota/i, + /limit (reached|exceeded|überschritten)/i, + /rate[- ]?limit/i, + /too many requests/i, + /\b(401|403|429)\b/, + /Falscher (User|Username|Passwort)/i, + /Incorrect (Login|Password)/i, + /invalid (credentials|api[- ]?key|token|session)/i, + /(account|user) (banned|suspended|disabled|gesperrt)/i, + /not authorized/i, + /forbidden/i, + /session (expired|abgelaufen)/i, + // Session/CSRF hints — the account's server session went stale, which + // no amount of retrying will fix. Re-login happens on the next account. + /CSRF[- ]?Token nicht gefunden/i, + /CSRF[- ]?token not found/i, + /Bist du eingeloggt/i, + /not logged in/i, + // Storage exhaustion — account is full. Rotate instead of hammering it. + /not enough (disk )?(space|storage)/i, + /insufficient (disk )?space/i, + /disk (space )?full/i, + /storage (exhausted|full|voll|limit)/i, + /account (full|voll)/i + ]; + return PATTERNS.some(p => p.test(m)); + } + + updateSettings(hosterSettings, globalSettings) { + this.hosterSettings = hosterSettings || this.hosterSettings; + this.globalSettings = globalSettings || this.globalSettings; + // Live-update semaphores for running uploads + for (const [hoster, sem] of Object.entries(this.semaphores)) { + const settings = this._getSettings(hoster); + sem.updateLimit(settings.parallelCount); + } + // Update global throttle if speed limit changed + const newKbs = (this.globalSettings.globalMaxSpeedKbs || 0); + if (newKbs > 0) { + if (this.globalThrottle) { + this.globalThrottle.updateRate(newKbs * 1024); + } else { + this.globalThrottle = new Throttle(newKbs * 1024); + } + } else { + this.globalThrottle = null; + } + // Update global semaphore live + const globalLimit = this._getGlobalParallelLimit(); + if (globalLimit > 0 && this.globalSemaphore) { + this.globalSemaphore.updateLimit(globalLimit); + } + } + + _getSettings(hoster) { + const settings = { ...DEFAULT_SETTINGS, ...(this.hosterSettings[hoster] || {}) }; + const globalLimit = this._getGlobalParallelLimit(); + if (this.globalSettings.scaleParallelUploads && globalLimit > 0) { + settings.parallelCount = Math.min(settings.parallelCount || 1, globalLimit); + } + return settings; + } + + _getGlobalParallelLimit() { + const raw = Number(this.globalSettings.parallelUploadCount || 0); + if (!Number.isFinite(raw) || raw <= 0) return 0; + return Math.max(1, Math.min(100, Math.round(raw))); + } + + _getGlobalSemaphore() { + const limit = this._getGlobalParallelLimit(); + if (limit <= 0) return null; + if (!this.globalSemaphore) { + this.globalSemaphore = new Semaphore(limit); + } else { + this.globalSemaphore.updateLimit(limit); + } + return this.globalSemaphore; + } + + _getGlobalThrottle() { + const kbs = Number(this.globalSettings.globalMaxSpeedKbs || 0); + if (!Number.isFinite(kbs) || kbs <= 0) return null; + if (!this.globalThrottle) { + this.globalThrottle = new Throttle(kbs * 1024); + } else { + this.globalThrottle.updateRate(kbs * 1024); + } + return this.globalThrottle; + } + + _getSemaphore(hoster) { + if (!this.semaphores[hoster]) { + const settings = this._getSettings(hoster); + this.semaphores[hoster] = new Semaphore(settings.parallelCount); + } else { + this.semaphores[hoster].updateLimit(this._getSettings(hoster).parallelCount); + } + return this.semaphores[hoster]; + } + + async startBatch(tasks, opts = {}) { + this.running = true; + this.stopAfterActive = false; + this.abortController = new AbortController(); + this.startTime = Date.now(); + this.sessionBytes = 0; + this.activeJobs.clear(); + this.jobAbortControllers.clear(); + this.cancelledJobIds.clear(); + this._doodApiKeyCache.clear(); // re-derive doodstream keys fresh each batch + this._baselineCache.clear(); // re-fetch baselines per batch (a long batch could outlast remote-side relevance) + this.semaphores = {}; + this.globalSemaphore = null; + this.globalThrottle = null; + this.lastStartTime = {}; + // Reset account-rotation state each batch — but optionally re-prime from + // app-session memory so a "Retry failed" right after batch-done doesn't + // burn 5 retries on the account we already know is dead. Caller (main.js) + // passes the session-scoped failed/override state. + this._failedAccounts.clear(); + this._accountOverrides.clear(); + this._suspectSizeMemo.clear(); + this._suspectGoodAccounts.clear(); + if (Array.isArray(opts.primeFailedAccounts)) { + for (const key of opts.primeFailedAccounts) this._failedAccounts.set(key, true); + } + if (Array.isArray(opts.primeOverrides)) { + for (const entry of opts.primeOverrides) { + if (Array.isArray(entry) && entry.length === 2) this._accountOverrides.set(entry[0], entry[1]); + } + } + this._rotLog('batch-start', { + taskCount: tasks.length, + primedFailed: this._failedAccounts.size, + primedOverrides: this._accountOverrides.size + }); + + const { signal } = this.abortController; + const batchId = `batch-${Date.now()}`; + const results = new Map(); // filePath -> { name, size, results: [] } + this._batchResults = results; + this._additionalPromises = []; // Track jobs added mid-batch via addJobs() + + const DEDUP_CHUNK = 200; + for (let i = 0; i < tasks.length; i += DEDUP_CHUNK) { + if (signal.aborted) break; + const end = Math.min(i + DEDUP_CHUNK, tasks.length); + const toStat = []; + for (let j = i; j < end; j++) { + const task = tasks[j]; + if (!results.has(task.file)) { + results.set(task.file, { name: path.basename(task.file), size: 0, results: [] }); + toStat.push(task.file); + } + } + await Promise.all(toStat.map(async (f) => { + try { const st = await fs.promises.stat(f); const e = results.get(f); if (e) e.size = st.size; } catch {} + })); + } + + this._startStatsTimer(); + + const SPAWN_CHUNK = 100; + const promises = []; + for (let i = 0; i < tasks.length; i += SPAWN_CHUNK) { + if (signal.aborted) break; + const end = Math.min(i + SPAWN_CHUNK, tasks.length); + for (let j = i; j < end; j++) promises.push(this._runJob(tasks[j], results, signal)); + if (end < tasks.length) await new Promise(setImmediate); + } + await Promise.allSettled(promises); + // Wait for any jobs added mid-batch via addJobs() + while (this._additionalPromises.length > 0) { + const batch = this._additionalPromises.splice(0); + await Promise.allSettled(batch); + } + + this._stopStatsTimer(); + this.running = false; + + const files = Array.from(results.values()); + const total = tasks.length; + const succeeded = files.reduce((count, file) => count + file.results.filter((result) => result.status === 'done').length, 0); + + const summary = { + id: batchId, + timestamp: new Date().toISOString(), + total, + succeeded, + failed: total - succeeded, + files + }; + + this.emit('batch-done', summary); + } + + async _runJob(task, results, batchSignal) { + const settings = this._getSettings(task.hoster); + const hosterSemaphore = this._getSemaphore(task.hoster); + const globalSemaphore = this._getGlobalSemaphore(); + const uploadId = crypto.randomBytes(8).toString('hex'); + const jobId = task.jobId || uploadId; + const fileName = path.basename(task.file); + let fileSize = 0; + let fileNotFound = false; + const cachedResult = results && results.get(task.file); + if (cachedResult && typeof cachedResult.size === 'number' && cachedResult.size > 0) { + fileSize = cachedResult.size; + } else { + try { fileSize = (await fs.promises.stat(task.file)).size; } catch { fileNotFound = true; } + } + + const maxAttempts = Math.max(1, (settings.retries || 0) + 1); + const jobAbortController = new AbortController(); + const { signal, cleanup: cleanupSignals } = this._combineSignals(batchSignal, jobAbortController.signal); + this.jobAbortControllers.set(jobId, jobAbortController); + + let hosterSlotAcquired = false; + let globalSlotAcquired = false; + let finalResultRecorded = false; + let lastError = null; + + const recordFinalResult = (status, payload = {}) => { + if (finalResultRecorded) return; + finalResultRecorded = true; + + const result = { + hoster: task.hoster, + status, + error: payload.error || null, + download_url: payload.result ? payload.result.download_url || null : null, + embed_url: payload.result ? payload.result.embed_url || null : null, + file_code: payload.result ? payload.result.file_code || null : null + }; + + results.get(task.file).results.push(result); + }; + + const emitFinalStatus = (status, payload = {}) => { + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, + status, + progress: status === 'done' ? 1 : 0, + bytesUploaded: status === 'done' ? fileSize : 0, + bytesTotal: fileSize, + speedKbs: payload.speedKbs || 0, + elapsed: payload.elapsed || 0, + remaining: 0, + error: payload.error || null, + result: payload.result || null, + attempt: payload.attempt || maxAttempts, + maxAttempts + }); + }; + + try { + if (fileNotFound) { + const error = 'Datei nicht gefunden'; + emitFinalStatus('skipped', { error, attempt: 0 }); + recordFinalResult('error', { error }); + return; + } + if (fileSize <= 0) { + const error = 'Datei ist leer (0 Bytes)'; + emitFinalStatus('skipped', { error, attempt: 0 }); + recordFinalResult('error', { error }); + return; + } + if (settings.maxSizeMb > 0 && fileSize > settings.maxSizeMb * 1024 * 1024) { + const error = `Datei zu groß (Max: ${settings.maxSizeMb} MB)`; + emitFinalStatus('skipped', { error, attempt: 0 }); + recordFinalResult('error', { error }); + return; + } + + // The initial 'queued' emit per job is suppressed: with N=2000+ tasks + // it produces 2000+ main→renderer IPCs back-to-back at startBatch and + // freezes the renderer event loop for tens of seconds. The renderer + // already holds each job in 'queued'/'preview' state from its own + // queueJobs array; the first event it actually needs from main is the + // 'getting-server' / 'uploading' transition for the jobs that the + // semaphore lets through. + await hosterSemaphore.acquire(signal); + hosterSlotAcquired = true; + + let fileProbe = null; + try { + fileProbe = await probeFileHead(task.file, 512); + } catch (err) { + fileProbe = { ok: false, error: err && err.message, kind: 'unreadable' }; + } + this._rotLog('upload-start', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + fileSize, + detectedKind: fileProbe && fileProbe.kind ? fileProbe.kind : 'unknown', + isVideoLike: !!(fileProbe && fileProbe.isVideoLike), + headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null + }); + + if (globalSemaphore) { + await globalSemaphore.acquire(signal); + globalSlotAcquired = true; + } + + if (settings.timeIntervalSec > 0) { + await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, signal); + } + + // Pre-job-swap: if this account was marked failed WHILE this task was + // waiting in the semaphore queue, jump straight to the override instead + // of burning a guaranteed-to-fail upload attempt. Critical at scale: + // with 500 queued jobs and 1 parallel slot, without this check every + // job still hits the original dead account first. + if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) { + const override = this._accountOverrides.get(task.hoster); + if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) { + this._rotLog('pre-job-swap', { + jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id + }); + task.accountId = override.id; + task.username = override.username; + task.password = override.password; + task.apiKey = override.apiKey; + } else { + this._rotLog('pre-job-swap-blocked', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + hasOverride: !!override, + overrideAlsoFailed: override ? this._failedAccounts.has(task.hoster + ':' + override.id) : false + }); + } + } + + // A previous file of at least this size already got a suspect rejection + // on this exact account — skip the guaranteed-to-fail multi-GB upload + // and go straight to the alternate-account walk below. + let memoSuspect = null; + if (fileProbe && fileProbe.isVideoLike === true && task.accountId + && this._suspectMemoBlocks(task.hoster, task.accountId, fileSize)) { + memoSuspect = new Error('Bekanntes Größen-Limit auf diesem Account (frühere verdächtige Ablehnung)'); + memoSuspect.fileRejected = true; + memoSuspect.suspectReject = true; + lastError = memoSuspect; + this._rotLog('suspect-memo-skip', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, fileSize + }); + } + + for (let attempt = 1; attempt <= maxAttempts && !memoSuspect; attempt++) { + if (signal.aborted || this.stopAfterActive) break; + + if (attempt > 1) { + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, + status: 'retrying', + progress: 0, + bytesUploaded: 0, + bytesTotal: fileSize, + speedKbs: 0, + elapsed: 0, + remaining: 0, + error: lastError ? lastError.message : null, + result: null, + attempt, + maxAttempts + }); + await this._sleep(3000, signal); + } + + const jobStart = Date.now(); + let lastBytes = 0; + let lastSpeedTime = jobStart; + let currentSpeedKbs = 0; + let lowSpeedSince = 0; + let speedAbort = null; + let speedMonitor = null; + let uploadSignalBundle = { signal, cleanup() {} }; + + try { + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, + status: 'getting-server', + progress: 0, + bytesUploaded: 0, + bytesTotal: fileSize, + speedKbs: 0, + elapsed: 0, + remaining: 0, + error: null, + result: null, + attempt, + maxAttempts + }); + + const hosterThrottle = settings.maxSpeedKbs > 0 + ? new Throttle(settings.maxSpeedKbs * 1024) + : null; + const globalThrottle = this._getGlobalThrottle(); + const throttle = hosterThrottle && globalThrottle + ? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } } + : hosterThrottle || globalThrottle; + + if (settings.restartBelowKbs > 0) { + speedAbort = new AbortController(); + uploadSignalBundle = this._combineManySignals([signal, speedAbort.signal]); + speedMonitor = setInterval(() => { + try { + if (currentSpeedKbs > 0 && currentSpeedKbs < settings.restartBelowKbs) { + if (!lowSpeedSince) lowSpeedSince = Date.now(); + if (Date.now() - lowSpeedSince > 6000) { + speedAbort.abort(); + } + } else { + lowSpeedSince = 0; + } + } catch (e) { this._rotLog('speed-monitor-error', { jobId, error: e && e.message }); } + }, 2000); + } + + // Mutate this single object on each progress callback instead of + // allocating a fresh one — callback fires on every stream chunk + // (hundreds/sec per active job). + const activeEntry = { jobId, speedKbs: 0, bytesUploaded: 0, hoster: task.hoster }; + this.activeJobs.set(uploadId, activeEntry); + + let lastEmitTime = 0; + const PROGRESS_EMIT_INTERVAL = 250; // ms – throttle UI updates + + const progressCb = (bytesUploaded, bytesTotal) => { + try { + const now = Date.now(); + const elapsed = Math.round((now - jobStart) / 1000); + const timeDelta = (now - lastSpeedTime) / 1000; + if (Number.isFinite(timeDelta) && timeDelta >= 1) { + const bytesDelta = bytesUploaded - lastBytes; + currentSpeedKbs = Math.round(bytesDelta / timeDelta / 1024); + lastBytes = bytesUploaded; + lastSpeedTime = now; + } + + activeEntry.speedKbs = currentSpeedKbs; + activeEntry.bytesUploaded = bytesUploaded; + + if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return; + lastEmitTime = now; + + const remaining = currentSpeedKbs > 0 + ? Math.round((bytesTotal - bytesUploaded) / (currentSpeedKbs * 1024)) + : 0; + + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, + status: 'uploading', + progress: bytesTotal > 0 ? Math.min(1, bytesUploaded / bytesTotal) : 0, + bytesUploaded, + bytesTotal, + speedKbs: currentSpeedKbs, + elapsed, + remaining, + error: null, + result: null, + attempt, + maxAttempts + }); + } catch { /* progress callbacks must never throw — swallowing is correct, the stream keeps going */ } + }; + + const result = await this._executeUpload(task, progressCb, uploadSignalBundle.signal, throttle, fileProbe); + + const elapsed = Math.round((Date.now() - jobStart) / 1000); + this.sessionBytes += fileSize; + this.activeJobs.delete(uploadId); + + emitFinalStatus('done', { + result, + speedKbs: currentSpeedKbs, + elapsed, + attempt + }); + recordFinalResult('done', { result }); + return; + } catch (err) { + this.activeJobs.delete(uploadId); + if (this._isTransientNetworkError(err)) this._transientErrorTotal++; + + const isSpeedRestart = speedAbort && speedAbort.signal.aborted && !signal.aborted; + if (!signal.aborted && !isSpeedRestart) { + const diag = (err && typeof err === 'object' && err.diagnostic) || {}; + this._rotLog('upload-failure', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + attempt, + error: err && err.message ? err.message : String(err), + fileRejected: !!(err && err.fileRejected), + accountError: !!(err && err.accountError), + hosterTransient: !!(err && err.hosterTransient), + http: diag.http || null, + contentType: diag.contentType || null, + detectedKind: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.kind) ? fileProbe.kind : null, + isVideoLike: !!(typeof fileProbe !== 'undefined' && fileProbe && fileProbe.isVideoLike), + headHex: (typeof fileProbe !== 'undefined' && fileProbe && fileProbe.headHex) ? fileProbe.headHex.slice(0, 32) : null, + payloadSnippet: diag.payloadSnippet || null + }); + } + if (signal.aborted) { + lastError = new Error('Abgebrochen'); + break; + } + + if (this.stopAfterActive) { + lastError = new Error('Angehalten'); + break; + } + + if (isSpeedRestart && attempt < maxAttempts) { + lastError = new Error('Geschwindigkeit zu niedrig - Neustart'); + await this._sleep(3000, signal); + continue; + } + + lastError = err; + // File-specific rejection — re-uploading won't change the server's + // mind. Break out immediately; the outer file-rejected branch then + // records the final error without burning through 5 × 3s retries. + if (this._isFileRejectedError(err)) break; + // Hoster-side transient flake (e.g. doodstream empty CDN form). Server + // flake won't clear in 3s and re-uploading the whole file 4× is pure + // bandwidth waste; bail out of the retry loop so the post-loop branch + // can fail this file without blacklisting the account. + if (this._isHosterTransientError(err)) { + this._rotLog('hoster-transient', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + attempt, error: err && err.message ? err.message : String(err) + }); + break; + } + // Account-specific errors — don't waste retries on the same account, + // jump straight to rotation. + if (this._shouldSkipRetryOnAccountError(err)) { + this._rotLog('fast-fail', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + attempt, error: err && err.message ? err.message : String(err) + }); + break; + } + // Generic non-transient error AND a fallback is already resolved for + // this hoster: bail to rotation instead of burning more retries on a + // possibly-dead primary. The fallback (pre-resolved at batch-start) + // deserves a real shot. Transient network errors stay on the same + // account — the network is the issue, not the account. + if (!this._isTransientNetworkError(err) && + this._hasPendingOverride(task.hoster, task.accountId)) { + this._rotLog('try-alternate-after-fail', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + attempt, error: err && err.message ? err.message : String(err) + }); + break; + } + if (attempt >= maxAttempts) break; + // Wait 3 seconds before retry + await this._sleep(3000, signal); + } finally { + if (speedMonitor) clearInterval(speedMonitor); + uploadSignalBundle.cleanup(); + } + } + + const wasStopped = this.stopAfterActive && !signal.aborted; + const wasAborted = signal.aborted || this.cancelledJobIds.has(jobId); + if (wasStopped || wasAborted) { + const error = wasStopped ? 'Warteschlange angehalten' : 'Abgebrochen'; + emitFinalStatus('aborted', { error }); + recordFinalResult('aborted', { error }); + return; + } + + // Account rotation: mark the current account failed (if not already), + // wait for main to resolve the next fallback, then retry. Loops so + // A → B → C → ... works for hosters with 3+ accounts. + // + // CRITICAL: we must ALWAYS check for an existing override, even if this + // account is already in _failedAccounts (e.g. another concurrent job + // already marked it failed). Otherwise the second job falls straight + // through to final-error instead of using the already-resolved fallback. + this._rotLog('retries-exhausted', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + // File-specific rejection → same file will get the same verdict on + // every other account, rotation is pointless. Don't blacklist, don't + // retry siblings, just fail this file cleanly. + // + // EXCEPT suspect rejections (err.suspectReject, e.g. byse "Not video + // file format" on a probe-verified video): those verdicts are + // account-conditional in practice (per-account size tiers), so the file + // gets one attempt on each remaining account — WITHOUT blacklisting the + // current one, which keeps working for files the hoster does accept. + if (this._isFileRejectedError(lastError)) { + if (lastError.suspectReject === true && fileProbe && fileProbe.isVideoLike === true) { + this._noteSuspectReject(task.hoster, task.accountId, fileSize); + const alt = await this._trySuspectRejectAlternates(task, { uploadId, jobId, fileName, fileSize, settings, signal, fileProbe }); + if (alt) { + emitFinalStatus('done', { result: alt.result, speedKbs: alt.speedKbs, elapsed: alt.elapsed, attempt: 1 }); + recordFinalResult('done', { result: alt.result }); + return; + } + const stoppedInAlternates = this.stopAfterActive && !signal.aborted; + const abortedInAlternates = signal.aborted || this.cancelledJobIds.has(jobId); + if (stoppedInAlternates || abortedInAlternates) { + const error = stoppedInAlternates ? 'Warteschlange angehalten' : 'Abgebrochen'; + emitFinalStatus('aborted', { error }); + recordFinalResult('aborted', { error }); + return; + } + } + this._rotLog('skip-rotation-file-rejected', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + const error = lastError.message || 'Datei abgelehnt'; + emitFinalStatus('error', { error }); + recordFinalResult('error', { error }); + return; + } + // Hoster-side transient flake → identical handling to network-transient: + // the account is fine, don't blacklist it, just fail this file. Critical + // to keep the account usable across batches — otherwise one empty-form + // response poisons every subsequent batch with `pre-job-swap-blocked`. + if (this._isHosterTransientError(lastError)) { + this._rotLog('skip-rotation-hoster-transient', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + const error = lastError.message || 'Hoster-Backend lieferte leeres Ergebnis'; + emitFinalStatus('error', { error }); + recordFinalResult('error', { error }); + return; + } + // If the reason for failure was a transient network error we do NOT + // blacklist the account. Other jobs on the same account in this batch + // can still try fresh. This file just errors out for now. + if (this._isTransientNetworkError(lastError)) { + this._rotLog('skip-rotation-transient', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + const error = lastError.message || 'Netzwerkfehler'; + emitFinalStatus('error', { error }); + recordFinalResult('error', { error }); + return; + } + while (task.accountId) { + if (signal.aborted || this.stopAfterActive) break; + // The rotated-to account failed with a file-class error (file + // rejection / hoster flake / network) — blacklisting it for that + // would poison a working account for the whole batch. Fail only + // this file instead. + if (this._isFileRejectedError(lastError) || this._isHosterTransientError(lastError) || this._isTransientNetworkError(lastError)) { + this._rotLog('skip-rotation-after-rotate', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + break; + } + const alreadyMarked = this._failedAccounts.has(task.hoster + ':' + task.accountId); + if (!alreadyMarked) { + this._failedAccounts.set(task.hoster + ':' + task.accountId, true); + this._rotLog('mark-failed', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId, + lastError: lastError ? lastError.message : null + }); + this.emit('account-failed', { hoster: task.hoster, accountId: task.accountId }); + await this._sleep(800, signal); + // Re-check after the await: the user could have cancelled while + // we were waiting for main.js to resolve the fallback. Without + // this, rotation proceeds another full attempt-loop's worth of + // work before the next signal-check inside _executeUpload notices. + if (signal.aborted || this.stopAfterActive) break; + } else { + this._rotLog('already-marked', { + jobId, hoster: task.hoster, fileName, accountId: task.accountId + }); + } + const override = this._accountOverrides.get(task.hoster); + if (!override) { + this._rotLog('rotation-end', { + jobId, hoster: task.hoster, fileName, reason: 'no-override-set', + lastFailedAccountId: task.accountId + }); + break; + } + if (this._failedAccounts.has(task.hoster + ':' + override.id)) { + this._rotLog('rotation-end', { + jobId, hoster: task.hoster, fileName, reason: 'override-already-failed', + overrideId: override.id, lastFailedAccountId: task.accountId + }); + break; + } + if (override.id === task.accountId) { + this._rotLog('rotation-end', { + jobId, hoster: task.hoster, fileName, reason: 'override-same-as-current', + lastFailedAccountId: task.accountId + }); + break; + } + // Switch to fallback account and retry this file + this._rotLog('rotate', { + jobId, hoster: task.hoster, fileName, + fromAccountId: task.accountId, toAccountId: override.id + }); + task.accountId = override.id; + task.username = override.username; + task.password = override.password; + task.apiKey = override.apiKey; + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, status: 'retrying', progress: 0, bytesUploaded: 0, bytesTotal: fileSize, + speedKbs: 0, elapsed: 0, remaining: 0, + error: 'Account-Wechsel zu Fallback', result: null, attempt: 1, maxAttempts + }); + // Retry loop with the new account. On exhausted failure, the while + // loop iterates: marks this account failed too, asks main for the next + // fallback, and so on. + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (signal.aborted || this.stopAfterActive) break; + if (attempt > 1) { + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, status: 'retrying', progress: 0, bytesUploaded: 0, bytesTotal: fileSize, + speedKbs: 0, elapsed: 0, remaining: 0, + error: lastError ? lastError.message : '', result: null, attempt, maxAttempts + }); + await this._sleep(3000, signal); + } + try { + const jobStart = Date.now(); + let lastBytes = 0; + let lastSpeedTime = jobStart; + let currentSpeedKbs = 0; + const activeEntry = { jobId, speedKbs: 0, bytesUploaded: 0, hoster: task.hoster }; + this.activeJobs.set(uploadId, activeEntry); + + let lastEmitTime = 0; + const PROGRESS_EMIT_INTERVAL = 250; + const progressCb = (bytesUploaded, bytesTotal) => { + const now = Date.now(); + const timeDelta = (now - lastSpeedTime) / 1000; + if (timeDelta >= 1) { + currentSpeedKbs = Math.round((bytesUploaded - lastBytes) / timeDelta / 1024); + lastBytes = bytesUploaded; + lastSpeedTime = now; + } + activeEntry.speedKbs = currentSpeedKbs; + activeEntry.bytesUploaded = bytesUploaded; + if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return; + lastEmitTime = now; + const elapsed = Math.round((now - jobStart) / 1000); + const remaining = currentSpeedKbs > 0 ? Math.round((bytesTotal - bytesUploaded) / (currentSpeedKbs * 1024)) : 0; + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, status: 'uploading', + progress: bytesTotal > 0 ? Math.min(1, bytesUploaded / bytesTotal) : 0, + bytesUploaded, bytesTotal, speedKbs: currentSpeedKbs, + elapsed, remaining, error: null, result: null, attempt, maxAttempts + }); + }; + + const hosterThrottle = settings.maxSpeedKbs > 0 ? new Throttle(settings.maxSpeedKbs * 1024) : null; + const globalThrottle = this._getGlobalThrottle(); + const throttle = hosterThrottle && globalThrottle + ? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } } + : hosterThrottle || globalThrottle; + + const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe); + this.activeJobs.delete(uploadId); + this.sessionBytes += fileSize; + emitFinalStatus('done', { result, speedKbs: currentSpeedKbs, elapsed: Math.round((Date.now() - jobStart) / 1000), attempt }); + recordFinalResult('done', { result }); + return; + } catch (err) { + this.activeJobs.delete(uploadId); + lastError = err; + if (!signal.aborted) { + this._rotLog('upload-failure', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + attempt, + error: err && err.message ? err.message : String(err), + fileRejected: !!(err && err.fileRejected), + accountError: !!(err && err.accountError), + hosterTransient: !!(err && err.hosterTransient), + rotationRetry: true + }); + } + if (signal.aborted || this.stopAfterActive) break; + if (this._isFileRejectedError(err)) break; + if (this._isHosterTransientError(err)) break; + if (this._isTransientNetworkError(err)) break; + if (attempt >= maxAttempts) break; + } + } + } + + const stoppedLate = this.stopAfterActive && !signal.aborted; + const abortedLate = signal.aborted || this.cancelledJobIds.has(jobId); + if (stoppedLate || abortedLate) { + const error = stoppedLate ? 'Warteschlange angehalten' : 'Abgebrochen'; + emitFinalStatus('aborted', { error }); + recordFinalResult('aborted', { error }); + return; + } + const error = lastError && lastError.message ? lastError.message : 'Unbekannter Fehler'; + this._rotLog('final-error', { + jobId, hoster: task.hoster, fileName, lastFailedAccountId: task.accountId, error + }); + emitFinalStatus('error', { error }); + recordFinalResult('error', { error }); + } catch (err) { + const wasStopped = this.stopAfterActive && !signal.aborted; + const error = wasStopped + ? 'Warteschlange angehalten' + : (signal.aborted || this.cancelledJobIds.has(jobId) ? 'Abgebrochen' : (err && err.message ? err.message : 'Unbekannter Fehler')); + const status = signal.aborted || this.cancelledJobIds.has(jobId) || wasStopped ? 'aborted' : 'error'; + emitFinalStatus(status, { error }); + recordFinalResult(status === 'error' ? 'error' : 'aborted', { error }); + } finally { + this.activeJobs.delete(uploadId); + this.jobAbortControllers.delete(jobId); + cleanupSignals(); + // Release in reverse order of acquire (global first, then hoster) + if (globalSlotAcquired && globalSemaphore) globalSemaphore.release(); + if (hosterSlotAcquired) hosterSemaphore.release(); + } + } + + async _trySuspectRejectAlternates(task, ctx) { + const { uploadId, jobId, fileName, fileSize, settings, signal, fileProbe } = ctx; + const pool = this.accountPools && Array.isArray(this.accountPools[task.hoster]) + ? this.accountPools[task.hoster] + : []; + const original = { accountId: task.accountId, username: task.username, password: task.password, apiKey: task.apiKey }; + const goodId = this._suspectGoodAccounts.get(task.hoster); + const ordered = []; + for (const account of pool) { + if (account && account.id === goodId) ordered.unshift(account); + else ordered.push(account); + } + const tried = new Set([task.accountId]); + let attempted = 0; + for (const account of ordered) { + if (signal.aborted || this.stopAfterActive) break; + if (!account || !account.id || tried.has(account.id)) continue; + if (this._failedAccounts.has(task.hoster + ':' + account.id)) continue; + tried.add(account.id); + if (this._suspectMemoBlocks(task.hoster, account.id, fileSize)) { + this._rotLog('suspect-memo-skip-alt', { + jobId, hoster: task.hoster, fileName, accountId: account.id, fileSize + }); + continue; + } + attempted += 1; + this._rotLog('suspect-reject-alt', { + jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: account.id + }); + task.accountId = account.id; + task.username = account.username; + task.password = account.password; + task.apiKey = account.apiKey; + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, status: 'retrying', progress: 0, bytesUploaded: 0, bytesTotal: fileSize, + speedKbs: 0, elapsed: 0, remaining: 0, + error: 'Ablehnung verdächtig - Versuch auf anderem Account', result: null, attempt: 1, maxAttempts: 1 + }); + const jobStart = Date.now(); + let lastBytes = 0; + let lastSpeedTime = jobStart; + let currentSpeedKbs = 0; + const activeEntry = { jobId, speedKbs: 0, bytesUploaded: 0, hoster: task.hoster }; + this.activeJobs.set(uploadId, activeEntry); + let lastEmitTime = 0; + const PROGRESS_EMIT_INTERVAL = 250; + const progressCb = (bytesUploaded, bytesTotal) => { + const now = Date.now(); + const timeDelta = (now - lastSpeedTime) / 1000; + if (timeDelta >= 1) { + currentSpeedKbs = Math.round((bytesUploaded - lastBytes) / timeDelta / 1024); + lastBytes = bytesUploaded; + lastSpeedTime = now; + } + activeEntry.speedKbs = currentSpeedKbs; + activeEntry.bytesUploaded = bytesUploaded; + if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return; + lastEmitTime = now; + const elapsed = Math.round((now - jobStart) / 1000); + const remaining = currentSpeedKbs > 0 ? Math.round((bytesTotal - bytesUploaded) / (currentSpeedKbs * 1024)) : 0; + this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId, + jobId, status: 'uploading', + progress: bytesTotal > 0 ? Math.min(1, bytesUploaded / bytesTotal) : 0, + bytesUploaded, bytesTotal, speedKbs: currentSpeedKbs, + elapsed, remaining, error: null, result: null, attempt: 1, maxAttempts: 1 + }); + }; + const hosterThrottle = settings.maxSpeedKbs > 0 ? new Throttle(settings.maxSpeedKbs * 1024) : null; + const globalThrottle = this._getGlobalThrottle(); + const throttle = hosterThrottle && globalThrottle + ? { consume: async (bytes, sig) => { await hosterThrottle.consume(bytes, sig); await globalThrottle.consume(bytes, sig); } } + : hosterThrottle || globalThrottle; + try { + const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe); + this.activeJobs.delete(uploadId); + this.sessionBytes += fileSize; + this._suspectGoodAccounts.set(task.hoster, account.id); + return { result, speedKbs: currentSpeedKbs, elapsed: Math.round((Date.now() - jobStart) / 1000) }; + } catch (err) { + this.activeJobs.delete(uploadId); + if (!signal.aborted) { + this._rotLog('upload-failure', { + jobId, hoster: task.hoster, accountId: task.accountId, fileName, + attempt: 1, + error: err && err.message ? err.message : String(err), + fileRejected: !!(err && err.fileRejected), + accountError: !!(err && err.accountError), + hosterTransient: !!(err && err.hosterTransient), + suspectAlternate: true + }); + } + if (signal.aborted || this.stopAfterActive) break; + if (err && err.suspectReject === true) { + this._noteSuspectReject(task.hoster, account.id, fileSize); + } + // A genuine account-class error (quota, ban, full disk) positively + // identifies a dead account — remember it so parallel and later + // suspect jobs stop re-uploading multi-GB files to it. Deliberately + // no 'account-failed' emit: that would re-point the hoster-wide + // override and reroute normal-sized files away from a primary that + // still works for them. + if (err && err.accountError === true) { + this._failedAccounts.set(task.hoster + ':' + account.id, true); + this._rotLog('mark-failed', { + jobId, hoster: task.hoster, fileName, accountId: account.id, + lastError: err && err.message ? err.message : String(err), + suspectAlternate: true + }); + } + } + } + task.accountId = original.accountId; + task.username = original.username; + task.password = original.password; + task.apiKey = original.apiKey; + if (!signal.aborted && !this.stopAfterActive) { + this._rotLog('suspect-reject-exhausted', { + jobId, hoster: task.hoster, fileName, alternatesTried: attempted + }); + } + return null; + } + + async _executeUpload(task, progressCb, signal, throttle, fileProbe) { + if (task.hoster === 'vidmoly.me' && task.username) { + const vidmoly = new VidmolyUploader(); + await vidmoly.login(task.username, task.password); + return vidmoly.upload(task.file, progressCb, signal, throttle); + } else if (task.hoster === 'voe.sx' && task.username) { + const voe = new VoeUploader(); + await voe.login(task.username, task.password); + return voe.upload(task.file, progressCb, signal, throttle); + } else if (task.hoster === 'doodstream.com' && task.username) { + // Login-path reliability fix: the web-form upload returns the filecode in + // an HTML form that comes back empty for large files (doodstream backend + // registration timeout). Derive the account's API key from the logged-in + // session ONCE per batch and upload via the official API instead — it + // returns result[0].filecode directly and has no empty-form failure mode. + // Falls back to the web-form upload if no valid key can be derived. + const apiKey = await this._resolveDoodstreamApiKey(task); + if (apiKey) { + this._rotLog('doodstream-via-api', { accountId: task.accountId, fileName: path.basename(task.file) }); + return uploadFile('doodstream.com', task.file, apiKey, progressCb, signal, throttle, { + doodBaseline: await this._getBaseline('doodstream.com', apiKey, signal) + }); + } + this._rotLog('doodstream-via-web', { accountId: task.accountId, fileName: path.basename(task.file) }); + const dood = new DoodstreamUploader(); + await dood.login(task.username, task.password); + return dood.upload(task.file, progressCb, signal, throttle); + } else if (task.hoster === 'clouddrop.cc') { + const clouddrop = new ClouddropUploader(task.apiKey); + return clouddrop.upload(task.file, progressCb, signal, throttle); + } else { + const baselineOpts = {}; + if (task.hoster === 'byse.sx') { + baselineOpts.byseBaseline = await this._getBaseline('byse.sx', task.apiKey, signal); + if (fileProbe && fileProbe.ok !== false) baselineOpts.probeIsVideoLike = fileProbe.isVideoLike === true; + } + if (task.hoster === 'doodstream.com') baselineOpts.doodBaseline = await this._getBaseline('doodstream.com', task.apiKey, signal); + return uploadFile(task.hoster, task.file, task.apiKey, progressCb, signal, throttle, baselineOpts); + } + } + + _getBaseline(hosterName, apiKey, signal) { + if (!apiKey) return Promise.resolve(null); + const key = `${hosterName}:${apiKey}`; + let pending = this._baselineCache.get(key); + if (pending) return pending; + pending = prefetchBaseline(hosterName, apiKey, signal); + this._baselineCache.set(key, pending); + return pending; + } + + // Resolve (and cache per batch) the doodstream API key for a login-only + // account by logging in once and scraping+validating it from the session. + // Returns the key string, or '' when none could be derived (cached either way + // so a 40-file batch logs in + derives ONCE, not per file). The empty-string + // sentinel distinguishes "tried, none" from "not yet tried" (undefined). + async _resolveDoodstreamApiKey(task) { + const cacheKey = task.accountId || task.username; + const cached = this._doodApiKeyCache.get(cacheKey); + if (cached !== undefined) return cached || null; + + let key = ''; + try { + const probe = new DoodstreamUploader(); + await probe.login(task.username, task.password); + key = (await probe.deriveApiKey()) || ''; + } catch { + key = ''; + } + this._doodApiKeyCache.set(cacheKey, key); + return key || null; + } + + _emitProgress(uploadId, fileName, hoster, data) { + this.emit('progress', { uploadId, fileName, hoster, ...data }); + } + + _startStatsTimer() { + if (this.statsInterval) clearInterval(this.statsInterval); + this.statsInterval = setInterval(() => { + try { + let globalSpeedKbs = 0; + let activeCount = 0; + let inProgressBytes = 0; + for (const job of this.activeJobs.values()) { + globalSpeedKbs += job.speedKbs || 0; + inProgressBytes += job.bytesUploaded || 0; + activeCount++; + } + + const elapsed = Math.round((Date.now() - this.startTime) / 1000); + + this.emit('stats', { + state: this.running ? (this.stopAfterActive ? 'stopping' : 'uploading') : 'idle', + globalSpeedKbs, + totalBytes: this.sessionBytes + inProgressBytes, + elapsed, + activeJobs: activeCount, + pendingJobs: Object.values(this.semaphores).reduce((sum, semaphore) => sum + semaphore.pending, 0) + }); + } catch { /* never let a stats tick crash the timer + caller */ } + }, 1000); + } + + _stopStatsTimer() { + if (this.statsInterval) { + clearInterval(this.statsInterval); + this.statsInterval = null; + } + } + + _combineSignals(signal1, signal2) { + const controller = new AbortController(); + if (signal1.aborted || signal2.aborted) { + controller.abort(); + return { signal: controller.signal, cleanup() {} }; + } + + const onAbort = () => { + controller.abort(); + cleanup(); + }; + + const cleanup = () => { + signal1.removeEventListener('abort', onAbort); + signal2.removeEventListener('abort', onAbort); + }; + + signal1.addEventListener('abort', onAbort, { once: true }); + signal2.addEventListener('abort', onAbort, { once: true }); + return { signal: controller.signal, cleanup }; + } + + _combineManySignals(signals) { + const liveSignals = signals.filter(Boolean); + const controller = new AbortController(); + + if (liveSignals.some((signal) => signal.aborted)) { + controller.abort(); + return { signal: controller.signal, cleanup() {} }; + } + + const listeners = liveSignals.map((signal) => { + const handler = () => { + controller.abort(); + cleanup(); + }; + signal.addEventListener('abort', handler, { once: true }); + return { signal, handler }; + }); + + const cleanup = () => { + for (const entry of listeners) { + entry.signal.removeEventListener('abort', entry.handler); + } + }; + + return { signal: controller.signal, cleanup }; + } + + _sleep(ms, signal) { + return new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer); + reject(new Error('Aborted')); + }; + + const timer = setTimeout(() => { + if (signal) signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + + if (signal) { + if (signal.aborted) { + clearTimeout(timer); + reject(new Error('Aborted')); + return; + } + signal.addEventListener('abort', onAbort, { once: true }); + } + }); + } + + _waitForInterval(hoster, intervalMs, signal) { + // Serialize interval waits per hoster so concurrent jobs queue up properly + const prev = this.intervalLocks[hoster] || Promise.resolve(); + const next = prev.then(async () => { + const now = Date.now(); + const last = this.lastStartTime[hoster] || 0; + const elapsed = now - last; + if (elapsed < intervalMs) { + await this._sleep(intervalMs - elapsed, signal); + } + this.lastStartTime[hoster] = Date.now(); + }); + this.intervalLocks[hoster] = next.catch(() => {}); + return next; + } + + addJobs(tasks) { + if (!this.running || !tasks || tasks.length === 0) { + return { added: 0, alreadyInBatchJobIds: [] }; + } + const { signal } = this.abortController; + const results = this._batchResults || new Map(); + const addResult = { added: 0, alreadyInBatchJobIds: [] }; + for (const task of tasks) { + // Skip if this job is already being processed (prevent duplicates) + if (task.jobId && this.jobAbortControllers.has(task.jobId)) { + addResult.alreadyInBatchJobIds.push(task.jobId); + continue; + } + const fileName = path.basename(task.file); + if (!results.has(task.file)) { + let size = 0; + try { size = fs.statSync(task.file).size; } catch {} + results.set(task.file, { name: fileName, size, results: [] }); + } + this._additionalPromises.push(this._runJob(task, results, signal)); + addResult.added++; + } + return addResult; + } + + cancelJobs(jobIds) { + for (const jobId of jobIds || []) { + if (!jobId) continue; + this.cancelledJobIds.add(jobId); + const controller = this.jobAbortControllers.get(jobId); + if (controller && !controller.signal.aborted) { + controller.abort(); + } + } + } + + finishAfterActive() { + this.stopAfterActive = true; + } + + cancel() { + if (!this.running) return; + this.abortController.abort(); + this.stopAfterActive = false; + this.running = false; + for (const controller of this.jobAbortControllers.values()) { + if (!controller.signal.aborted) controller.abort(); + } + this._stopStatsTimer(); + } +} + +module.exports = UploadManager; diff --git a/lib/vidmoly-upload.js b/lib/vidmoly-upload.js new file mode 100644 index 0000000..cd9b145 --- /dev/null +++ b/lib/vidmoly-upload.js @@ -0,0 +1,518 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request } = require('undici'); + +const BASE_URL = 'https://vidmoly.me'; +const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const UPLOAD_TIMEOUT = 1800000; // 30 min +const RESULT_POLL_ATTEMPTS = 10; +const RESULT_POLL_DELAY_MS = 2000; + +/** + * XFileSharing-based upload for Vidmoly (login + form upload) + */ +class VidmolyUploader { + constructor() { + this.cookies = new Map(); + } + + _cookieHeader() { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + _parseCookiesFromHeaders(headers) { + // Handle both undici response headers and fetch Headers + let setCookies; + if (typeof headers.getSetCookie === 'function') { + setCookies = headers.getSetCookie(); + } else if (headers['set-cookie']) { + setCookies = Array.isArray(headers['set-cookie']) ? headers['set-cookie'] : [headers['set-cookie']]; + } else { + return; + } + for (const raw of setCookies) { + const pair = raw.split(';')[0]; + const eq = pair.indexOf('='); + if (eq > 0) { + this.cookies.set(pair.substring(0, eq).trim(), pair.substring(eq + 1).trim()); + } + } + } + + /** + * Simple GET/POST using built-in fetch (handles redirects) + */ + async _fetch(url, opts = {}, _redirectCount = 0) { + const MAX_REDIRECTS = 10; + const headers = { + 'User-Agent': USER_AGENT, + ...(opts.headers || {}) + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + const res = await fetch(url, { + ...opts, + headers, + redirect: 'manual' // handle manually to capture cookies from redirect responses + }); + + this._parseCookiesFromHeaders(res.headers); + + // Follow redirects manually (to capture cookies at each hop) + if ([301, 302, 303, 307, 308].includes(res.status)) { + // Drain body to prevent connection leak + try { await res.text(); } catch {} + if (_redirectCount >= MAX_REDIRECTS) { + throw new Error('Zu viele Redirects'); + } + const location = res.headers.get('location'); + if (location) { + const nextUrl = new URL(location, url).href; + return this._fetch(nextUrl, { ...opts, method: 'GET', body: undefined }, _redirectCount + 1); + } + } + + return res; + } + + /** + * Login to Vidmoly via the new JSON API (replaces the old XFS form POST + * at `/` with `op=login`, which the SPA redesign deprecated). The response + * sets a `vidmoly_session` HttpOnly cookie that the upload API checks. + */ + async login(username, password) { + // Warm up — get baseline cookies (cf_clearance etc.) + try { + const initRes = await this._fetch(BASE_URL); + await initRes.text(); + } catch {} + + const res = await this._fetch(`${BASE_URL}/api/auth/login`, { + method: 'POST', + body: JSON.stringify({ login: username, password }), + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Origin': BASE_URL, + 'Referer': `${BASE_URL}/login` + } + }); + + const body = await res.text(); + if (res.status === 401 || res.status === 403 || /incorrect|invalid|wrong/i.test(body)) { + throw new Error('Vidmoly Login fehlgeschlagen: Falscher Username oder Passwort'); + } + if (res.status < 200 || res.status >= 300) { + throw new Error(`Vidmoly Login fehlgeschlagen: HTTP ${res.status}`); + } + if (!this.cookies.has('vidmoly_session')) { + throw new Error('Vidmoly Login fehlgeschlagen: Keine Session erhalten (vidmoly_session fehlt)'); + } + + // Probe the upload API so downstream getUploadParams() has a warm path. + const probe = await this._fetch(`${BASE_URL}/api/upload/config`); + const probeBody = await probe.text(); + let probeJson = null; + try { probeJson = JSON.parse(probeBody); } catch {} + if (!probeJson || !probeJson.sess_id || !probeJson.upload_url) { + throw new Error('Vidmoly Login fehlgeschlagen: Session konnte nicht verifiziert werden (API-Probe)'); + } + } + + /** + * Fetch the upload session config from Vidmoly's new SPA API. + * Replaces the old HTML-form scrape at /?op=upload which the redesign + * removed. Returns an XFS-style session token + a transit-server URL. + */ + async getUploadParams() { + const res = await this._fetch(`${BASE_URL}/api/upload/config`); + const body = await res.text(); + let payload = null; + try { payload = JSON.parse(body); } catch { + throw new Error('Vidmoly: /api/upload/config lieferte kein JSON — evtl. nicht eingeloggt?'); + } + if (!payload || !payload.sess_id || !payload.upload_url) { + throw new Error('Vidmoly: /api/upload/config unvollständig (sess_id/upload_url fehlt)'); + } + return { + uploadUrl: payload.upload_url, + // Fields verified from a real browser POST capture. + // to_json=1 forces a JSON response instead of an HTML redirect page. + params: { sess_id: payload.sess_id, to_json: '1', fld_id: '0' }, + fileFieldName: 'file' + }; + } + + /** + * Upload a file to Vidmoly (uses undici.request for streaming progress) + */ + async upload(filePath, onProgress, signal, throttle) { + const fileName = path.basename(filePath); + const fileSize = fs.statSync(filePath).size; + const baselineCodes = await this._captureVmFileCodes(); + + const { uploadUrl, params, fileFieldName } = await this.getUploadParams(); + + const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex'); + + // XFS form fields + const formFields = {}; + for (const [k, v] of Object.entries(params)) { + if (!/^file(?:_\d+)?$/i.test(k)) { // eslint-disable-line security/detect-unsafe-regex -- safe: no backtracking + formFields[k] = v; + } + } + + // Build multipart + let preamble = ''; + for (const [key, value] of Object.entries(formFields)) { + preamble += `--${boundary}\r\n`; + preamble += `Content-Disposition: form-data; name="${key}"\r\n\r\n`; + preamble += `${value}\r\n`; + } + preamble += `--${boundary}\r\n`; + const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + preamble += `Content-Disposition: form-data; name="${fileFieldName || 'file'}"; filename="${safeFileName}"\r\n`; + preamble += `Content-Type: application/octet-stream\r\n\r\n`; + + const epilogue = `\r\n--${boundary}--\r\n`; + + const preambleBuf = Buffer.from(preamble, 'utf-8'); + const epilogueBuf = Buffer.from(epilogue, 'utf-8'); + const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; + + let bytesRead = 0; + const CHUNK_SIZE = 1024 * 1024; + + async function* generate() { + yield preambleBuf; + const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE }); + for await (const chunk of fileStream) { + if (signal && signal.aborted) throw new Error('Aborted'); + if (throttle) await throttle.consume(chunk.length, signal); + bytesRead += chunk.length; + yield chunk; + if (onProgress) onProgress(bytesRead, fileSize); + } + yield epilogueBuf; + } + + // Transit server lives on a different domain (*.vmwesa.online) and runs + // the nginx-upload-progress module. It requires an X-Progress-ID query + // parameter on the POST URL — without it the upload hangs at the final + // byte because the module can't finalize the session. Browsers append it + // automatically before submitting the form. + const progressId = Date.now().toString() + Math.floor(Math.random() * 1e6).toString().padStart(6, '0'); + const targetUrl = uploadUrl + (uploadUrl.includes('?') ? '&' : '?') + 'X-Progress-ID=' + progressId; + + // Browsers don't send vidmoly.me cookies across origins, so we don't either. + const { body, statusCode, headers } = await request(targetUrl, { + method: 'POST', + body: generate(), + signal, + headers: { + 'User-Agent': USER_AGENT, + 'Accept': '*/*', + 'Origin': BASE_URL, + 'Referer': `${BASE_URL}/`, + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize) + }, + headersTimeout: UPLOAD_TIMEOUT, + bodyTimeout: UPLOAD_TIMEOUT + }); + + this._parseCookiesFromHeaders(headers || {}); + + // Check if upload response is a redirect (XFS often redirects to result page) + let resultHtml; + if ([301, 302, 303].includes(statusCode)) { + const location = headers && headers.location; + // Always drain the original body to prevent connection leak + try { await body.text(); } catch {} + if (location) { + const resultRes = await this._fetch(new URL(location, uploadUrl).href); + resultHtml = await resultRes.text(); + } else { + resultHtml = ''; + } + } else { + resultHtml = await body.text(); + } + + // Try JSON first. The current transit server returns + // { status: "OK", file_code: "...", msg: "Upload Completed" }. + // Legacy XFS shapes (json.files / json.result) are kept as fallback. + try { + const json = JSON.parse(resultHtml); + if (json.status && /ok/i.test(json.status) && json.file_code) { + return this._buildUrlsFromCode(json.file_code); + } + if (json.file_code || json.filecode) { + return this._buildUrlsFromCode(json.file_code || json.filecode); + } + if (json.files && json.files.length > 0) { + const f = json.files[0]; + return this._buildUrlsFromCode(f.filecode || f.file_code); + } + if (json.result) { + const r = Array.isArray(json.result) ? json.result[0] : json.result; + const code = r.filecode || r.file_code; + const urls = this._buildUrlsFromCode(code); + if (urls) return urls; + } + if (json.status && !/ok/i.test(json.status) && json.msg) { + throw new Error(`Vidmoly Upload abgelehnt: ${json.msg}`); + } + } catch (err) { + if (err && /Vidmoly Upload abgelehnt/.test(err.message)) throw err; + } + + try { + return this._parseUploadResult(resultHtml); + } catch (primaryErr) { + const fallback = await this._resolveUploadedFileFromVmApi(fileName, baselineCodes, signal); + if (fallback) return fallback; + throw primaryErr; + } + } + + _normalizeTitle(value) { + return String(value || '') + .toLowerCase() + .normalize('NFKD') + .replace(/[^a-z0-9]+/g, ''); + } + + _scoreVmCandidate(file, expectedTitle) { + if (!file || !file.file_code) return -1; + if (!expectedTitle) return 0; + + const title = this._normalizeTitle(file.full_title || file.title_txt || ''); + if (!title) return -1; + if (title === expectedTitle) return 120; + if (title.startsWith(expectedTitle) || expectedTitle.startsWith(title)) return 90; + if (title.includes(expectedTitle) || expectedTitle.includes(title)) return 70; + return 0; + } + + _buildUrlsFromCode(fileCode) { + const code = String(fileCode || '').trim(); + if (!code) return null; + + return { + download_url: `${BASE_URL}/w/${code}`, + embed_url: `${BASE_URL}/embed-${code}.html`, + file_code: code + }; + } + + async _captureVmFileCodes() { + try { + const files = await this._fetchVmList(); + return new Set( + files + .map((f) => String(f.file_code || '').trim()) + .filter(Boolean) + ); + } catch { + return new Set(); + } + } + + async _fetchVmList() { + const params = new URLSearchParams({ + op: 'vm', + api: 'list', + page: '1', + per: '100', + sort: 'date', + order: 'desc', + fld_id: '0' + }); + + const res = await this._fetch(`${BASE_URL}/?${params.toString()}`); + const body = await res.text(); + + let payload; + try { + payload = JSON.parse(body); + } catch { + throw new Error('Vidmoly VM API lieferte kein JSON'); + } + + if (!payload || !Array.isArray(payload.files)) return []; + return payload.files; + } + + async _resolveUploadedFileFromVmApi(fileName, baselineCodes, signal) { + const expectedTitle = this._normalizeTitle(path.parse(fileName).name); + + for (let attempt = 0; attempt < RESULT_POLL_ATTEMPTS; attempt++) { + if (signal && signal.aborted) { + const err = new Error('Aborted'); + err.name = 'AbortError'; + throw err; + } + + let files = []; + try { + files = await this._fetchVmList(); + } catch { + files = []; + } + + const withCode = files.filter((f) => f && typeof f.file_code === 'string' && f.file_code.trim()); + const newFiles = withCode.filter((f) => !baselineCodes.has(f.file_code)); + + if (newFiles.length > 0) { + let best = null; + let bestScore = -1; + + for (const file of newFiles) { + const score = this._scoreVmCandidate(file, expectedTitle); + if (score > bestScore) { + bestScore = score; + best = file; + } + } + + if (best && bestScore > 0) { + return this._buildUrlsFromCode(best.file_code); + } + } + + if (expectedTitle) { + let bestMatch = null; + let bestScore = -1; + + for (const file of withCode) { + const score = this._scoreVmCandidate(file, expectedTitle); + if (score > bestScore) { + bestScore = score; + bestMatch = file; + } + } + + if (bestMatch && bestScore >= 90) { + return this._buildUrlsFromCode(bestMatch.file_code); + } + } + + if (attempt < RESULT_POLL_ATTEMPTS - 1) { + await this._sleep(RESULT_POLL_DELAY_MS, signal); + } + } + + return null; + } + + _sleep(ms, signal) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + if (signal) signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + + function onAbort() { + clearTimeout(timer); + if (signal) signal.removeEventListener('abort', onAbort); + const err = new Error('Aborted'); + err.name = 'AbortError'; + reject(err); + } + + if (signal) { + if (signal.aborted) return onAbort(); + signal.addEventListener('abort', onAbort); + } + }); + } + + _parseUploadResult(html) { + let download_url = null; + let embed_url = null; + let file_code = null; + + const fnMatch = html.match(/<(?:input|textarea)[^>]*name=["']fn["'][^>]*(?:value=["']([^"']+)["'])?[^>]*>([^<]*)/i); // eslint-disable-line security/detect-unsafe-regex -- parses trusted hoster HTML only + if (fnMatch) { + const codeFromFn = (fnMatch[1] || fnMatch[2] || '').trim(); + if (/^[a-z0-9]{8,16}$/i.test(codeFromFn)) { + file_code = codeFromFn; + } + } + + if (!file_code) { + const fnAltMatch = html.match(/(?:^|[?&])fn=([a-z0-9]{8,16})(?:&|$)/i); + if (fnAltMatch) file_code = fnAltMatch[1]; + } + + // Vidmoly URL patterns - includes /w/ path format + const linkPatterns = [ + /https?:\/\/vidmoly\.[a-z]+\/w\/[a-z0-9]{12}/gi, + /https?:\/\/vidmoly\.[a-z]+\/embed-[a-z0-9]{12}[^\s"']*/gi, + /https?:\/\/vidmoly\.[a-z]+\/[a-z0-9]{12}\.html/gi, + /https?:\/\/vidmoly\.[a-z]+\/[a-z0-9]{12}/gi + ]; + + for (const pattern of linkPatterns) { + const matches = html.match(pattern); + if (matches) { + for (const url of matches) { + if (url.includes('/embed-') || url.includes('/embed/')) { + if (!embed_url) embed_url = url; + } else { + if (!download_url) download_url = url; + } + } + } + } + + // Extract file code from URLs + const codeMatch = (download_url || embed_url || '').match(/\/(?:w\/)?([a-z0-9]{12})/i) + || (download_url || embed_url || '').match(/embed-([a-z0-9]{12})/i); + if (codeMatch) { + file_code = codeMatch[1]; + } + + // Try input/textarea fields + if (!download_url) { + const inputMatch = html.match(/<(?:input|textarea)[^>]*value=["'](https?:\/\/vidmoly[^"']+)["']/i); + if (inputMatch) { + download_url = inputMatch[1]; + const code = download_url.match(/\/(?:w\/)?([a-z0-9]{12})/i); + if (code) file_code = code[1]; + } + } + + // Try to find file code in any filecode reference + if (!file_code) { + const codeInPage = html.match(/filecode['":\s]+['"]?([a-z0-9]{12})['"]?/i) + || html.match(/file_code['":\s]+['"]?([a-z0-9]{12})['"]?/i); + if (codeInPage) file_code = codeInPage[1]; + } + + // Build URLs from file_code + if (file_code && !download_url) { + download_url = `${BASE_URL}/w/${file_code}`; + } + if (file_code && !embed_url) { + embed_url = `${BASE_URL}/embed-${file_code}.html`; + } + + if (!download_url && !file_code) { + const errMatch = html.match(/class=["']err["'][^>]*>([^<]+)/i); + const errMsg = errMatch ? errMatch[1].trim() : 'Kein Download-Link gefunden'; + throw new Error(`Vidmoly Upload-Ergebnis: ${errMsg}`); + } + + return { download_url, embed_url, file_code }; + } +} + +module.exports = VidmolyUploader; diff --git a/lib/voe-upload.js b/lib/voe-upload.js new file mode 100644 index 0000000..4c105c3 --- /dev/null +++ b/lib/voe-upload.js @@ -0,0 +1,409 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request } = require('undici'); + +const BASE_URL = 'https://voe.sx'; +const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const UPLOAD_TIMEOUT = 1800000; // 30 min +const RESULT_POLL_ATTEMPTS = 10; +const RESULT_POLL_DELAY_MS = 2000; + +/** + * Login-based upload for VOE.sx (Laravel / FilePond) + * Fallback when API-based upload fails or is unavailable. + */ +class VoeUploader { + constructor() { + this.cookies = new Map(); + } + + _cookieHeader() { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + _parseCookiesFromHeaders(headers) { + let setCookies; + if (typeof headers.getSetCookie === 'function') { + setCookies = headers.getSetCookie(); + } else if (headers['set-cookie']) { + setCookies = Array.isArray(headers['set-cookie']) ? headers['set-cookie'] : [headers['set-cookie']]; + } else { + return; + } + for (const raw of setCookies) { + const pair = raw.split(';')[0]; + const eq = pair.indexOf('='); + if (eq > 0) { + this.cookies.set(pair.substring(0, eq).trim(), pair.substring(eq + 1).trim()); + } + } + } + + /** + * GET/POST with cookie management and manual redirect following + */ + async _fetch(url, opts = {}, _redirectCount = 0) { + const MAX_REDIRECTS = 10; + const headers = { + 'User-Agent': USER_AGENT, + ...(opts.headers || {}) + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + const res = await fetch(url, { + ...opts, + headers, + redirect: 'manual' + }); + + this._parseCookiesFromHeaders(res.headers); + + if ([301, 302, 303, 307, 308].includes(res.status)) { + try { await res.text(); } catch {} + if (_redirectCount >= MAX_REDIRECTS) { + throw new Error('Zu viele Redirects'); + } + const location = res.headers.get('location'); + if (location) { + const nextUrl = new URL(location, url).href; + return this._fetch(nextUrl, { ...opts, method: 'GET', body: undefined }, _redirectCount + 1); + } + } + + return res; + } + + /** + * Extract CSRF token from page HTML + */ + _extractCsrfToken(html) { + // Laravel meta tag + const metaMatch = html.match(/