commit 6d0ad84d2d36353df69bbedf620f9cf36c67fd11 Author: Sucukdeluxe <259325684+Sucukdeluxe@users.noreply.github.com> Date: Sat Aug 1 17:46:29 2026 +0200 Release Multi-Hoster Uploader 3.3.108 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42a6a46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +node_modules/ +release/ +.worktrees/ +__pycache__/ +*.pyc + +electron-config.json +electron-config.json.bak +electron-config.json.tmp +electron-config.json.pre-history-split.bak +electron-config.pre-import-*.json +electron-history.json +electron-history.json.tmp +*.log +debug.log +fileuploader.log +account-rotation.log +doodstream-debug.log +upload-debug.log +release-*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..7609905 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Multi-Hoster-Upload + +Multi-Hoster-Upload is a Windows desktop app for managing large file batches across multiple video hosting services from one queue. + + + +[Download the latest release](https://github.com/Sucukdeluxe/Multi-Hoster-Upload/releases/latest) + +## Capabilities + +- Upload one batch to several supported hosters in parallel. +- Manage multiple accounts per hoster with validation, health checks, and automatic rotation. +- Add files by drag and drop or file selection and monitor live queue progress. +- Control per-hoster concurrency, bandwidth limits, retries, and folder monitoring. +- Keep local upload history and copy completed links in bulk. + +## Supported hosters + +| Hoster | Authentication | +| --- | --- | +| Doodstream | Web login or API key | +| VOE | Web login or API key | +| Vidmoly | Web login | +| Byse | API key | +| Clouddrop | API key | + +## Installation + +1. Download the Setup or portable executable from the [latest GitHub release](https://github.com/Sucukdeluxe/Multi-Hoster-Upload/releases/latest). +2. Run the installer, or launch the portable executable directly. +3. Add and validate at least one hoster account in Settings, then select files and start the queue. + +## Local data and credentials + +Settings, queue state, and upload history are stored locally in the app's user-data directory. Hoster passwords and API keys are encrypted with Electron safeStorage before being written when operating-system encryption is available; on Windows this uses DPAPI for the current user profile. + +## Development + +```powershell +npm install +npm start +npm test +npm run lint +npm run release:win +``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..aa8784a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +Report security vulnerabilities through GitHub private vulnerability reporting when it is available for this repository: open the Security tab, choose Advisories, and select "Report a vulnerability." + +Do not post credentials, passwords, API keys, tokens, cookies, private logs, configuration files, or private upload links in public issues. Redact sensitive values from reproduction steps and attachments. + +If private vulnerability reporting is unavailable, contact the repository owner through a private channel before sharing technical details. diff --git a/assets/app_icon.ico b/assets/app_icon.ico new file mode 100644 index 0000000..ed6b808 Binary files /dev/null and b/assets/app_icon.ico differ diff --git a/assets/app_icon.png b/assets/app_icon.png new file mode 100644 index 0000000..ac13bab Binary files /dev/null and b/assets/app_icon.png differ diff --git a/assets/product-overview.png b/assets/product-overview.png new file mode 100644 index 0000000..e719b86 Binary files /dev/null and b/assets/product-overview.png differ diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 0000000..513008a --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,105 @@ +import security from 'eslint-plugin-security'; + +const sharedRules = { + // Security rules + // detect-object-injection disabled: 78 false positives from config lookups like obj[hosterName] + 'security/detect-object-injection': 'off', + 'security/detect-non-literal-regexp': 'warn', + 'security/detect-unsafe-regex': 'warn', + 'security/detect-buffer-noassert': 'warn', + 'security/detect-eval-with-expression': 'error', + 'security/detect-no-csrf-before-method-override': 'warn', + 'security/detect-possible-timing-attacks': 'warn', + 'security/detect-pseudoRandomBytes': 'warn', + // Code quality + 'no-unused-vars': ['warn', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }], + 'no-undef': 'error', + 'no-constant-condition': 'warn', + 'no-debugger': 'error', + 'no-duplicate-case': 'error', + 'no-empty': ['warn', { allowEmptyCatch: true }], + 'no-ex-assign': 'error', + 'no-extra-boolean-cast': 'warn', + 'no-func-assign': 'error', + 'no-inner-declarations': 'error', + 'no-irregular-whitespace': 'error', + 'no-unreachable': 'error', + 'use-isnan': 'error', + 'valid-typeof': 'error', + 'eqeqeq': ['warn', 'always'], + 'no-caller': 'error', + 'no-eval': 'error', + 'no-implied-eval': 'error', + 'no-new-func': 'error', + 'no-throw-literal': 'warn', + 'no-self-assign': 'error', + 'no-self-compare': 'error', + 'no-loss-of-precision': 'error', + 'no-dupe-keys': 'error', + 'no-unsafe-finally': 'error', + 'no-unmodified-loop-condition': 'warn', + 'no-template-curly-in-string': 'warn', +}; + +const nodeGlobals = { + process: 'readonly', + console: 'readonly', + setTimeout: 'readonly', + clearTimeout: 'readonly', + setInterval: 'readonly', + clearInterval: 'readonly', + setImmediate: 'readonly', + Buffer: 'readonly', + URL: 'readonly', + URLSearchParams: 'readonly', + fetch: 'readonly', + crypto: 'readonly', + structuredClone: 'readonly', +}; + +export default [ + { ignores: ['**/node_modules/**', 'release/**', 'tests/**'] }, + { + files: ['**/*.js'], + ignores: ['gateway/**'], + plugins: { security }, + languageOptions: { + ecmaVersion: 2022, + sourceType: 'commonjs', + globals: { + require: 'readonly', + module: 'readonly', + exports: 'readonly', + __dirname: 'readonly', + __filename: 'readonly', + ...nodeGlobals, + AbortController: 'readonly', + AbortSignal: 'readonly', + navigator: 'readonly', + document: 'readonly', + window: 'readonly', + localStorage: 'readonly', + HTMLElement: 'readonly', + alert: 'readonly', + confirm: 'readonly', + requestAnimationFrame: 'readonly', + queueMicrotask: 'readonly', + Intl: 'readonly', + performance: 'readonly', + EventSource: 'readonly', + } + }, + rules: sharedRules + }, + { + files: ['gateway/**/*.js', 'gateway/**/*.mjs'], + ignores: ['gateway/node_modules/**'], + plugins: { security }, + languageOptions: { + ecmaVersion: 2022, + sourceType: 'module', + globals: nodeGlobals + }, + rules: sharedRules + } +]; diff --git a/lib/account-auth.js b/lib/account-auth.js new file mode 100644 index 0000000..1fd0a4a --- /dev/null +++ b/lib/account-auth.js @@ -0,0 +1,36 @@ +// Decides which credential an upload task should use for a given hoster. +// Extracted from main.js buildTaskFromAccount so the routing can be unit-tested +// without Electron. +// +// DOODSTREAM SPECIAL CASE: prefer the official doodapi.co API key whenever the +// account has one. The web-login path (username/password) drives doodstream's +// browser upload flow, which hands the filecode back inside an XFileSharing +// HTML form. On long/large uploads that form comes back empty (no fn) because a +// per-page-load sess_id token ages out over the multi-minute upload and/or the +// server-side file-registration callback times out — the upload then "succeeds" +// (bytes sent, HTTP 200) but yields no link. The JSON API returns the filecode +// directly in result[0].filecode and authenticates with a persistent api_key, +// so it has no empty-form failure mode for result retrieval. The API path was +// doodstream's ORIGINAL upload path (present since the initial commit); web +// login was added later only as an alternative for keyless accounts — so +// preferring the key here restores the intended primary path, it doesn't fight +// a deliberate choice. Keyless accounts keep using web login unchanged. +function selectUploadAuth(hoster, account) { + if (!account || typeof account !== 'object') return {}; + + if (hoster === 'doodstream.com' && account.apiKey) { + return { apiKey: account.apiKey }; + } + if (account.authType === 'api' && account.apiKey) { + return { apiKey: account.apiKey }; + } + if (account.username && account.password) { + return { username: account.username, password: account.password }; + } + if (account.apiKey) { + return { apiKey: account.apiKey }; + } + return {}; +} + +module.exports = { selectUploadAuth }; diff --git a/lib/account-rotation.js b/lib/account-rotation.js new file mode 100644 index 0000000..de6ee7c --- /dev/null +++ b/lib/account-rotation.js @@ -0,0 +1,27 @@ +function enabledAccountsFor(hosters, hoster, hasCreds) { + const list = hosters && hosters[hoster]; + if (!Array.isArray(list)) return []; + return list.filter(a => a && a.enabled !== false && hasCreds(hoster, a)); +} + +function createAccountPicker({ hosters, hosterSettings, hasCreds, indices }) { + const rotIdx = Object.assign(Object.create(null), indices || {}); + let dirty = false; + function pick(hoster) { + const enabled = enabledAccountsFor(hosters, hoster, hasCreds); + if (enabled.length === 0) return null; + const hs = (hosterSettings && hosterSettings[hoster]) || {}; + if (hs.rotateAccounts === true && enabled.length > 1) { + const cursor = Number.isFinite(rotIdx[hoster]) ? rotIdx[hoster] : 0; + rotIdx[hoster] = cursor + 1; + dirty = true; + return enabled[cursor % enabled.length]; + } + return enabled[0]; + } + pick.indices = () => ({ ...rotIdx }); + pick.dirty = () => dirty; + return pick; +} + +module.exports = { createAccountPicker, enabledAccountsFor }; diff --git a/lib/backup-crypto.js b/lib/backup-crypto.js new file mode 100644 index 0000000..9fd9f22 --- /dev/null +++ b/lib/backup-crypto.js @@ -0,0 +1,95 @@ +const crypto = require('crypto'); + +const MAGIC = Buffer.from('MHU1'); +const SALT_LEN = 16; +const IV_LEN = 12; +const TAG_LEN = 16; +const KEY_LEN = 32; +const ITERATIONS = 100_000; +const DIGEST = 'sha512'; +const ALGO = 'aes-256-gcm'; + +// Fixed app-internal passphrase — backups are opaque without the app, which is +// enough protection for API keys stored locally. We keep the AES-GCM envelope +// (with random salt/iv) so each export is still distinct and authenticated. +const APP_PASSPHRASE = 'multi-hoster-upload::backup::v1'; + +function deriveKey(passphrase, salt) { + return crypto.pbkdf2Sync(passphrase, salt, ITERATIONS, KEY_LEN, DIGEST); +} + +/** + * Encrypt a config object. + * Returns a Buffer: MHU1 | salt(16) | iv(12) | tag(16) | ciphertext + */ +function encrypt(config) { + const plaintext = Buffer.from(JSON.stringify(config), 'utf-8'); + const salt = crypto.randomBytes(SALT_LEN); + const iv = crypto.randomBytes(IV_LEN); + const key = deriveKey(APP_PASSPHRASE, salt); + + const cipher = crypto.createCipheriv(ALGO, key, iv); + const encrypted = Buffer.concat([cipher.update(plaintext), cipher.final()]); + const tag = cipher.getAuthTag(); + + plaintext.fill(0); + key.fill(0); + return Buffer.concat([MAGIC, salt, iv, tag, encrypted]); +} + +/** + * Decrypt a .mhu buffer. + * Tries the app's built-in key first; if that fails and a user password is + * provided, falls back to legacy password-based decryption. Throws a special + * error with `needsPassword = true` if the app key fails and no password was + * given, so callers can prompt the user for the legacy password. + */ +function decrypt(buffer, userPassword) { + if (buffer.length < MAGIC.length + SALT_LEN + IV_LEN + TAG_LEN + 1) { + throw new Error('Ungültiges Backup-Format'); + } + + const magic = buffer.subarray(0, 4); + if (!magic.equals(MAGIC)) { + throw new Error('Keine gültige .mhu Backup-Datei'); + } + + let offset = MAGIC.length; + const salt = buffer.subarray(offset, offset += SALT_LEN); + const iv = buffer.subarray(offset, offset += IV_LEN); + const tag = buffer.subarray(offset, offset += TAG_LEN); + const ciphertext = buffer.subarray(offset); + + const tryPassphrase = (passphrase) => { + const key = deriveKey(passphrase, salt); + const decipher = crypto.createDecipheriv(ALGO, key, iv); + decipher.setAuthTag(tag); + try { + const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]); + const result = JSON.parse(decrypted.toString('utf-8')); + decrypted.fill(0); + key.fill(0); + return result; + } catch { + key.fill(0); + return null; + } + }; + + // 1) Try the app-internal key (new format, no password required). + const fromApp = tryPassphrase(APP_PASSPHRASE); + if (fromApp) return fromApp; + + // 2) Legacy format: user had set their own password. + if (userPassword) { + const fromUser = tryPassphrase(userPassword); + if (fromUser) return fromUser; + throw new Error('Falsches Passwort oder beschädigte Datei'); + } + + const err = new Error('Dieses Backup wurde mit einem Passwort verschlüsselt'); + err.needsPassword = true; + throw err; +} + +module.exports = { encrypt, decrypt }; diff --git a/lib/clouddrop-upload.js b/lib/clouddrop-upload.js new file mode 100644 index 0000000..5223b88 --- /dev/null +++ b/lib/clouddrop-upload.js @@ -0,0 +1,239 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request, Agent } = require('undici'); + +const BASE_URL = 'https://clouddrop.cc'; +const API_BASE = `${BASE_URL}/api/cloud`; +const CHUNK_UPLOAD_BASE = 'https://upload.clouddrop.cc/api/cloud'; +const USER_AGENT = 'multi-hoster-uploader/1.0'; + +const SIMPLE_UPLOAD_LIMIT = 16 * 1024 * 1024; // 16 MB +const CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB — server's fixed chunk size +const INIT_TIMEOUT = 60_000; +const CHUNK_TIMEOUT = 30 * 60_000; // 30 min per chunk +const COMPLETE_TIMEOUT = 5 * 60_000; +const SIMPLE_UPLOAD_TIMEOUT = 30 * 60_000; + +// Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under +// the server's per-IP limit of 100 concurrent connections (cd_conn). +// Shared across all ClouddropUploader instances via module-level agent. +const clouddropAgent = new Agent({ + connections: 50, + pipelining: 1, + keepAliveTimeout: 30_000, + keepAliveMaxTimeout: 60_000 +}); + +/** + * Clouddrop.cc uploader — uses API Key (Bearer) authentication. + * Files > 16 MB use the chunked protocol, smaller files use simple upload. + * After upload, a share link is created and returned as download_url. + */ +class ClouddropUploader { + constructor(apiKey) { + this.apiKey = String(apiKey || '').trim(); + } + + _headers(extra) { + return { + 'Authorization': `Bearer ${this.apiKey}`, + 'User-Agent': USER_AGENT, + 'Accept': 'application/json', + ...(extra || {}) + }; + } + + async _parseJsonResponse(res) { + const text = await res.body.text(); + let payload = null; + try { payload = text ? JSON.parse(text) : {}; } catch { + throw new Error(`Clouddrop: API-Antwort war kein JSON (HTTP ${res.statusCode}): ${text.slice(0, 200)}`); + } + if (res.statusCode < 200 || res.statusCode >= 300) { + const msg = (payload && (payload.error || payload.message)) + || `HTTP ${res.statusCode}`; + const err = new Error(`Clouddrop: ${msg}`); + err.status = res.statusCode; + throw err; + } + return payload; + } + + /** + * Upload a file. Returns { download_url, embed_url, file_code }. + */ + async upload(filePath, progressCb, signal, throttle) { + if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt'); + const fileName = path.basename(filePath); + let fileSize = 0; + try { fileSize = fs.statSync(filePath).size; } + catch { throw new Error(`Clouddrop: Datei nicht lesbar: ${fileName}`); } + if (fileSize <= 0) throw new Error('Clouddrop: Datei ist leer'); + + let fileId; + if (fileSize <= SIMPLE_UPLOAD_LIMIT) { + fileId = await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle); + } else { + fileId = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle); + } + + return { + download_url: `${BASE_URL}/share/${fileId}`, + embed_url: null, + file_code: fileId + }; + } + + /** + * Simple upload for files < 16 MB — single multipart POST. + */ + async _uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle) { + const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex'); + const safeFileName = fileName.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); + + const preamble = + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${safeFileName}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n`; + const epilogue = `\r\n--${boundary}--\r\n`; + + const preambleBuf = Buffer.from(preamble, 'utf-8'); + const epilogueBuf = Buffer.from(epilogue, 'utf-8'); + const totalSize = preambleBuf.length + fileSize + epilogueBuf.length; + + let bytesRead = 0; + async function* generate() { + yield preambleBuf; + const fileStream = fs.createReadStream(filePath, { highWaterMark: 1024 * 1024 }); + for await (const chunk of fileStream) { + if (signal && signal.aborted) throw new Error('Aborted'); + if (throttle) await throttle.consume(chunk.length, signal); + bytesRead += chunk.length; + yield chunk; + if (progressCb) progressCb(bytesRead, fileSize); + } + yield epilogueBuf; + } + + const res = await request(`${API_BASE}/upload?mode=rename`, { + method: 'POST', + dispatcher: clouddropAgent, + body: generate(), + signal, + headers: this._headers({ + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Content-Length': String(totalSize) + }), + headersTimeout: SIMPLE_UPLOAD_TIMEOUT, + bodyTimeout: SIMPLE_UPLOAD_TIMEOUT + }); + + const payload = await this._parseJsonResponse(res); + if (!payload.fileId) throw new Error(`Clouddrop: Keine fileId in Upload-Antwort`); + return payload.fileId; + } + + /** + * Chunked upload for files > 16 MB. + * Flow: POST /upload/init → PUT /upload/:sessionId/chunk/:n (0-based) → POST /upload/:sessionId/complete + */ + async _uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle) { + // 1. Init session + const initRes = await request(`${API_BASE}/upload/init`, { + method: 'POST', + dispatcher: clouddropAgent, + signal, + headers: this._headers({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ filename: fileName, size: fileSize, parentId: null }), + headersTimeout: INIT_TIMEOUT, + bodyTimeout: INIT_TIMEOUT + }); + const initPayload = await this._parseJsonResponse(initRes); + const sessionId = initPayload.sessionId; + const chunkSize = initPayload.chunkSize || CHUNK_SIZE; + const totalChunks = initPayload.totalChunks || Math.ceil(fileSize / chunkSize); + if (!sessionId) throw new Error('Clouddrop: Keine sessionId von /upload/init'); + + // 2. Read file and PUT chunks sequentially. + // Reuse a single buffer for all chunks (only the last chunk may be smaller, + // in which case we slice a view). Avoids 64× 16 MB allocations on a 1 GB + // file — real GC pressure during busy uploads. + const fh = await fs.promises.open(filePath, 'r'); + let bytesSent = 0; + const reusableBuf = Buffer.allocUnsafe(chunkSize); + try { + for (let i = 0; i < totalChunks; i++) { + if (signal && signal.aborted) throw new Error('Aborted'); + + const offset = i * chunkSize; + const remaining = fileSize - offset; + const thisChunkSize = Math.min(chunkSize, remaining); + await fh.read(reusableBuf, 0, thisChunkSize, offset); + const body = thisChunkSize === chunkSize + ? reusableBuf + : reusableBuf.subarray(0, thisChunkSize); + + if (throttle) await throttle.consume(thisChunkSize, signal); + + const chunkRes = await request(`${CHUNK_UPLOAD_BASE}/upload/${sessionId}/chunk/${i}`, { + method: 'PUT', + dispatcher: clouddropAgent, + signal, + body, + headers: this._headers({ + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(thisChunkSize) + }), + headersTimeout: CHUNK_TIMEOUT, + bodyTimeout: CHUNK_TIMEOUT + }); + await this._parseJsonResponse(chunkRes); + + bytesSent += thisChunkSize; + if (progressCb) progressCb(bytesSent, fileSize); + } + } finally { + try { await fh.close(); } catch {} + } + + // 3. Complete session — all bytes are already on the server at this point. + // We MUST NOT throw here, otherwise the upload-manager would retry the entire + // multi-GB upload. Any failure (timeout, non-JSON, missing fileId, server still + // post-processing) is swallowed and we fall back to sessionId as file_code. + try { + const completeRes = await request(`${API_BASE}/upload/${sessionId}/complete`, { + method: 'POST', + dispatcher: clouddropAgent, + signal, + headers: this._headers({ 'Content-Type': 'application/json' }), + body: '{}', + headersTimeout: COMPLETE_TIMEOUT, + bodyTimeout: COMPLETE_TIMEOUT + }); + const completePayload = await this._parseJsonResponse(completeRes).catch(() => ({})); + return completePayload.fileId || completePayload.id || sessionId; + } catch { + return sessionId; + } + } + + /** + * Lightweight auth check — GET /api/cloud/files (list root, small response). + */ + async checkAuth(signal) { + if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt'); + const res = await request(`${API_BASE}/files/?limit=1`, { + method: 'GET', + dispatcher: clouddropAgent, + signal, + headers: this._headers(), + headersTimeout: 15_000, + bodyTimeout: 15_000 + }); + await this._parseJsonResponse(res); + return true; + } +} + +module.exports = ClouddropUploader; diff --git a/lib/coalesced-set.js b/lib/coalesced-set.js new file mode 100644 index 0000000..4136d3f --- /dev/null +++ b/lib/coalesced-set.js @@ -0,0 +1,73 @@ +// Microtask-coalesced set. Adds are O(1); the apply callback runs once per +// scheduler tick with every id collected since the last flush. +// +// Used by the renderer to merge a burst of done-jobs (e.g. 500 jobs all +// finishing within milliseconds) into a single queueJobs.filter() pass — +// without this each event was its own O(N) sweep, so 500 finishes were +// O(N²) and visibly froze the UI on completion. +// +// Loaded both as a CommonJS module (Node tests) and as a browser global +// (renderer/app.js via index.html script tag). + +(function (root) { + 'use strict'; + + /** + * Build a coalesced set. + * @param {{ apply: (Set) => void, scheduler?: (cb: () => void) => void }} opts + * apply: called once per scheduler tick with the accumulated ids. + * scheduler: defaults to queueMicrotask. Tests can pass a synchronous + * stand-in to avoid async waits. + */ + function makeCoalescedSet(opts) { + if (!opts || typeof opts.apply !== 'function') { + throw new TypeError('makeCoalescedSet: { apply: fn } required'); + } + const apply = opts.apply; + const scheduler = typeof opts.scheduler === 'function' + ? opts.scheduler + : (typeof queueMicrotask === 'function' ? queueMicrotask : (cb) => Promise.resolve().then(cb)); + let pending = new Set(); + let scheduled = false; + + function flush() { + scheduled = false; + if (pending.size === 0) return; + const drop = pending; + pending = new Set(); + try { apply(drop); } catch (e) { + // Don't let a failing apply lock out the next batch — surface it + // but keep the coalescer usable. + if (typeof console !== 'undefined' && console.error) console.error(e); + } + } + + return { + add(id) { + pending.add(id); + if (!scheduled) { + scheduled = true; + scheduler(flush); + } + }, + /** + * Synchronously consume any pending ids. Used by beforeunload paths + * where we can't wait for the next microtask before persisting. + */ + drainSync() { + if (pending.size === 0) return; + const drop = pending; + pending = new Set(); + scheduled = false; + apply(drop); + }, + /** Introspection for tests + diagnostics. */ + pendingSize() { return pending.size; }, + isScheduled() { return scheduled; } + }; + } + + const api = { makeCoalescedSet }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + else if (root) root.CoalescedSet = api; +})(typeof window !== 'undefined' ? window : this); diff --git a/lib/config-store.js b/lib/config-store.js new file mode 100644 index 0000000..566372f --- /dev/null +++ b/lib/config-store.js @@ -0,0 +1,654 @@ +const fs = require('fs'); +const path = require('path'); +const secretStore = require('./secret-store'); +const { normalizeLogMode } = require('./log-mode'); + +const HOSTER_SETTINGS_DEFAULTS = { + retries: 3, + maxSpeedKbs: 0, // 0 = unlimited + parallelCount: 2, // 1-100 + restartBelowKbs: 0, // 0 = off + timeIntervalSec: 0, // delay between jobs + maxSizeMb: 0, // 0 = unlimited + logToFile: true, // write this hoster's successful links to fileuploader.log + rotateAccounts: false, + sizeMemoEnabled: true +}; + +// Template for each hoster type (used as defaults for new accounts) +const HOSTER_ACCOUNT_TEMPLATES = { + 'doodstream.com': { enabled: true, authType: 'login', username: '', password: '' }, + 'doodstream.com:api': { enabled: true, authType: 'api', apiKey: '' }, + 'voe.sx': { enabled: true, authType: 'login', username: '', password: '' }, + 'voe.sx:api': { enabled: true, authType: 'api', apiKey: '' }, + 'vidmoly.me': { enabled: true, authType: 'login', username: '', password: '' }, + 'byse.sx': { enabled: true, authType: 'api', apiKey: '' }, + 'clouddrop.cc': { enabled: true, authType: 'api', apiKey: '' } +}; + +// All known hoster names (used for iteration) +const HOSTER_NAMES = ['doodstream.com', 'voe.sx', 'vidmoly.me', 'byse.sx', 'clouddrop.cc']; + +// Dropdown options for "Add Account" modal: value -> label +const HOSTER_ADD_OPTIONS = [ + { value: 'doodstream.com', label: 'Doodstream (Web Login)', hoster: 'doodstream.com', authType: 'login' }, + { value: 'doodstream.com:api', label: 'Doodstream (API)', hoster: 'doodstream.com', authType: 'api' }, + { value: 'voe.sx', label: 'Voe (Web Login)', hoster: 'voe.sx', authType: 'login' }, + { value: 'voe.sx:api', label: 'Voe (API)', hoster: 'voe.sx', authType: 'api' }, + { value: 'vidmoly.me', label: 'Vidmoly (Web Login)', hoster: 'vidmoly.me', authType: 'login' }, + { value: 'byse.sx', label: 'Byse (API)', hoster: 'byse.sx', authType: 'api' }, + { value: 'clouddrop.cc', label: 'Clouddrop (API)', hoster: 'clouddrop.cc', authType: 'api' } +]; + +const DEFAULTS = { + hosters: { + 'doodstream.com': [], + 'voe.sx': [], + 'vidmoly.me': [], + 'byse.sx': [], + 'clouddrop.cc': [] + }, + hosterSettings: { + 'doodstream.com': { ...HOSTER_SETTINGS_DEFAULTS }, + 'voe.sx': { ...HOSTER_SETTINGS_DEFAULTS }, + 'vidmoly.me': { ...HOSTER_SETTINGS_DEFAULTS }, + 'byse.sx': { ...HOSTER_SETTINGS_DEFAULTS }, + 'clouddrop.cc': { ...HOSTER_SETTINGS_DEFAULTS } + }, + globalSettings: { + alwaysOnTop: false, + shutdownAfterFinish: 'nothing', // nothing | sleep | shutdown | restart + logFilePath: '', + sessionLog: false, // legacy boolean (kept for back-compat reads); normalized into logMode on load + logVerbose: false, // when true, [DEBUG] level entries are written to debug.log + webhookUrl: '', // POST target on batch-done (Discord or generic JSON) + webhookMention: '', // optional Discord ping target: user-id, role:id, @here, @everyone + autoRetryRounds: 0, // 0 = off; 1-5 automatic retry rounds for transient failures after batch end + autoRetryDelayMin: 5, // base delay in minutes between auto-retry rounds (linear backoff: round N waits N*delay) + historyRetention: 'all', // 'all' | '7d' | '30d' | '90d' | '1000' | '100' — storage cap for upload history + // NOTE: logMode is intentionally NOT in DEFAULTS. If it were, the deep-merge + // would seed logMode='single' for every load, which would beat (and silently + // erase) the legacy sessionLog:true → "daily" migration. normalizeLogMode in + // load() sets logMode after the merge, looking at the saved-only data. + resumeQueueOnLaunch: true, + parallelUploadCount: 0, // 0 = use per-hoster limits only + scaleParallelUploads: false, + removeFromQueueOnDone: false, + showDropTarget: false, + globalMaxSpeedKbs: 0, // 0 = unlimited global speed + pendingQueue: null, + scramble: { + active: false, + prefix: '', + suffix: '', + chars: 'both', // 'letters' | 'numbers' | 'both' + length: 0 // 0 = same as original basename length + }, + folderMonitor: { + enabled: false, + folderPath: '', + recursive: false, + filterMode: 'include', // 'include' | 'exclude' + extensions: '', // comma-separated: 'mp4,mkv,avi' + skipDuplicates: true, + delaySec: 3, + autoStart: true, + hosters: [] // pre-selected hosters, empty = ask via modal + }, + remote: { + enabled: false, + port: 9100, + token: '', + allowInput: true + }, + diagnostics: { + enabled: false, + port: 9110, + token: '', + label: '', + codeIssuedAt: 0, + bindMode: 'local', + publicHost: '', + allowlist: [], + bindAddress: '127.0.0.1' + } + }, + history: [], + rotationCursors: {} +}; + +const HISTORY_RETENTION_OPTIONS = [ + { value: 'all', label: 'Alles behalten' }, + { value: '7d', label: 'Letzte 7 Tage' }, + { value: '30d', label: 'Letzte 30 Tage' }, + { value: '90d', label: 'Letzte 90 Tage' }, + { value: '1000', label: 'Letzte 1000 Uploads' }, + { value: '100', label: 'Letzte 100 Uploads' } +]; + +function batchTimestampMs(batch) { + const raw = batch && batch.timestamp; + if (raw === null || raw === undefined || raw === '') return null; + const ms = typeof raw === 'number' ? raw : Date.parse(raw); + return Number.isFinite(ms) ? ms : null; +} + +function batchRowCount(batch) { + let n = 0; + const files = (batch && batch.files) || []; + for (const file of files) { + for (const result of (file.results || [])) { + if (result.status === 'aborted' || result.status === 'error') continue; + n++; + } + } + return n; +} + +function countHistoryRows(history) { + let n = 0; + for (const batch of (history || [])) n += batchRowCount(batch); + return n; +} + +function applyHistoryRetention(history, retention, nowMs) { + if (!Array.isArray(history) || history.length === 0) return history; + const policy = String(retention || 'all'); + if (policy === 'all') return history; + + if (/^\d+d$/.test(policy)) { + const days = parseInt(policy, 10); + if (!Number.isFinite(days) || days <= 0) return history; + const cutoff = nowMs - days * 86400000; + return history.filter(b => { + const ts = batchTimestampMs(b); + return ts === null || ts >= cutoff; + }); + } + + const maxRows = parseInt(policy, 10); + if (!Number.isFinite(maxRows) || maxRows <= 0) return history; + const keptReversed = []; + let acc = 0; + for (let i = history.length - 1; i >= 0; i--) { + keptReversed.push(history[i]); + acc += batchRowCount(history[i]); + if (acc >= maxRows) break; + } + return keptReversed.reverse(); +} + +class ConfigStore { + constructor(app) { + const dir = app && app.isPackaged + ? app.getPath('userData') + : path.join(__dirname, '..'); + this.filePath = path.join(dir, 'electron-config.json'); + this.historyPath = path.join(dir, 'electron-history.json'); + this._writeQueue = Promise.resolve(); // Serializes all writes to prevent race conditions + this._historyWriteQueue = Promise.resolve(); + this._historyMigrated = false; + this._cache = null; + this._cacheKey = ''; + this._perfLog = null; + this._wqDepth = 0; + + // Migrate config from old location if current doesn't exist + if (!fs.existsSync(this.filePath) && app && app.isPackaged) { + this._migrateFromOldPath(app); + } + if (app && app.isPackaged) { + this._migrateHistory(); + } + } + + _readHistoryFile() { + try { + const raw = fs.readFileSync(this.historyPath, 'utf-8'); + if (!raw || raw.trim().length < 2) return []; + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed; + if (parsed && Array.isArray(parsed.history)) return parsed.history; + return []; + } catch { + return null; + } + } + + _writeHistoryFileDurable(arr) { + const tmp = this.historyPath + '.tmp'; + const fd = fs.openSync(tmp, 'w'); + try { + fs.writeSync(fd, JSON.stringify(arr)); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(tmp, this.historyPath); + } + + _writeHistoryFileAtomic(arr) { + return new Promise((resolve, reject) => { + const tmp = this.historyPath + '.tmp'; + fs.writeFile(tmp, JSON.stringify(arr), 'utf-8', (err) => { + if (err) return reject(err); + try { fs.renameSync(tmp, this.historyPath); } catch (e) { return reject(e); } + resolve(); + }); + }); + } + + _enqueueHistoryWrite(fn) { + this._historyWriteQueue = this._historyWriteQueue.then(fn, fn); + return this._historyWriteQueue; + } + + _migrateHistory() { + try { + if (fs.existsSync(this.historyPath)) { + this._historyMigrated = Array.isArray(this._readHistoryFile()); + return; + } + let cfg = null; + try { cfg = this._readAndParse(this.filePath); } catch {} + const hist = (cfg && Array.isArray(cfg.history)) ? cfg.history : []; + this._writeHistoryFileDurable(hist); + const check = this._readHistoryFile(); + if (Array.isArray(check) && check.length === hist.length) { + if (hist.length > 0) { + try { fs.copyFileSync(this.filePath, this.filePath + '.pre-history-split.bak'); } catch {} + } + this._historyMigrated = true; + } else { + this._historyMigrated = false; + } + } catch { + this._historyMigrated = false; + } + } + + _migrateFromOldPath(app) { + try { + const appDataDir = path.dirname(app.getPath('userData')); + // Check alternate folder names that may have been used + const candidates = ['multi-hoster-uploader', 'Multi-Hoster-Upload']; + for (const name of candidates) { + const oldPath = path.join(appDataDir, name, 'electron-config.json'); + if (oldPath !== this.filePath && fs.existsSync(oldPath)) { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true }); + fs.copyFileSync(oldPath, this.filePath); + return; + } + } + // Also check next to the executable (portable mode previous location) + const exeDir = path.dirname(app.getPath('exe')); + const portablePath = path.join(exeDir, 'electron-config.json'); + if (portablePath !== this.filePath && fs.existsSync(portablePath)) { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true }); + fs.copyFileSync(portablePath, this.filePath); + } + } catch {} + } + + _readAndParse(filePath) { + const raw = fs.readFileSync(filePath, 'utf-8'); + if (!raw || raw.trim().length < 2) return null; + return JSON.parse(raw); + } + + _clone(obj) { + try { return structuredClone(obj); } + catch { return JSON.parse(JSON.stringify(obj)); } + } + + setPerfLog(fn) { this._perfLog = typeof fn === 'function' ? fn : null; } + + _pqLen(globalSettings) { + const pq = globalSettings && globalSettings.pendingQueue; + return pq && Array.isArray(pq.queueJobs) ? pq.queueJobs.length : 0; + } + + _callerTag() { + const lines = (new Error().stack || '').split('\n'); + const out = []; + for (let i = 2; i < lines.length && out.length < 3; i++) { + const line = lines[i].trim(); + if (/config-store\.js/.test(line)) continue; + const m = line.match(/at (?:async )?([^ (]+)/); + if (m) out.push(m[1].split('.').pop()); + } + return out.join('<') || '?'; + } + + load() { + if (!this._perfLog) return this._loadImpl(); + const hadCache = !!this._cache; + const t0 = performance.now(); + const r = this._loadImpl(); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = this._pqLen(r && r.globalSettings); + const h = (r && r.history || []).length; + this._perfLog(`config-load wall=${dt.toFixed(0)}ms cache=${hadCache ? 'hit' : 'miss'} hist=${h} queue=${q} via=${this._callerTag()}`); + } + return r; + } + + _loadImpl() { + try { + // In-memory cache keyed on the file's mtime+size. The processed config + // (merged + credential-decrypted) is reparsed/re-decrypted from disk ONLY + // when the file actually changes. Our own writes refresh the cache (see + // _commit), and an external edit changes mtime/size so the cache misses + // and we reread. Without this, every one of the ~38 main.js load() call + // sites (incl. the per-500ms log-flush path) re-read disk + JSON.parse the + // whole growing history + DPAPI-decrypt every credential — the dominant + // long-running main-thread drag. load() always returns a CLONE so callers + // can mutate the result without corrupting the cache. + let stat = null; + try { stat = fs.statSync(this.filePath); } catch {} + const statKey = stat ? `${stat.mtimeMs}:${stat.size}` : ''; + if (stat && this._cache && this._cacheKey === statKey) { + return this._clone(this._cache); + } + + let data = null; + // Try main config + try { data = this._readAndParse(this.filePath); } catch {} + // Fallback to backup if main is empty/corrupt + if (!data) { + try { data = this._readAndParse(this.filePath + '.bak'); } catch {} + } + if (!data) { + try { data = this._readAndParse(this.filePath + '.pre-history-split.bak'); } catch {} + } + if (!data) { + const fresh = JSON.parse(JSON.stringify(DEFAULTS)); + fresh.globalSettings.logMode = normalizeLogMode(fresh.globalSettings); + return fresh; + } + + // Migrate old single-object format to array format + for (const [name, val] of Object.entries(data.hosters || {})) { + if (val && !Array.isArray(val)) { + if (!val.id) val.id = `${name}-migrated-${Date.now()}`; + // Infer authType for old format accounts + if (!val.authType) { + if (name === 'byse.sx') val.authType = 'api'; + else if (name === 'vidmoly.me') val.authType = 'login'; + else if (val.username && val.password) val.authType = 'login'; + else if (val.apiKey) val.authType = 'api'; + else val.authType = 'login'; + } + data.hosters[name] = [val]; + } + } + + // Merge hosters: ensure all known hosters exist as arrays + const hosters = {}; + for (const name of HOSTER_NAMES) { + const saved = data.hosters && data.hosters[name]; + if (Array.isArray(saved) && saved.length > 0) { + hosters[name] = saved.map((acc, i) => { + // Ensure authType is set on every account + if (!acc.authType) { + if (name === 'byse.sx') acc.authType = 'api'; + else if (name === 'vidmoly.me') acc.authType = 'login'; + else if (acc.username && acc.password) acc.authType = 'login'; + else if (acc.apiKey) acc.authType = 'api'; + else acc.authType = 'login'; + } + return { + ...acc, + id: acc.id || `${name}-${Date.now()}-${i}` + }; + }); + } else { + hosters[name] = []; + } + } + + // Merge hoster settings with defaults + const hosterSettings = {}; + for (const name of Object.keys(DEFAULTS.hosterSettings)) { + hosterSettings[name] = { + ...HOSTER_SETTINGS_DEFAULTS, + ...(data.hosterSettings && data.hosterSettings[name] || {}) + }; + } + const savedGlobal = data.globalSettings || {}; + const globalSettings = { + ...DEFAULTS.globalSettings, + ...savedGlobal + }; + // Deep-merge nested objects so new keys are always present + for (const key of Object.keys(DEFAULTS.globalSettings)) { + const def = DEFAULTS.globalSettings[key]; + if (def && typeof def === 'object' && !Array.isArray(def)) { + globalSettings[key] = { ...def, ...(savedGlobal[key] || {}) }; + } + } + // Normalize logMode at this single boundary. Legacy sessionLog: true + // means *daily* (the old field was named after a misnomer); see log-mode.js. + // Downstream readers consume logMode only and must NOT derive from + // sessionLog at call sites. + globalSettings.logMode = normalizeLogMode(globalSettings); + const rotationCursors = (data.rotationCursors && typeof data.rotationCursors === 'object' && !Array.isArray(data.rotationCursors)) + ? data.rotationCursors + : {}; + const result = { hosters, hosterSettings, globalSettings, history: this._historyMigrated ? [] : (data.history || []), rotationCursors }; + // Decrypt credentials stored with safeStorage so the rest of the app + // keeps working with plaintext in memory. + secretStore.decryptCredentials(result); + if (stat) { + this._cache = result; + this._cacheKey = statKey; + } + return this._clone(result); + } catch { + const fresh = JSON.parse(JSON.stringify(DEFAULTS)); + fresh.globalSettings.logMode = normalizeLogMode(fresh.globalSettings); + return fresh; + } + } + + // Encrypt credential fields without mutating the caller's plaintext object. + // Only `hosters` carries credentials, so we clone ONLY that subtree — the rest + // (history, globalSettings, …) is referenced read-only into the stringified + // object. Deep-cloning the whole config here (incl. an ever-growing history) + // on every write was a primary long-running main-thread stall. + _serializeForDisk(config) { + const hosters = this._clone(config.hosters || {}); + secretStore.encryptCredentials({ hosters }); + return JSON.stringify({ ...config, hosters }, null, 2); + } + + _commit(config) { + if (!this._perfLog) return this._atomicWrite(this._serializeForDisk(config)); + const t0 = performance.now(); + const data = this._serializeForDisk(config); + const dt = performance.now() - t0; + if (dt >= 20) { + const q = this._pqLen(config.globalSettings); + const h = (config.history || []).length; + this._perfLog(`config-serialize wall=${dt.toFixed(0)}ms bytes=${data.length} hist=${h} queue=${q} wqDepth=${this._wqDepth} via=${this._callerTag()}`); + } + return this._atomicWrite(data); + } + + _enqueueWrite(fn) { + this._wqDepth++; + const done = () => { this._wqDepth--; }; + this._writeQueue = this._writeQueue.then(fn, fn).then(done, done); + return this._writeQueue; + } + + _anyHosters(cfg) { + const h = cfg && cfg.hosters; + return !!h && typeof h === 'object' && Object.values(h).some(a => Array.isArray(a) && a.length > 0); + } + + _recoverHostersFromDisk() { + for (const p of [this.filePath, this.filePath + '.bak', this.filePath + '.pre-history-split.bak']) { + try { + const raw = fs.readFileSync(p, 'utf-8'); + if (!raw || raw.trim().length < 2) continue; + const data = JSON.parse(raw); + if (this._anyHosters(data)) return data.hosters; + } catch {} + } + return null; + } + + _guardHosters(current, hostersIntentional) { + if (!hostersIntentional && !this._anyHosters(current)) { + const recovered = this._recoverHostersFromDisk(); + if (recovered) { + current.hosters = recovered; + if (this._perfLog) this._perfLog('config-guard: prevented account wipe — restored hosters from on-disk backup after a corrupt/empty read'); + } + } + return current; + } + + save(config) { + return this._enqueueWrite(() => { + const current = this.load(); + if (config.hosters) current.hosters = config.hosters; + if (config.hosterSettings) current.hosterSettings = config.hosterSettings; + if (config.globalSettings) current.globalSettings = config.globalSettings; + this._guardHosters(current, !!config.hosters); + return this._commit(current); + }); + } + + loadHistory() { + if (this._historyMigrated) { + return this._readHistoryFile() || []; + } + const config = this.load(); + return config.history || []; + } + + _atomicWrite(data) { + return new Promise((resolve, reject) => { + const tmpPath = this.filePath + '.tmp'; + const backupPath = this.filePath + '.bak'; + let fd; + try { + fd = fs.openSync(tmpPath, 'w'); + fs.writeSync(fd, data); + fs.fsyncSync(fd); + } catch (e) { + try { if (fd !== undefined) fs.closeSync(fd); } catch {} + return reject(e); + } + try { fs.closeSync(fd); } catch {} + Promise.resolve().then(() => { + try { + try { + if (fs.existsSync(this.filePath)) { + const cur = fs.readFileSync(this.filePath, 'utf-8'); + if (cur && cur.trim().length > 2) fs.writeFileSync(backupPath, cur, 'utf-8'); + } + } catch {} + fs.renameSync(tmpPath, this.filePath); + } catch (e) { return reject(e); } + // Invalidate the read cache: the next load() re-reads + re-merges the + // freshly-written file (the on-disk format is sparse — load() fills + // defaults — so we must NOT serve a pre-merge in-memory object). + this._cache = null; + this._cacheKey = ''; + resolve(); + }); + }); + } + + appendHistory(entry) { + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => { + const cur = this._readHistoryFile(); + if (cur === null && fs.existsSync(this.historyPath)) return; + const arr = cur || []; + arr.push(entry); + const gs = this.load().globalSettings; + const retention = (gs && gs.historyRetention) || 'all'; + const pruned = applyHistoryRetention(arr, retention, Date.now()); + return this._writeHistoryFileAtomic(pruned); + }); + } + return this._enqueueWrite(() => { + const config = this.load(); + config.history.push(entry); + const retention = (config.globalSettings && config.globalSettings.historyRetention) || 'all'; + config.history = applyHistoryRetention(config.history, retention, Date.now()); + return this._commit(config); + }); + } + + pruneHistory(retention, opts = {}) { + const dryRun = !!opts.dryRun; + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => { + const current = this._readHistoryFile() || []; + const beforeBatches = current.length; + const beforeRows = countHistoryRows(current); + const pruned = applyHistoryRetention(current, retention, Date.now()); + const result = { + removedBatches: beforeBatches - pruned.length, + removedRows: beforeRows - countHistoryRows(pruned), + keptBatches: pruned.length, + keptRows: countHistoryRows(pruned) + }; + if (dryRun) return result; + return this._writeHistoryFileAtomic(pruned) + .then(() => this.save({ globalSettings: { ...this.load().globalSettings, historyRetention: String(retention || 'all') } })) + .then(() => result); + }); + } + return this._enqueueWrite(() => { + const config = this.load(); + const beforeBatches = config.history.length; + const beforeRows = countHistoryRows(config.history); + const pruned = applyHistoryRetention(config.history, retention, Date.now()); + const result = { + removedBatches: beforeBatches - pruned.length, + removedRows: beforeRows - countHistoryRows(pruned), + keptBatches: pruned.length, + keptRows: countHistoryRows(pruned) + }; + if (dryRun) return result; + config.history = pruned; + if (config.globalSettings) config.globalSettings.historyRetention = String(retention || 'all'); + return this._commit(config).then(() => result); + }); + } + + clearHistory() { + if (this._historyMigrated) { + return this._enqueueHistoryWrite(() => this._writeHistoryFileAtomic([])); + } + return this._enqueueWrite(() => { + const config = this.load(); + config.history = []; + return this._commit(config); + }); + } + + saveRotationCursors(cursors) { + return this._enqueueWrite(() => { + const config = this.load(); + config.rotationCursors = (cursors && typeof cursors === 'object' && !Array.isArray(cursors)) ? cursors : {}; + this._guardHosters(config, false); + return this._commit(config); + }); + } +} + +module.exports = ConfigStore; +module.exports.HOSTER_ACCOUNT_TEMPLATES = HOSTER_ACCOUNT_TEMPLATES; +module.exports.HOSTER_NAMES = HOSTER_NAMES; +module.exports.HOSTER_ADD_OPTIONS = HOSTER_ADD_OPTIONS; +module.exports.HISTORY_RETENTION_OPTIONS = HISTORY_RETENTION_OPTIONS; +module.exports.applyHistoryRetention = applyHistoryRetention; +module.exports.countHistoryRows = countHistoryRows; diff --git a/lib/diagnostics-agent.js b/lib/diagnostics-agent.js new file mode 100644 index 0000000..f5c2239 --- /dev/null +++ b/lib/diagnostics-agent.js @@ -0,0 +1,32 @@ +function createAgent(collectors) { + const OPS = { + get_system_info: (a) => collectors.getSystemInfo(a), + server_health: (a) => collectors.serverHealth(a), + get_config_redacted: (a) => collectors.getConfigRedacted(a), + list_logs: () => collectors.listLogs(), + read_log: (a) => collectors.readLog(a), + tail_log: (a) => collectors.readLog(a), + get_app_events: (a) => collectors.getAppEvents(a), + list_errors: (a) => collectors.listErrors(a), + get_queue_state: (a) => collectors.getQueueState(a), + get_history: (a) => collectors.getHistory(a), + get_rotation_state: () => collectors.getRotationState(), + get_health: () => collectors.getHealth() + }; + + function handle(op, args) { + const fn = (typeof op === 'string' && Object.prototype.hasOwnProperty.call(OPS, op)) ? OPS[op] : null; + if (typeof fn !== 'function') return { ok: false, error: `unknown or non-readonly op: ${op}` }; + try { + const data = fn(args || {}); + if (data && data.ok === false) return data; + return { ok: true, data }; + } catch (e) { + return { ok: false, error: String((e && e.message) || e) }; + } + } + + return { handle, ops: Object.keys(OPS) }; +} + +module.exports = { createAgent }; diff --git a/lib/diagnostics-collectors.js b/lib/diagnostics-collectors.js new file mode 100644 index 0000000..dcfaf23 --- /dev/null +++ b/lib/diagnostics-collectors.js @@ -0,0 +1,277 @@ +const fs = require('fs'); +const path = require('path'); + +const READABLE_LOGS = { + debug: 'debug', + fileuploader: 'fileuploader', + accountRotation: 'accountRotation', + crash: 'crashLog' +}; + +const QUEUE_STATUSES = ['preview', 'queued', 'getting-server', 'uploading', 'retrying', 'done', 'error', 'aborted', 'skipped']; + +function createCollectors(deps) { + const { loadConfig, loadHistory, getAllLogPaths, support, stats, appInfo, systemInfo, agentInfo } = deps; + + function _secrets() { + try { return support.collectSecretValues(loadConfig()); } catch { return []; } + } + + function _deepRedact(value, secrets) { + const s = secrets || _secrets(); + const walk = (v) => { + if (typeof v === 'string') return support.redactLogText(v, s); + if (Array.isArray(v)) return v.map(walk); + if (v && typeof v === 'object') { + const o = {}; + for (const k of Object.keys(v)) o[k] = walk(v[k]); + return o; + } + return v; + }; + try { return walk(value); } catch { return value; } + } + + function _resolveLogPath(name, backup) { + const key = READABLE_LOGS[name]; + if (!key) return null; + const paths = getAllLogPaths(); + let p = paths[key]; + if (!p) return null; + if (backup === 1 || backup === 2) p = `${p}.${backup}`; + return p; + } + + function getSystemInfo() { + return { app: appInfo(), system: systemInfo(), agent: agentInfo() }; + } + + function getConfigRedacted(args) { + const section = (args && args.section) || 'all'; + const cfg = loadConfig(); + const secrets = support.collectSecretValues(cfg); + const sanitized = support.sanitizeConfig(cfg); + let pick; + let note; + if (section === 'all') { + pick = { ...sanitized }; + delete pick.history; + note = 'history omitted from config — use get_history'; + } else { + pick = sanitized[section] !== undefined ? sanitized[section] : null; + } + return { section, note, config: _deepRedact(pick, secrets) }; + } + + function listLogs() { + const paths = getAllLogPaths(); + const dir = paths.logDir; + const files = []; + for (const [name, key] of Object.entries(READABLE_LOGS)) { + const base = paths[key]; + if (!base) continue; + const variants = []; + for (const suffix of ['', '.1', '.2']) { + const fp = base + suffix; + try { + const st = fs.statSync(fp); + variants.push({ backup: suffix === '' ? 0 : Number(suffix.slice(1)), sizeBytes: st.size, mtime: st.mtime.toISOString() }); + } catch {} + } + files.push({ name, path: base, readable: true, present: variants.length > 0, variants }); + } + let siblings = []; + try { + siblings = fs.readdirSync(dir) + .filter(f => /\.log(\.\d+)?$/i.test(f)) + .filter(f => !files.some(x => path.basename(x.path) === f || f.startsWith(path.basename(x.path)))); + siblings = siblings.map(f => { + let size = 0, mtime = null; + try { const st = fs.statSync(path.join(dir, f)); size = st.size; mtime = st.mtime.toISOString(); } catch {} + return { name: f, readable: false, sizeBytes: size, mtime }; + }); + } catch {} + return { dir, files, otherLogs: siblings }; + } + + function readLog(args) { + const a = args || {}; + const name = a.name; + const p = _resolveLogPath(name, a.backup); + if (!p) return { ok: false, error: `unknown or non-readable log: ${name}` }; + const tailKb = Math.min(Math.max(Number(a.tailKb) || 256, 1), 1024); + const raw = support.collectFile(p, name, tailKb * 1024); + let content = support.redactLogText(raw, _secrets()); + let matchedLines; + if (a.grep && typeof a.grep === 'string' && a.grep.length <= 200) { + const terms = a.grep.split('|').map(s => s.trim().toLowerCase()).filter(Boolean); + if (terms.length) { + const lines = content.split('\n').filter(l => { + const low = l.toLowerCase(); + return terms.some(t => low.includes(t)); + }); + matchedLines = lines.length; + content = lines.join('\n'); + } + } + let sizeBytes = null; + try { sizeBytes = fs.statSync(p).size; } catch {} + return { name, path: p, sizeBytes, returnedBytes: Buffer.byteLength(content), tailKb, matchedLines, content }; + } + + function getAppEvents(args) { + const limit = Math.min(Math.max(Number(args && args.limit) || 50, 1), 500); + const out = []; + const secrets = _secrets(); + for (const name of ['crash', 'debug']) { + const p = _resolveLogPath(name); + if (!p) continue; + const raw = support.redactLogText(support.collectFile(p, name, 256 * 1024), secrets); + const lines = raw.split('\n').filter(l => l.trim() && !l.startsWith('===')); + for (const line of lines.slice(-limit)) out.push({ source: name, text: line }); + } + return { events: out.slice(-limit), truncated: out.length > limit }; + } + + function _historyErrors(history, opts) { + const o = opts || {}; + const sinceMs = Number.isFinite(o.sinceMs) ? o.sinceMs : null; + const secrets = _secrets(); + const errors = []; + const byCategory = {}; + for (const batch of (Array.isArray(history) ? history : [])) { + if (!batch || !Array.isArray(batch.files)) continue; + const ts = batch.timestamp ? Date.parse(batch.timestamp) : null; + if (sinceMs !== null && ts !== null && ts < sinceMs) continue; + for (const file of batch.files) { + if (!file || !Array.isArray(file.results)) continue; + for (const r of file.results) { + if (!r || r.status === 'done') continue; + const category = stats.classifyErrorCategory(r.error); + if (o.category && o.category !== category) continue; + if (o.hoster && o.hoster !== r.hoster) continue; + byCategory[category] = (byCategory[category] || 0) + 1; + errors.push({ + ts: batch.timestamp || null, + fileName: file.name || file.fileName || '', + hoster: r.hoster || '', + accountId: r.accountId || undefined, + category, + error: support.redactLogText(String(r.error || ''), secrets) + }); + } + } + } + return { errors, byCategory }; + } + + function listErrors(args) { + const a = args || {}; + const cfg = loadConfig(); + const { errors, byCategory } = _historyErrors(cfg.history, a); + const limit = Math.min(Math.max(Number(a.limit) || 100, 1), 1000); + const window = Number.isFinite(a.sinceMs) ? `since ${new Date(a.sinceMs).toISOString()}` : 'all history'; + return { window, total: errors.length, byCategory, errors: errors.slice(-limit) }; + } + + function getQueueState(args) { + const a = args || {}; + const cfg = loadConfig(); + const pending = cfg.globalSettings && cfg.globalSettings.pendingQueue; + if (!pending || typeof pending !== 'object') { + return { source: 'empty', stale: false, counts: {}, selectedHosters: [] }; + } + const counts = {}; + for (const s of QUEUE_STATUSES) counts[s] = 0; + const jobs = Array.isArray(pending.queueJobs) ? pending.queueJobs : []; + for (const j of jobs) { if (counts[j.status] !== undefined) counts[j.status]++; } + const result = { + source: 'persisted', + stale: true, + savedAt: pending.savedAt || null, + selectedHosters: Array.isArray(pending.selectedUploadHosters) ? pending.selectedUploadHosters : [], + fileCount: Array.isArray(pending.selectedFiles) ? pending.selectedFiles.length : 0, + counts + }; + if (a.includeJobs !== false) { + const maxJobs = Math.min(Math.max(Number(a.maxJobs) || 200, 1), 2000); + result.jobs = _deepRedact(jobs.slice(0, maxJobs).map(j => ({ + file: j.file, fileName: j.fileName, hoster: j.hoster, status: j.status, error: j.error || null + }))); + result.jobsTruncated = jobs.length > maxJobs; + } + return result; + } + + function getHistory(args) { + const a = args || {}; + const history = typeof loadHistory === 'function' + ? (loadHistory() || []) + : (Array.isArray(loadConfig().history) ? loadConfig().history : []); + const limit = Math.min(Math.max(Number(a.limit) || 20, 1), 200); + const perHoster = stats.summarizePerHoster(history); + const recent = [...history].slice(-limit).reverse(); + const secrets = _secrets(); + const batches = recent.map(b => { + const out = { timestamp: b.timestamp || null, fileCount: Array.isArray(b.files) ? b.files.length : 0 }; + if (a.includeFiles) { + out.files = (b.files || []).map(f => ({ + name: f.name || f.fileName || '', + results: (f.results || []).map(r => { + const rr = { hoster: r.hoster, status: r.status }; + if (r.error) rr.error = support.redactLogText(String(r.error), secrets); + if (a.includeUrls && r.url) rr.url = r.url; + return rr; + }) + })); + } + return out; + }); + return { totalBatches: history.length, returned: batches.length, perHoster, batches }; + } + + function getRotationState() { + const cfg = loadConfig(); + return { rotationCursors: _deepRedact(cfg.rotationCursors || {}) }; + } + + function getHealth() { + const cfg = loadConfig(); + const hosters = cfg.hosters && typeof cfg.hosters === 'object' ? Object.keys(cfg.hosters).filter(h => Array.isArray(cfg.hosters[h]) && cfg.hosters[h].length > 0) : []; + return { + reachabilityKnown: false, + hint: 'Live hoster probing (run_health_check) is disabled in this build. Configured hosters with at least one account are listed.', + configuredHosters: hosters + }; + } + + function serverHealth(args) { + const a = args || {}; + const errorLimit = Math.min(Math.max(Number(a.errorLimit) || 20, 1), 200); + const errArgs = Number.isFinite(a.errorSinceMs) ? { sinceMs: a.errorSinceMs, limit: errorLimit } : { limit: errorLimit }; + const errors = listErrors(errArgs); + const queue = getQueueState({ includeJobs: false }); + const history = getHistory({ limit: 5 }); + const warnings = []; + if (queue.source === 'persisted' && queue.stale) warnings.push('queue state is from the persisted snapshot (may lag live state; UploadManager not introspected in this build).'); + if (errors.total > 0) warnings.push(`${errors.total} non-success result(s) in the error window.`); + return { + server: getSystemInfo(), + queue, + recentBatches: history.batches, + perHoster: history.perHoster, + errors, + hosters: getHealth(), + logs: listLogs(), + warnings + }; + } + + return { + getSystemInfo, getConfigRedacted, listLogs, readLog, getAppEvents, + listErrors, getQueueState, getHistory, getRotationState, getHealth, serverHealth, + READABLE_LOGS + }; +} + +module.exports = { createCollectors, READABLE_LOGS }; diff --git a/lib/doodstream-upload.js b/lib/doodstream-upload.js new file mode 100644 index 0000000..1c3821b --- /dev/null +++ b/lib/doodstream-upload.js @@ -0,0 +1,705 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { request } = require('undici'); + +const BASE_URL = 'https://doodstream.com'; +const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const UPLOAD_TIMEOUT = 1800000; // 30 min + +// Cap doodstream's per-hoster debug log alongside the main log files so +// dev-mode sessions don't accumulate gigabytes of upload trace. +const { maybeRotateLogFile } = require('./log-rotation'); +const _DOODSTREAM_LOG_MAX_BYTES = 10 * 1024 * 1024; +const _DOODSTREAM_LOG_MAX_BACKUPS = 1; + +// Resolve the log path at write-time. In a packaged build __dirname lives +// inside app.asar (read-only) — writing there fails silently and we lose every +// production trace. Prefer Electron's writable userData dir, fall back to the +// repo root only when running outside Electron (tests / plain node). +function _doodstreamLogPath() { + try { + const { app } = require('electron'); + if (app && typeof app.getPath === 'function') { + return path.join(app.getPath('userData'), 'doodstream-debug.log'); + } + } catch { /* not running under Electron */ } + return path.join(__dirname, '..', 'doodstream-debug.log'); +} + +let _debugVerbose = false; +function setDebugVerbose(v) { _debugVerbose = !!v; } + +function _debugLog(msg) { + if (!_debugVerbose) return; + try { + const logPath = _doodstreamLogPath(); + maybeRotateLogFile(logPath, _DOODSTREAM_LOG_MAX_BYTES, _DOODSTREAM_LOG_MAX_BACKUPS); + const ts = new Date().toISOString(); + fs.appendFileSync(logPath, `[${ts}] ${msg}\n`); + } catch {} +} + +class DoodstreamUploader { + constructor() { + this.cookies = new Map(); + this.sessId = ''; + this.apiKey = ''; // optionally derived from the logged-in session (deriveApiKey) + } + + _cookieHeader() { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + _parseCookiesFromHeaders(headers) { + let setCookies; + if (typeof headers.getSetCookie === 'function') { + setCookies = headers.getSetCookie(); + } else if (headers['set-cookie']) { + setCookies = Array.isArray(headers['set-cookie']) ? headers['set-cookie'] : [headers['set-cookie']]; + } else { + return; + } + for (const raw of setCookies) { + const pair = raw.split(';')[0]; + const eq = pair.indexOf('='); + if (eq > 0) { + this.cookies.set(pair.substring(0, eq).trim(), pair.substring(eq + 1).trim()); + } + } + } + + async _fetch(url, opts = {}, _redirectCount = 0) { + const MAX_REDIRECTS = 10; + const headers = { + 'User-Agent': USER_AGENT, + ...(opts.headers || {}) + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + // The small discovery/result requests that bookend a multi-minute upload + // occasionally hit a transient blip ("fetch failed", ECONNRESET, a hung TLS + // handshake). A blip here shouldn't throw away the whole upload, so retry a + // few times with short backoff. Each attempt gets its own 20s timeout — + // Node's fetch has none by default, and a hung socket would otherwise stall + // the attempt for minutes. The big file upload (undici) is retried at the + // upload-manager level, not here. + let res; + for (let attempt = 1; ; attempt++) { + const timeoutSignal = AbortSignal.timeout(20000); + const signal = opts.signal ? AbortSignal.any([opts.signal, timeoutSignal]) : timeoutSignal; + try { + res = await fetch(url, { ...opts, headers, redirect: 'manual', signal }); + break; + } catch (err) { + if (opts.signal && opts.signal.aborted) throw err; // caller abort: don't retry + if (attempt >= 3) throw err; + _debugLog(`_fetch transient (${attempt}/3) ${url}: ${err && err.message}; retry`); + await new Promise(r => setTimeout(r, 400 * attempt)); + } + } + + this._parseCookiesFromHeaders(res.headers); + + if ([301, 302, 303, 307, 308].includes(res.status)) { + try { await res.text(); } catch {} + if (_redirectCount >= MAX_REDIRECTS) throw new Error('Zu viele Redirects'); + const location = res.headers.get('location'); + if (location) { + const nextUrl = new URL(location, url).href; + return this._fetch(nextUrl, { ...opts, method: 'GET', body: undefined }, _redirectCount + 1); + } + } + + return res; + } + + /** + * Login to DoodStream via web form + */ + async login(username, password, otp) { + // GET homepage first to collect cookies + const homeRes = await this._fetch(BASE_URL); + await homeRes.text(); + + // POST login via AJAX (op in body, XHR header required for JSON response) + const loginData = new URLSearchParams({ + op: 'login_ajax', + login: username, + password: password, + loginotp: otp || '' + }); + + // Use raw fetch with redirect: 'manual' to detect success redirects + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Referer': BASE_URL + '/', + 'X-Requested-With': 'XMLHttpRequest', + 'User-Agent': USER_AGENT + }; + if (this.cookies.size > 0) { + headers['Cookie'] = this._cookieHeader(); + } + + const res = await fetch(BASE_URL + '/', { + method: 'POST', + body: loginData.toString(), + headers, + redirect: 'manual' + }); + + this._parseCookiesFromHeaders(res.headers); + + // On successful login, server may redirect (3xx) to dashboard + if ([301, 302, 303, 307, 308].includes(res.status)) { + try { await res.text(); } catch {} + // Redirect means login succeeded + } else { + const body = await res.text(); + let json; + try { json = JSON.parse(body); } catch { json = null; } + + if (json && json.status === 'success') { + // Explicit success response + } else if (json && json.message && /otp/i.test(json.message)) { + // OTP required — signal caller to collect OTP from user + const err = new Error(`Doodstream Login: ${json.message}`); + err.otpRequired = true; + throw err; + } else if (json && json.status === 'fail') { + throw new Error(`Doodstream Login: ${json.message || 'Login fehlgeschlagen'}`); + } else if (body.includes('Dashboard')) { + // Got dashboard HTML directly — login worked + } else { + const msg = (json && json.message) || 'Login fehlgeschlagen'; + throw new Error(`Doodstream Login: ${msg}`); + } + } + + // Extract sess_id from the upload page + await this._extractSessId(); + } + + async _extractSessId() { + const res = await this._fetch(BASE_URL + '/?op=upload'); + const html = await res.text(); + + // Hidden input: + const hiddenMatch = html.match(/name=["']sess_id["'][^>]*value=["']([a-zA-Z0-9]+)["']/); + if (hiddenMatch) { + this.sessId = hiddenMatch[1]; + return; + } + + // Vue component prop or JS: sess_id: "xxx" or sess_id="xxx" + const sessMatch = html.match(/sess_id['":\s]+['"]([a-zA-Z0-9]+)['"]/); + if (sessMatch) { + this.sessId = sessMatch[1]; + return; + } + + // Assignment: sess_id = 'xxx' + const altMatch = html.match(/sess_id\s*=\s*['"]([a-zA-Z0-9]+)['"]/); + if (altMatch) { + this.sessId = altMatch[1]; + return; + } + + throw new Error('Doodstream: sess_id nicht gefunden nach Login'); + } + + /** + * Get upload server URL from web interface + */ + async _getUploadServer() { + // Use the standard upload server endpoint + const res = await this._fetch(BASE_URL + '/?op=upload_server'); + const text = await res.text(); + const ctype = (res.headers && res.headers.get) ? (res.headers.get('content-type') || '') : ''; + _debugLog(`upload_server: status=${res.status} ctype=${ctype} body(800)=${(text || '').slice(0, 800)}`); + let json; + try { json = JSON.parse(text); } catch { json = null; } + + if (json && json.result && /^https?:\/\//i.test(json.result)) { + return json.result; + } + + // Fallback: try fetching from upload page HTML + const pageRes = await this._fetch(BASE_URL + '/?op=upload'); + const html = await pageRes.text(); + + // Current doodstream format: the upload server is the action of the + // multipart upload form, e.g. + //