Release v2.1.65: safe MCP updates and measured upload scheduling
CI / verify (push) Canceled after 0s

This commit is contained in:
Sucukdeluxe committed 2026-10-16 05:36:42 +02:00
1 parent e3c03d195e
commit 1ff552fc40
26 files changed
+552 -92

No files matched your search

+3 -3
View File
@@ -1,4 +1,5 @@
const fs = require('fs');
const { statUploadFile, readUploadFile } = require('./upload-file-io');
const path = require('path');
const crypto = require('crypto');
const { request, Agent } = require('undici');
@@ -67,7 +68,7 @@ class ClouddropUploader {
if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt');
const fileName = path.basename(filePath);
let fileSize = 0;
try { fileSize = fs.statSync(filePath).size; }
try { fileSize = (await statUploadFile(filePath)).size; }
catch { throw new Error(`Clouddrop: Datei nicht lesbar: ${fileName}`); }
if (fileSize <= 0) throw new Error('Clouddrop: Datei ist leer');
@@ -105,8 +106,7 @@ class ClouddropUploader {
let bytesRead = 0;
async function* generate() {
yield preambleBuf;
const fileStream = fs.createReadStream(filePath, { highWaterMark: 1024 * 1024 });
for await (const chunk of fileStream) {
for await (const chunk of readUploadFile(filePath, signal)) {
if (signal && signal.aborted) throw new Error('Aborted');
if (throttle) await throttle.consume(chunk.length, signal);
bytesRead += chunk.length;
+3 -4
View File
@@ -1,4 +1,5 @@
const fs = require('fs');
const { statUploadFile, readUploadFile } = require('./upload-file-io');
const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
@@ -383,7 +384,7 @@ class DoodstreamUploader {
*/
async upload(filePath, progressCb, signal, throttle) {
const fileName = path.basename(filePath);
const fileSize = fs.statSync(filePath).size;
const fileSize = (await statUploadFile(filePath)).size;
// Get upload server
const uploadUrl = await this._getUploadServer();
@@ -413,13 +414,11 @@ class DoodstreamUploader {
const epilogueBuf = Buffer.from(epilogue, 'utf-8');
const totalSize = preambleBuf.length + fileSize + epilogueBuf.length;
const CHUNK_SIZE = 1024 * 1024;
let bytesRead = 0;
async function* generate() {
yield preambleBuf;
const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE });
for await (const chunk of fileStream) {
for await (const chunk of readUploadFile(filePath, signal)) {
if (signal && signal.aborted) throw new Error('Aborted');
if (throttle) await throttle.consume(chunk.length, signal);
bytesRead += chunk.length;
+7 -9
View File
@@ -1,4 +1,4 @@
const fs = require('fs');
const { statUploadFile, readUploadFile } = require('./upload-file-io');
const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
@@ -259,10 +259,10 @@ function parseByseResult(payload) {
// --- Multipart upload with progress ---
function buildMultipart(filePath, formFields) {
async function buildMultipart(filePath, formFields) {
const boundary = '----FormBoundary' + crypto.randomBytes(16).toString('hex');
const fileName = path.basename(filePath);
const fileSize = fs.statSync(filePath).size;
const fileSize = (await statUploadFile(filePath)).size;
let preamble = '';
for (const [key, value] of Object.entries(formFields)) {
@@ -284,16 +284,14 @@ function buildMultipart(filePath, formFields) {
return { boundary, preambleBuf, epilogueBuf, totalSize, fileSize };
}
function createUploadBody(filePath, formFields, onProgress, throttle, signal) {
const { boundary, preambleBuf, epilogueBuf, totalSize, fileSize } = buildMultipart(filePath, formFields);
async function createUploadBody(filePath, formFields, onProgress, throttle, signal) {
const { boundary, preambleBuf, epilogueBuf, totalSize, fileSize } = await buildMultipart(filePath, formFields);
let bytesRead = 0;
const CHUNK_SIZE = 1024 * 1024;
async function* generate() {
yield preambleBuf;
const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE });
for await (const chunk of fileStream) {
for await (const chunk of readUploadFile(filePath, signal)) {
if (signal && signal.aborted) throw new Error('Aborted');
if (throttle) await throttle.consume(chunk.length, signal);
bytesRead += chunk.length;
@@ -558,7 +556,7 @@ async function uploadFile(hosterName, filePath, apiKey, onProgress, signal, thro
const targetUrl = config.buildUploadUrl(uploadUrl, apiKey);
const formFields = config.formFields(apiKey);
const { iterable, boundary, totalSize } = createUploadBody(filePath, formFields, onProgress, throttle, signal);
const { iterable, boundary, totalSize } = await createUploadBody(filePath, formFields, onProgress, throttle, signal);
const { body, statusCode, headers } = await request(targetUrl, {
method: 'POST',
+4 -2
View File
@@ -13,7 +13,7 @@ function page(items, args) {
}
function createMcpApp({ loadConfig, invoke, invokeGuarded = (_context, channel, ...args) => invoke(channel, ...args), renderer, collectors, getGrants, trashFile, canWrite,
getCapabilities, getProtectedPaths, getInstallationId, getDiagnosticSources, getRuntimeDiagnostics = () => null }) {
getCapabilities, getProtectedPaths, getInstallationId, getDiagnosticSources, getRuntimeDiagnostics = () => null, getPerformance = () => null, updates }) {
const transfers = createMcpTransfers({ getGrants, getCapabilities, getProtectedPaths, getInstallationId });
const sanitize = createMcpSanitizer(loadConfig);
const guard = context => {
@@ -57,6 +57,7 @@ function createMcpApp({ loadConfig, invoke, invokeGuarded = (_context, channel,
...transfers.handlers,
...createMcpDiagnostics({ getSources: getDiagnosticSources, redact: text => sanitize(redactLogText(text, collectSecretValues(loadConfig()))) }),
mhu_app_info: () => invoke('get-app-info'),
mhu_performance_get: () => getPerformance(),
mhu_app_state: async (args, context) => ({ ...(await renderer('state', {}, context)), automation: await invoke('automation:get-status') }),
mhu_queue_list: (args, context) => renderer('queue-list', args, context),
mhu_queue_job_log: async args => page(await invoke('get-job-log', args.jobId), args),
@@ -155,7 +156,8 @@ function createMcpApp({ loadConfig, invoke, invokeGuarded = (_context, channel,
mhu_backup_delete: args => invoke('online-backup:delete-managed', args.id),
mhu_backup_restore: async (args, context) => { const result = await invokeGuarded(context, 'online-backup:restore', args.key); if (!result.ok) throw new McpAccessError('RESTORE_FAILED'); await refresh(context); return { ok: true, warnings: result.warnings }; },
mhu_update_check: () => invoke('app:check-updates', { forceRefresh: true }),
mhu_update_install: (args, context) => invokeGuarded(context, 'app:install-update'),
mhu_update_install: (args, context) => updates.start(args, context),
mhu_update_status: (args, context) => updates.status(args, context),
mhu_app_restart: () => invoke('app:restart'),
mhu_app_quit: () => invoke('app:quit'),
mhu_secrets_read: args => { const { password, apiKey } = find(args.accountId); return { accountId: args.accountId, password: password || '', apiKey: apiKey || '' }; }
+1 -1
View File
@@ -86,7 +86,7 @@ class McpService {
}
configure(args) {
validateInput(args, { type: 'object', additionalProperties: false, required: ['enabled', 'name', 'capabilities'], properties: {
enabled: { type: 'boolean' }, name: { type: 'string', minLength: 1, maxLength: 100 }, capabilities: { type: 'array', maxItems: 4, items: { type: 'string', enum: CAPABILITIES } }
enabled: { type: 'boolean' }, name: { type: 'string', minLength: 1, maxLength: 100 }, capabilities: { type: 'array', maxItems: 5, items: { type: 'string', enum: CAPABILITIES } }
} });
if (!args.name.trim()) throw new McpAccessError('INVALID_ARGUMENTS');
this.active = false;
+1 -1
View File
@@ -4,7 +4,7 @@ const { randomUUID, randomBytes } = require('node:crypto');
const { McpAccessError } = require('./mcp-access');
const RELAY_URL = 'https://downloader.24-music.de/mcp-relay';
const CAPABILITIES = ['read', 'control', 'files', 'secrets'];
const CAPABILITIES = ['read', 'control', 'files', 'secrets', 'updates'];
class McpStore {
constructor({ directory, encrypt, decrypt }) {
+3 -1
View File
@@ -17,6 +17,7 @@ const file = { rootId: id, relativePath: text() };
const definitions = [
['app_info', 'Read application version and runtime information.', 'read', {}, []],
['performance_get', 'Read bounded in-memory main-thread, renderer, CPU, memory, I/O and transfer-slot measurements. Missing samples do not prove a freeze.', 'read', {}, []],
['app_state', 'Read live transfer and automation summary.', 'read', {}, []],
['queue_list', 'Read a bounded page of upload jobs.', 'read', { ...page, status: text(40) }, []],
['queue_job_log', 'Read a bounded, secret-filtered log for one upload job.', 'read', { jobId: id, ...page }, ['jobId']],
@@ -94,7 +95,8 @@ const definitions = [
['backup_delete', 'Delete a selected managed online backup.', 'backup', { id }, ['id'], true],
['backup_restore', 'Restore settings from an explicitly supplied online key.', 'backup', { key: text(200) }, ['key'], true],
['update_check', 'Check for an application update without installing it.', 'read', {}, []],
['update_install', 'Install an available application update.', 'lifecycle', {}, [], true],
['update_install', 'Submit an idempotent official installer update for one exact version. Requires separate local updates permission. Poll update_status; completion requires restart and reconnect.', 'updates', { targetVersion: { type: 'string', pattern: '^\\d+\\.\\d+\\.\\d+$', maxLength: 40 }, requestKey: id }, ['targetVersion', 'requestKey']],
['update_status', 'Read an owned persistent update job, including progress and post-restart version verification.', 'read', { jobId: id }, []],
['app_restart', 'Restart the application after persisting its state.', 'lifecycle', {}, [], true],
['app_quit', 'Close the application after persisting its state.', 'lifecycle', {}, [], true],
['secrets_read', 'Explicitly reveal credentials for one account; disabled by default.', 'secrets.read', account, ['accountId']]
+165
View File
@@ -0,0 +1,165 @@
const fs = require('node:fs/promises');
const path = require('node:path');
const { createHash, randomUUID } = require('node:crypto');
const { McpAccessError } = require('./mcp-access');
const VERSION = /^\d+\.\d+\.\d+$/;
const TERMINAL = new Set(['completed', 'failed']);
class McpUpdate {
constructor(options) {
this.options = options;
this.file = path.join(options.directory, 'mcp-update-jobs.json');
this.jobs = [];
this.writes = Promise.resolve();
this.task = null;
this.resumeTask = null;
this.ready = false;
}
async load() {
try {
const stat = await fs.lstat(this.file);
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 16 * 1024 * 1024) throw new Error('invalid');
const value = JSON.parse(await fs.readFile(this.file, 'utf8'));
if (value.version !== 1 || !Array.isArray(value.jobs) || value.jobs.length > 20
|| value.jobs.some(job => !job || !VERSION.test(job.targetVersion) || !/^[a-f0-9]{64}$/.test(job.owner) || !/^[A-Za-z0-9_.:-]{1,128}$/.test(job.requestKey))) throw new Error('invalid');
this.jobs = value.jobs;
} catch (error) {
if (error.code !== 'ENOENT') throw new McpAccessError('UPDATE_STORE_UNAVAILABLE');
}
this.ready = true;
}
async save() {
const data = JSON.stringify({ version: 1, jobs: this.jobs });
if (Buffer.byteLength(data) > 16 * 1024 * 1024) throw new McpAccessError('UPDATE_STORE_LIMIT');
const operation = this.writes.then(async () => {
await fs.mkdir(path.dirname(this.file), { recursive: true });
const temporary = this.file + '.' + randomUUID() + '.tmp';
let handle;
try {
handle = await fs.open(temporary, 'wx', 0o600);
await handle.writeFile(data);
await handle.sync();
await handle.close(); handle = null;
await fs.rename(temporary, this.file);
} finally {
await handle?.close();
await fs.unlink(temporary).catch(() => {});
}
});
this.writes = operation.catch(() => {});
return operation;
}
owner(context) { return createHash('sha256').update(context.principalId).digest('hex'); }
active() { return this.jobs.find(job => !TERMINAL.has(job.phase)); }
blocksTransfers() { const job = this.active(); return !!job && job.phase !== 'awaiting_reconnect'; }
public(job) {
if (!job) return null;
const { owner: _owner, requestKey: _key, resume: _resume, ...value } = job;
return value;
}
status(args, context) {
const job = args.jobId ? this.jobs.find(item => item.id === args.jobId) : this.jobs.at(-1);
if (job && job.owner !== this.owner(context)) throw new McpAccessError('UPDATE_NOT_FOUND');
return { job: this.public(job) };
}
assertAllowed() {
if (!this.ready) throw new McpAccessError('UPDATE_STORE_UNAVAILABLE');
if (!this.options.canUpdate()) throw new McpAccessError('SCOPE_DENIED');
}
async phase(job, phase, patch = {}) {
Object.assign(job, patch, { phase, updatedAt: Date.now() });
await this.save();
}
async start({ targetVersion, requestKey }, context) {
this.assertAllowed();
if (!VERSION.test(targetVersion)) throw new McpAccessError('INVALID_ARGUMENTS');
const owner = this.owner(context);
const existing = this.jobs.find(job => job.owner === owner && job.requestKey === requestKey);
if (existing) {
if (existing.targetVersion !== targetVersion) throw new McpAccessError('REQUEST_CONFLICT');
return this.public(existing);
}
if (this.active() || this.task) throw new McpAccessError('UPDATE_BUSY');
if (!await this.options.supported()) throw new McpAccessError('UPDATE_INSTALLATION_UNSUPPORTED');
this.assertAllowed();
if (this.active()) throw new McpAccessError('UPDATE_BUSY');
const job = { id: randomUUID(), owner, requestKey, targetVersion, sourceVersion: this.options.version(), phase: 'accepted', createdAt: Date.now(), updatedAt: Date.now() };
const previous = this.jobs;
this.jobs = [...this.jobs.slice(-19), job];
try { await this.save(); }
catch (error) { this.jobs = previous; throw error; }
this.task = new Promise(resolve => setImmediate(resolve)).then(() => this.execute(job)).finally(() => { this.task = null; });
return this.public(job);
}
async execute(job) {
let parked = false;
try {
this.assertAllowed();
await this.phase(job, 'checking');
const release = await this.options.check();
if (!release?.available || release.remoteVersion !== job.targetVersion) throw new McpAccessError('UPDATE_VERSION_MISMATCH');
this.assertAllowed();
await this.phase(job, 'downloading');
const prepared = await this.options.prepare(release, progress => {
job.progress = { stage: progress.stage, percent: progress.percent, bytesDownloaded: progress.bytesDownloaded, bytesTotal: progress.bytesTotal };
});
this.assertAllowed();
if (prepared.remoteVersion !== job.targetVersion || !prepared.sha512Verified || !prepared.sha512) throw new McpAccessError('UPDATE_INTEGRITY_REQUIRED');
await this.phase(job, 'waiting_uploads', { sha512: prepared.sha512 });
job.resume = await this.options.park();
await this.save();
parked = true;
await this.options.drain(() => this.assertAllowed());
this.assertAllowed();
await this.options.verify(prepared);
this.assertAllowed();
await this.phase(job, 'quiescing');
await this.options.handoff(prepared, job);
} catch (error) {
const code = error instanceof McpAccessError ? error.code : 'UPDATE_FAILED';
try {
if (parked && job.resume) await this.options.restore(job.resume);
await this.phase(job, 'failed', { errorCode: code, resume: null });
} catch { job.phase = 'failed'; job.errorCode = 'UPDATE_RECOVERY_FAILED'; }
}
}
async markLaunching() {
const job = this.active();
if (!job || job.phase !== 'quiescing') throw new McpAccessError('UPDATE_STATE_INVALID');
this.assertAllowed();
await this.phase(job, 'awaiting_restart');
}
async launchFailed() {
const job = this.active();
if (job) await this.phase(job, 'failed', { errorCode: 'UPDATE_LAUNCH_FAILED' });
}
async recover({ rendererReady, connected }) {
if (!this.ready || this.task || this.resumeTask || !rendererReady) return;
const job = this.active();
if (!job) return;
this.resumeTask = (async () => {
if (job.phase === 'awaiting_reconnect') {
if (connected) await this.phase(job, 'completed', { completedAt: Date.now(), reconnectedAt: Date.now() });
return;
}
if (job.phase !== 'awaiting_restart' && job.phase !== 'resuming') {
if (job.resume) await this.options.restore(job.resume);
await this.phase(job, 'failed', { errorCode: 'UPDATE_INTERRUPTED', resume: null });
return;
}
if (this.options.version() !== job.targetVersion) {
if (job.resume) await this.options.restore(job.resume);
await this.phase(job, 'failed', { errorCode: 'UPDATE_VERSION_NOT_INSTALLED', installedVersion: this.options.version(), resume: null });
return;
}
await this.phase(job, 'resuming', { installedVersion: this.options.version() });
if (job.resume) await this.options.restore(job.resume);
await this.phase(job, connected ? 'completed' : 'awaiting_reconnect', { restoredAt: Date.now(), resume: null,
...(connected ? { completedAt: Date.now(), reconnectedAt: Date.now() } : {}) });
})().catch(async () => { await this.phase(job, 'failed', { errorCode: 'UPDATE_RECOVERY_FAILED' }).catch(() => {}); }).finally(() => { this.resumeTask = null; });
return this.resumeTask;
}
}
module.exports = { McpUpdate };
+88
View File
@@ -0,0 +1,88 @@
const { performance } = require('node:perf_hooks');
const { randomUUID } = require('node:crypto');
class PerformanceMonitor {
constructor({ now = Date.now, monotonic = () => performance.now(), cpu = () => process.cpuUsage(), memory = () => process.memoryUsage(), limit = 600 } = {}) {
this.now = now;
this.monotonic = monotonic;
this.cpu = cpu;
this.memory = memory;
this.limit = limit;
this.bootId = randomUUID();
this.startedAt = now();
this.samples = [];
this.events = [];
this.operations = new Map();
this.pendingRenderer = new Map();
this.sequence = 0;
this.timer = null;
}
record(name, durationMs, failed = false) {
if (!/^[a-z][a-z0-9_.:-]{0,79}$/.test(name) || !Number.isFinite(durationMs) || durationMs < 0) return;
if (!this.operations.has(name) && this.operations.size >= 128) return;
const item = this.operations.get(name) || { name, count: 0, failed: 0, totalMs: 0, maxMs: 0, active: 0 };
item.count++;
item.failed += Number(failed);
item.totalMs += durationMs;
item.maxMs = Math.max(item.maxMs, durationMs);
item.lastMs = durationMs;
item.lastAt = this.now();
this.operations.set(name, item);
if (durationMs >= 1000) this.event('operation', { name, durationMs, failed });
}
async measure(name, operation) {
const started = this.monotonic();
let failed = false;
try { return await operation(); }
catch (error) { failed = true; throw error; }
finally { this.record(name, this.monotonic() - started, failed); }
}
event(type, details = {}) {
this.events.push({ id: ++this.sequence, at: this.now(), type, ...details });
if (this.events.length > this.limit) this.events.splice(0, this.events.length - this.limit);
}
rendererReply(id) {
const started = this.pendingRenderer.get(id);
if (started === undefined) return;
this.pendingRenderer.delete(id);
const durationMs = this.monotonic() - started;
this.record('renderer.roundtrip', durationMs);
}
start({ version, getTransfers = () => null, pingRenderer = () => false } = {}) {
if (this.timer) return;
this.version = version;
let previous = this.monotonic();
let previousCpu = this.cpu();
this.timer = setInterval(() => {
const current = this.monotonic();
const elapsed = current - previous;
const currentCpu = this.cpu();
const delayMs = Math.max(0, elapsed - 1000);
const sample = { at: this.now(), elapsedMs: elapsed, delayMs, cpuPercent: elapsed > 0 ? (currentCpu.user + currentCpu.system - previousCpu.user - previousCpu.system) / (elapsed * 10) : 0,
memory: this.memory(), transfers: getTransfers() };
this.samples.push(sample);
if (this.samples.length > this.limit) this.samples.shift();
if (delayMs > 30000) this.event('measurement-gap', { durationMs: delayMs });
else if (delayMs >= 1000) this.event('main-delay', { durationMs: delayMs });
for (const [id, started] of this.pendingRenderer) if (current - started >= 5000) {
this.pendingRenderer.delete(id);
this.event('renderer-unanswered', { durationMs: current - started });
}
if (!this.pendingRenderer.size) {
const id = ++this.sequence;
if (pingRenderer(id)) this.pendingRenderer.set(id, current);
}
previous = current;
previousCpu = currentCpu;
}, 1000);
this.timer.unref?.();
}
stop() { clearInterval(this.timer); this.timer = null; this.pendingRenderer.clear(); }
snapshot() {
return { bootId: this.bootId, version: this.version || null, startedAt: this.startedAt, sampledAt: this.now(), running: !!this.timer,
samples: this.samples.slice(-300), events: this.events.slice(), operations: [...this.operations.values()].map(item => ({ ...item })) };
}
}
const runtimePerformance = new PerformanceMonitor();
module.exports = { PerformanceMonitor, runtimePerformance };
+3 -3
View File
@@ -46,12 +46,12 @@ class Semaphore {
}
release() {
if (this.queue.length > 0) {
this.active = Math.max(0, this.active - 1);
if (this.active < this.limit && this.queue.length > 0) {
const entry = this.queue.shift();
this._cleanupEntry(entry);
this.active++;
entry.resolve();
} else {
this.active = Math.max(0, this.active - 1);
}
}
+53 -22
View File
@@ -232,12 +232,6 @@ async function parseLatestYml(url, expected = {}, fetchImpl = fetch) {
}
}
function verifyExeHeader(buf) {
// Check MZ header
if (buf.length < 128 * 1024) return false;
return buf[0] === 0x4D && buf[1] === 0x5A; // 'MZ'
}
async function prepareUpdate(onProgress, options = {}) {
if (activeAbort) activeAbort.abort();
const controller = new AbortController();
@@ -246,6 +240,7 @@ async function prepareUpdate(onProgress, options = {}) {
const fetchImpl = options.fetchImpl || fetch;
const now = typeof options.now === 'function' ? options.now : Date.now;
let stagedInstallerPath = '';
let downloadDirectory = '';
try {
// Stage: starting
@@ -273,9 +268,12 @@ async function prepareUpdate(onProgress, options = {}) {
// Stage: downloading
const tmpDir = options.tempDir || app.getPath('temp');
const installerPath = path.join(tmpDir, check.assetName);
downloadDirectory = await fs.promises.mkdtemp(path.join(tmpDir, 'mhu-update-'));
const installerPath = path.join(downloadDirectory, path.basename(check.assetName));
stagedInstallerPath = installerPath;
let fileBuffer;
let actualSha;
let firstBytes;
for (let attempt = 1; attempt <= 3; attempt++) {
signal.throwIfAborted();
const downloadController = new AbortController();
@@ -307,7 +305,10 @@ async function prepareUpdate(onProgress, options = {}) {
let downloadedBytes = 0;
let lastReportedPercent = -1;
const downloadStartedAt = now();
const chunks = [];
const output = await fs.promises.open(installerPath, 'w', 0o600);
const hash = crypto.createHash('sha512');
firstBytes = Buffer.alloc(0);
try {
if (attempt > 1 && onProgress) onProgress({ stage: 'downloading', percent: 0, bytesDownloaded: 0, bytesTotal: manifest.size, bytesPerSecond: 0, etaSeconds: null, attempt });
while (true) {
signal.throwIfAborted();
@@ -317,7 +318,9 @@ async function prepareUpdate(onProgress, options = {}) {
if (done) break;
downloadedBytes += value.length;
if (downloadedBytes > manifest.size) throw new Error('Heruntergeladene Datei hat eine abweichende Größe');
chunks.push(value);
if (firstBytes.length < 2) firstBytes = Buffer.concat([firstBytes, Buffer.from(value).subarray(0, 2 - firstBytes.length)]);
hash.update(value);
await output.writeFile(value);
const percent = Math.floor((downloadedBytes / manifest.size) * 100);
if (onProgress && percent !== lastReportedPercent) {
const elapsedMs = Math.max(1, now() - downloadStartedAt);
@@ -334,7 +337,9 @@ async function prepareUpdate(onProgress, options = {}) {
error.retryable = true;
throw error;
}
fileBuffer = Buffer.concat(chunks);
await output.sync();
actualSha = hash.digest('base64');
} finally { await output.close(); }
} catch (error) {
signal.throwIfAborted();
retry = timedOut || error.retryable === true || ['UND_ERR_SOCKET', 'ECONNRESET', 'ETIMEDOUT', 'EPIPE', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_BODY_TIMEOUT'].includes(error.cause?.code || error.code) || /^(terminated|fetch failed)$/i.test(error.message);
@@ -356,36 +361,35 @@ async function prepareUpdate(onProgress, options = {}) {
// Stage: verifying
if (onProgress) onProgress({ stage: 'verifying', percent: 0 });
if (!verifyExeHeader(fileBuffer)) {
if (manifest.size < 128 * 1024 || firstBytes[0] !== 0x4d || firstBytes[1] !== 0x5a) {
throw new Error('Heruntergeladene Datei ist keine gültige EXE');
}
const actualSha = crypto.createHash('sha512').update(fileBuffer).digest('base64');
if (actualSha !== expectedSha) {
const actualHex = crypto.createHash('sha512').update(fileBuffer).digest('hex');
const actualHex = Buffer.from(actualSha, 'base64').toString('hex');
if (actualHex !== expectedSha.toLowerCase()) {
throw new Error('SHA-512 Prüfung fehlgeschlagen');
}
}
stagedInstallerPath = path.join(tmpDir, `${path.basename(check.assetName)}.${crypto.randomUUID()}.download`);
fs.writeFileSync(stagedInstallerPath, fileBuffer);
fs.rmSync(installerPath, { force: true });
fs.renameSync(stagedInstallerPath, installerPath);
stagedInstallerPath = '';
const prepared = {
installerPath,
sha512: actualSha,
size: manifest.size,
sha512Verified: true,
assetName: check.assetName,
remoteVersion: check.remoteVersion || '',
transportTag: check.transportTag || ''
};
await verifyPreparedUpdate(prepared);
stagedInstallerPath = '';
if (onProgress) onProgress({ stage: 'prepared', percent: 100 });
return prepared;
} catch (err) {
const failure = signal.aborted ? new Error('Update abgebrochen') : err;
if (stagedInstallerPath) fs.rmSync(stagedInstallerPath, { force: true });
if (stagedInstallerPath) await fs.promises.unlink(stagedInstallerPath).catch(() => {});
if (downloadDirectory) await fs.promises.rmdir(downloadDirectory).catch(() => {});
if (onProgress) onProgress({ stage: signal.aborted ? 'aborted' : 'error', error: failure.message });
throw failure;
} finally {
@@ -393,6 +397,32 @@ async function prepareUpdate(onProgress, options = {}) {
}
}
async function verifyPreparedUpdate(prepared) {
if (!prepared?.sha512Verified || !prepared.sha512 || !Number.isSafeInteger(prepared.size)) throw new Error('Update-Prüfsumme fehlt');
const stat = await fs.promises.lstat(prepared.installerPath);
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size !== prepared.size) throw new Error('Update-Datei wurde verändert');
const handle = await fs.promises.open(prepared.installerPath, 'r');
try {
const hash = crypto.createHash('sha512');
const before = await handle.stat();
if (before.ino !== stat.ino || before.dev !== stat.dev || before.size !== stat.size) throw new Error('Update-Datei wurde ausgetauscht');
for await (const chunk of handle.createReadStream({ autoClose: false })) hash.update(chunk);
const after = await handle.stat();
if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs || hash.digest('base64') !== prepared.sha512) throw new Error('SHA-512 Prüfung fehlgeschlagen');
} finally { await handle.close(); }
return true;
}
async function launchVerifiedUpdate(prepared, directory, options = {}) {
await verifyPreparedUpdate(prepared);
const spawn = options.spawnImpl || require('node:child_process').spawn;
const child = spawn(prepared.installerPath, ['/S', '--updated', '--force-run', `/D=${directory}`], { detached: true, windowsHide: true, stdio: 'ignore' });
await new Promise((resolve, reject) => { child.once('error', reject); child.once('spawn', resolve); });
child.unref();
launchedInstallerPaths.add(path.resolve(prepared.installerPath).toLowerCase());
return true;
}
function launchPreparedUpdate(prepared, options = {}) {
const installerPath = prepared && typeof prepared.installerPath === 'string' ? prepared.installerPath : '';
if (!installerPath) throw new Error('Vorbereitetes Update ist unvollständig');
@@ -403,6 +433,7 @@ function launchPreparedUpdate(prepared, options = {}) {
try {
spawnImpl(installerPath, ['/S', '--updated', '--force-run'], {
detached: true,
windowsHide: true,
stdio: 'ignore'
}).unref();
return true;
@@ -418,4 +449,4 @@ function abortUpdate() {
return true;
}
module.exports = { checkForUpdate, fetchGithubReleaseNotes, prepareUpdate, launchPreparedUpdate, abortUpdate, isNewer, resolveReleaseVersion, pickSetupAsset, parseLatestYml, createUpdateAnnouncementState };
module.exports = { checkForUpdate, fetchGithubReleaseNotes, prepareUpdate, verifyPreparedUpdate, launchVerifiedUpdate, launchPreparedUpdate, abortUpdate, isNewer, resolveReleaseVersion, pickSetupAsset, parseLatestYml, createUpdateAnnouncementState };
+23
View File
@@ -0,0 +1,23 @@
const fs = require('node:fs');
const { runtimePerformance } = require('./performance-monitor');
const statUploadFile = file => runtimePerformance.measure('file.stat', () => fs.promises.stat(file));
async function* readUploadFile(file, signal) {
signal?.throwIfAborted();
const stream = fs.createReadStream(file, { highWaterMark: 1024 * 1024, ...(signal ? { signal } : {}) });
const iterator = stream[Symbol.asyncIterator]();
try {
for (;;) {
const result = await runtimePerformance.measure('file.read', () => iterator.next());
if (result.done) break;
signal?.throwIfAborted();
yield result.value;
}
} finally {
stream.destroy();
await iterator.return?.();
}
}
module.exports = { statUploadFile, readUploadFile };
+41 -22
View File
@@ -10,6 +10,7 @@ const DoodstreamUploader = require('./doodstream-upload');
const ClouddropUploader = require('./clouddrop-upload');
const Semaphore = require('./semaphore');
const Throttle = require('./throttle');
const { runtimePerformance } = require('./performance-monitor');
const { probeFileHead } = require('./file-probe');
const { normalizeFailureDetails } = require('./upload-diagnostics');
const { classifyAccountFailure, selectFallbackAccount } = require('./account-rotation');
@@ -47,6 +48,7 @@ class UploadManager extends EventEmitter {
this.pendingCancelledJobIds = new Set();
this.pendingCancelAll = false;
this.sessionBytes = 0;
this.transferredBytes = 0;
this._transientErrorTotal = 0;
this.lastStartTime = {}; // hoster -> timestamp of last upload start
this.intervalLocks = {}; // hoster -> Promise chain for serialized interval waits
@@ -115,13 +117,28 @@ class UploadManager extends EventEmitter {
getDiagnostics() {
const activeByHoster = {};
const phases = {};
for (const v of this.activeJobs.values()) {
const h = v && v.hoster ? v.hoster : 'unknown';
activeByHoster[h] = (activeByHoster[h] || 0) + 1;
const phase = v.phase || 'getting-server';
phases[phase] = (phases[phase] || 0) + 1;
}
let pending = 0;
for (const sem of Object.values(this.semaphores)) pending += (sem && sem.pending) || 0;
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size };
return { activeByHoster, transientErrors: this._transientErrorTotal, pending, active: this.activeJobs.size,
globalSlots: this.globalSemaphore ? { active: this.globalSemaphore.active, pending: this.globalSemaphore.pending, limit: this.globalSemaphore.limit } : null,
hosterSlots: Object.fromEntries(Object.entries(this.semaphores).map(([hoster, sem]) => [hoster, { active: sem.active, pending: sem.pending, limit: sem.limit }])),
sessionBytes: this.sessionBytes, transferredBytes: this.transferredBytes, batchStartedAt: this.startTime,
speedKbs: [...this.activeJobs.values()].reduce((sum, job) => sum + (job.speedKbs || 0), 0), phases };
}
_recordTransferred(entry, bytes, total) {
if (!Number.isFinite(bytes) || bytes < 0) return;
const next = Math.min(bytes, total);
this.transferredBytes += Math.max(0, next - entry.bytesUploaded);
entry.bytesUploaded = next;
entry.phase = next >= total ? 'awaiting-confirmation' : 'uploading';
}
clearFailedAccount(hoster, accountId) {
@@ -368,6 +385,7 @@ class UploadManager extends EventEmitter {
if (pendingCancelAll) this.abortController.abort();
this.startTime = Date.now();
this.sessionBytes = 0;
this.transferredBytes = 0;
this.activeJobs.clear();
this.jobAbortControllers.clear();
this.cancelledJobIds.clear();
@@ -478,6 +496,7 @@ class UploadManager extends EventEmitter {
} else {
try { fileSize = (await fs.promises.stat(task.file)).size; } catch { fileNotFound = true; }
}
if (cachedResult && fileSize > 0) cachedResult.size = fileSize;
const maxAttempts = Math.max(1, (settings.retries || 0) + 1);
const jobAbortController = new AbortController();
@@ -502,6 +521,7 @@ class UploadManager extends EventEmitter {
if (finalResultRecorded) return;
finalResultRecorded = true;
finalStatus = status;
if (status === 'error') runtimePerformance.event('upload-error', { hoster: task.hoster });
const result = {
jobId,
@@ -573,7 +593,7 @@ class UploadManager extends EventEmitter {
// queueJobs array; the first event it actually needs from main is the
// 'getting-server' / 'uploading' transition for the jobs that the
// semaphore lets through.
await hosterSemaphore.acquire(admissionSignal);
await runtimePerformance.measure('slots.hoster', () => hosterSemaphore.acquire(admissionSignal));
hosterSlotAcquired = true;
if (this.stopAfterActive) {
@@ -585,7 +605,7 @@ class UploadManager extends EventEmitter {
let fileProbe = null;
try {
fileProbe = await probeFileHead(task.file, 512);
fileProbe = await runtimePerformance.measure('file.probe', () => probeFileHead(task.file, 512));
} catch (err) {
fileProbe = { ok: false, error: err && err.message, kind: 'unreadable' };
}
@@ -597,16 +617,15 @@ class UploadManager extends EventEmitter {
headHex: fileProbe && fileProbe.headHex ? fileProbe.headHex.slice(0, 32) : null
});
if (globalSemaphore) {
await globalSemaphore.acquire(admissionSignal);
globalSlotAcquired = true;
}
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
const acquireGlobal = async () => {
if (globalSemaphore) {
await runtimePerformance.measure('slots.global', () => globalSemaphore.acquire(admissionSignal));
globalSlotAcquired = true;
}
};
if (settings.timeIntervalSec > 0) {
await this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal);
}
await runtimePerformance.measure('slots.interval', () => this._waitForInterval(task.hoster, settings.timeIntervalSec * 1000, admissionSignal, acquireGlobal));
} else await acquireGlobal();
if (this.stopAfterActive) throw new Error('Warteschlange angehalten');
@@ -746,7 +765,7 @@ class UploadManager extends EventEmitter {
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
@@ -1080,7 +1099,7 @@ class UploadManager extends EventEmitter {
lastSpeedTime = now;
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
const elapsed = Math.round((now - jobStart) / 1000);
@@ -1234,7 +1253,7 @@ class UploadManager extends EventEmitter {
lastSpeedTime = now;
}
activeEntry.speedKbs = currentSpeedKbs;
activeEntry.bytesUploaded = bytesUploaded;
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
if (now - lastEmitTime < PROGRESS_EMIT_INTERVAL) return;
lastEmitTime = now;
const elapsed = Math.round((now - jobStart) / 1000);
@@ -1490,18 +1509,20 @@ class UploadManager extends EventEmitter {
});
}
_waitForInterval(hoster, intervalMs, signal) {
_waitForInterval(hoster, intervalMs, signal, admit = async () => {}) {
// Serialize interval waits per hoster so concurrent jobs queue up properly
const prev = this.intervalLocks[hoster] || Promise.resolve();
const next = prev.then(async () => {
const prev = this.intervalLocks[hoster];
const waitAndAdmit = async () => {
const now = Date.now();
const last = this.lastStartTime[hoster] || 0;
const elapsed = now - last;
if (elapsed < intervalMs) {
await this._sleep(intervalMs - elapsed, signal);
}
await admit();
this.lastStartTime[hoster] = Date.now();
});
};
const next = prev ? prev.then(waitAndAdmit) : waitAndAdmit();
this.intervalLocks[hoster] = next.catch(() => {});
return next;
}
@@ -1521,9 +1542,7 @@ class UploadManager extends EventEmitter {
}
const fileName = path.basename(task.file);
if (!results.has(task.file)) {
let size = 0;
try { size = fs.statSync(task.file).size; } catch {}
results.set(task.file, { name: fileName, size, results: [] });
results.set(task.file, { name: fileName, size: 0, results: [] });
}
this._additionalPromises.push(this._runJob(task, results, signal));
addResult.added++;
+3 -5
View File
@@ -1,4 +1,4 @@
const fs = require('fs');
const { statUploadFile, readUploadFile } = require('./upload-file-io');
const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
@@ -179,7 +179,7 @@ class VidmolyUploader {
*/
async upload(filePath, onProgress, signal, throttle) {
const fileName = path.basename(filePath);
const fileSize = fs.statSync(filePath).size;
const fileSize = (await statUploadFile(filePath)).size;
const baselineCodes = await this._captureVmFileCodes();
const { uploadUrl, params, fileFieldName } = await this.getUploadParams();
@@ -213,12 +213,10 @@ class VidmolyUploader {
const totalSize = preambleBuf.length + fileSize + epilogueBuf.length;
let bytesRead = 0;
const CHUNK_SIZE = 1024 * 1024;
async function* generate() {
yield preambleBuf;
const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE });
for await (const chunk of fileStream) {
for await (const chunk of readUploadFile(filePath, signal)) {
if (signal && signal.aborted) throw new Error('Aborted');
if (throttle) await throttle.consume(chunk.length, signal);
bytesRead += chunk.length;
+3 -5
View File
@@ -1,4 +1,4 @@
const fs = require('fs');
const { statUploadFile, readUploadFile } = require('./upload-file-io');
const path = require('path');
const crypto = require('crypto');
const { request } = require('undici');
@@ -212,7 +212,7 @@ class VoeUploader {
*/
async upload(filePath, onProgress, signal, throttle) {
const fileName = path.basename(filePath);
const fileSize = fs.statSync(filePath).size;
const fileSize = (await statUploadFile(filePath)).size;
const baselineCodes = await this._captureFileCodes();
// Step 1: Get CSRF token from upload page
@@ -243,12 +243,10 @@ class VoeUploader {
const totalSize = preambleBuf.length + fileSize + epilogueBuf.length;
let bytesRead = 0;
const CHUNK_SIZE = 1024 * 1024;
async function* generate() {
yield preambleBuf;
const fileStream = fs.createReadStream(filePath, { highWaterMark: CHUNK_SIZE });
for await (const chunk of fileStream) {
for await (const chunk of readUploadFile(filePath, signal)) {
if (signal && signal.aborted) throw new Error('Aborted');
if (throttle) await throttle.consume(chunk.length, signal);
bytesRead += chunk.length;