diff --git a/lib/clouddrop-lease-crypto.js b/lib/clouddrop-lease-crypto.js new file mode 100644 index 0000000..2a86f88 --- /dev/null +++ b/lib/clouddrop-lease-crypto.js @@ -0,0 +1,122 @@ +const { createCipheriv } = require('node:crypto'); + +const FIRST_SEGMENT = 128 * 1024; +const MAX_SEGMENT = 1024 * 1024; +const MAC_SLICE = 256 * 1024; + +function segmentLengths(size) { + const lengths = []; + let position = 0; + let next = FIRST_SEGMENT; + while (position < size) { + const length = Math.min(next, size - position); + lengths.push(length); + position += length; + if (next < MAX_SEGMENT) next = Math.min(MAX_SEGMENT, next + FIRST_SEGMENT); + } + return lengths; +} + +function endsOnSegmentBoundary(size) { + let position = 0; + let next = FIRST_SEGMENT; + while (position + next < size) { + position += next; + if (next < MAX_SEGMENT) next = Math.min(MAX_SEGMENT, next + FIRST_SEGMENT); + } + return Math.ceil((size - position) / 16) * 16 === next; +} + +function planChunks(size, targetBytes) { + if (!Number.isSafeInteger(size) || size <= 0) throw new TypeError('size must be a positive integer'); + const target = Math.max(FIRST_SEGMENT, Math.floor(targetBytes)); + const segments = segmentLengths(size); + const chunks = []; + let offset = 0; + let current = null; + for (const length of segments) { + if (!current) current = { index: chunks.length, offset, length: 0, segments: [] }; + current.segments.push(length); + current.length += length; + offset += length; + if (current.length >= target) { + chunks.push(current); + current = null; + } + } + if (current) chunks.push(current); + return { chunks, trailingMac: endsOnSegmentBoundary(size) }; +} + +function ctrIv(nonce, offset) { + if (offset % 16 !== 0) throw new RangeError('chunk offset must be a multiple of 16'); + const iv = Buffer.alloc(16); + nonce.copy(iv, 0, 0, 8); + iv.writeBigUInt64BE(BigInt(offset / 16), 8); + return iv; +} + +function encryptChunk({ aesKey, nonce, offset, data, segments }) { + const key = Buffer.from(aesKey); + const iv8 = Buffer.from(nonce); + const plain = Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); + const cipher = createCipheriv('aes-128-ctr', key, ctrIv(iv8, offset)).update(plain); + const macIv = Buffer.alloc(16); + iv8.copy(macIv, 0, 0, 8); + iv8.copy(macIv, 8, 0, 8); + const macs = Buffer.alloc(16 * segments.length); + let position = 0; + for (let s = 0; s < segments.length; s++) { + const end = position + segments[s]; + const cbc = createCipheriv('aes-128-cbc', key, macIv).setAutoPadding(false); + const fullEnd = position + Math.floor(segments[s] / 16) * 16; + let last = null; + for (let cursor = position; cursor < fullEnd;) { + const sliceEnd = Math.min(fullEnd, cursor + MAC_SLICE); + const out = cbc.update(plain.subarray(cursor, sliceEnd)); + last = out.subarray(out.length - 16); + cursor = sliceEnd; + } + if (fullEnd < end) { + const padded = Buffer.alloc(16); + plain.copy(padded, 0, fullEnd, end); + last = cbc.update(padded); + } + last.copy(macs, s * 16); + position = end; + } + if (position !== plain.length) throw new RangeError('segment lengths do not cover the chunk'); + return { cipher, macs }; +} + +function condenseMacs(aesKey, segmentMacs, { trailingMac = false, nonce = null } = {}) { + const ecb = createCipheriv('aes-128-ecb', Buffer.from(aesKey), null).setAutoPadding(false); + const items = [...segmentMacs]; + if (trailingMac) { + const iv = Buffer.alloc(16); + Buffer.from(nonce).copy(iv, 0, 0, 8); + Buffer.from(nonce).copy(iv, 8, 0, 8); + items.push(iv); + } + let mac = Buffer.alloc(16, 0); + for (const item of items) { + for (let offset = 0; offset < item.length; offset += 16) { + for (let j = 0; j < 16; j++) mac[j] ^= item[offset + j]; + mac = ecb.update(mac); + } + } + const out = Buffer.alloc(8); + out.writeInt32BE(mac.readInt32BE(0) ^ mac.readInt32BE(4), 0); + out.writeInt32BE(mac.readInt32BE(8) ^ mac.readInt32BE(12), 4); + return out; +} + +function e64(buffer) { + return Buffer.from(buffer).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function d64(value) { + return Buffer.from(String(value).replace(/-/g, '+').replace(/_/g, '/'), 'base64'); +} + +module.exports = { segmentLengths, endsOnSegmentBoundary, planChunks, encryptChunk, condenseMacs, e64, d64 }; diff --git a/lib/clouddrop-lease-worker.js b/lib/clouddrop-lease-worker.js new file mode 100644 index 0000000..2e659de --- /dev/null +++ b/lib/clouddrop-lease-worker.js @@ -0,0 +1,27 @@ +const { parentPort } = require('node:worker_threads'); +const { performance } = require('node:perf_hooks'); +const { encryptChunk } = require('./clouddrop-lease-crypto'); + +function transferable(buffer) { + if (buffer.byteOffset === 0 && buffer.buffer.byteLength === buffer.length) return buffer.buffer; + return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.length); +} + +parentPort.on('message', (job) => { + const started = performance.now(); + try { + const { cipher, macs } = encryptChunk({ + aesKey: Buffer.from(job.aesKey), + nonce: Buffer.from(job.nonce), + offset: job.offset, + data: Buffer.from(job.data, 0, job.length), + segments: job.segments, + }); + const cipherBuffer = transferable(cipher); + const macBuffer = transferable(macs); + parentPort.postMessage({ id: job.id, cipher: cipherBuffer, macs: macBuffer, plain: job.data, ms: performance.now() - started }, + [cipherBuffer, macBuffer, job.data]); + } catch (error) { + parentPort.postMessage({ id: job.id, error: error && error.message ? error.message : String(error), plain: job.data }, [job.data]); + } +}); diff --git a/lib/clouddrop-lease.js b/lib/clouddrop-lease.js new file mode 100644 index 0000000..af7caf4 --- /dev/null +++ b/lib/clouddrop-lease.js @@ -0,0 +1,472 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { Readable } = require('node:stream'); +const { Worker } = require('node:worker_threads'); +const { performance } = require('node:perf_hooks'); +const { setTimeout: delay } = require('node:timers/promises'); +const { request, Agent } = require('undici'); +const { planChunks, condenseMacs, e64, d64 } = require('./clouddrop-lease-crypto'); + +const MIB = 1024 * 1024; +const LEASE_PATH = '/api/cloud/mega-lease'; +const RETRYABLE_MEGA_CODES = new Set([-3, -4, -6]); +const DEFAULTS = Object.freeze({ + connections: 12, + chunkSize: 8 * MIB, + minStartRate: 0, + relativeStartRatio: 0.25, + maxSlowRestarts: 2, + maxPartAttempts: 4, + maxChunkRetries: 6, + warmupMs: 5000, + windowMs: 4000, + requestTimeoutMs: 120000, + apiTimeoutMs: 180000, + maxInflightChunks: 64, +}); + +const megaAgent = new Agent({ connections: 64, pipelining: 1, keepAliveTimeout: 30_000, keepAliveMaxTimeout: 60_000 }); + +class Semaphore { + constructor(count) { + this.count = count; + this.waiters = []; + } + async acquire(signal) { + if (this.count > 0) { this.count--; return; } + await new Promise((resolve, reject) => { + const entry = { resolve, reject }; + this.waiters.push(entry); + if (signal) { + const onAbort = () => { + const index = this.waiters.indexOf(entry); + if (index >= 0) this.waiters.splice(index, 1); + reject(signal.reason || new Error('Aborted')); + }; + if (signal.aborted) onAbort(); + else signal.addEventListener('abort', onAbort, { once: true }); + } + }); + } + release() { + const next = this.waiters.shift(); + if (next) next.resolve(); + else this.count++; + } +} + +class CryptoPool { + constructor(size) { + this.size = size; + this.workers = []; + this.idle = []; + this.queue = []; + this.pending = new Map(); + this.nextId = 1; + } + + _spawn() { + const worker = new Worker(path.join(__dirname, 'clouddrop-lease-worker.js')); + worker.on('message', (message) => this._onMessage(worker, message)); + worker.on('error', (error) => this._onError(worker, error)); + worker.on('exit', () => { + this.workers = this.workers.filter(item => item !== worker); + this.idle = this.idle.filter(item => item !== worker); + }); + worker.unref(); + this.workers.push(worker); + this.idle.push(worker); + } + + encrypt(job) { + return new Promise((resolve, reject) => { + const id = this.nextId++; + this.pending.set(id, { resolve, reject }); + this.queue.push({ id, ...job }); + if (!this.idle.length && this.workers.length < this.size) this._spawn(); + this._drain(); + }); + } + + _drain() { + while (this.idle.length && this.queue.length) { + const worker = this.idle.pop(); + const job = this.queue.shift(); + worker.currentJob = job.id; + worker.postMessage(job, [job.data]); + } + } + + _onMessage(worker, message) { + worker.currentJob = null; + this.idle.push(worker); + const entry = this.pending.get(message.id); + this.pending.delete(message.id); + if (entry) { + if (message.error) { + const error = new Error(message.error); + error.plain = message.plain; + entry.reject(error); + } else { + entry.resolve({ cipher: Buffer.from(message.cipher), macs: Buffer.from(message.macs), plain: message.plain }); + } + } + this._drain(); + } + + _onError(worker, error) { + const id = worker.currentJob; + if (id && this.pending.has(id)) { + this.pending.get(id).reject(error); + this.pending.delete(id); + } + } +} + +let sharedPool = null; +let sharedSlots = null; +const lineStats = { bestRate: 0, peakAggregate: 0, active: new Map() }; + +function cryptoPool() { + if (!sharedPool) sharedPool = new CryptoPool(Math.max(1, Math.min(6, os.cpus().length - 1))); + return sharedPool; +} + +function chunkSlots() { + if (!sharedSlots) sharedSlots = new Semaphore(DEFAULTS.maxInflightChunks); + return sharedSlots; +} + +class PartFailure extends Error { + constructor(message, details = {}) { + super(message); + this.details = details; + } +} + +function megaCode(body) { + const text = body.toString('utf8').trim(); + return /^-\d{1,3}$/.test(text) ? Number(text) : null; +} + +function bodyStream(buffer, onSent) { + return Readable.from((function* slices() { + for (let offset = 0; offset < buffer.length; offset += MIB) { + const slice = buffer.subarray(offset, Math.min(offset + MIB, buffer.length)); + onSent(slice.length); + yield slice; + } + })(), { objectMode: false }); +} + +async function readFully(handle, target, length, position) { + let done = 0; + while (done < length) { + const { bytesRead } = await handle.read(target, done, length - done, position + done); + if (bytesRead <= 0) throw new Error(`Clouddrop: Datei endet unerwartet bei Byte ${position + done}`); + done += bytesRead; + } +} + +function leaseError(prefix, reply) { + const payload = reply.payload; + const error = new Error(`Clouddrop: ${prefix}${payload && payload.error ? `: ${payload.error}` : ` (HTTP ${reply.status})`}`); + error.status = reply.status; + if (payload && typeof payload.code === 'string') error.code = payload.code; + return error; +} + +class LeaseClient { + constructor({ apiKey, baseUrl, dispatcher, userAgent, sleep, now }) { + this.apiKey = apiKey; + this.base = `${baseUrl}${LEASE_PATH}`; + this.dispatcher = dispatcher; + this.userAgent = userAgent; + this.sleep = sleep || ((ms, signal) => delay(ms, undefined, { signal })); + this.now = now || Date.now; + } + + async call(method, route, body, { signal, retries = 4, timeoutMs = DEFAULTS.apiTimeoutMs } = {}) { + for (let attempt = 0; ; attempt++) { + let reply; + try { + const res = await request(`${this.base}${route}`, { + method, + dispatcher: this.dispatcher, + signal, + headers: { + 'Authorization': `Bearer ${this.apiKey}`, + 'User-Agent': this.userAgent, + 'Accept': 'application/json', + ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + headersTimeout: timeoutMs, + bodyTimeout: timeoutMs, + }); + const text = await res.body.text(); + let payload = null; + try { payload = text ? JSON.parse(text) : null; } catch {} + reply = { status: res.statusCode, payload }; + } catch (error) { + if (signal && signal.aborted) throw error; + if (attempt >= retries) { error.transientNetwork = true; throw error; } + await this.sleep(Math.min(30_000, 1000 * 2 ** attempt), signal); + continue; + } + if (reply.status >= 200 && reply.status < 300) return reply.payload; + const transient = reply.status >= 500 && reply.status !== 507 && !(reply.payload && reply.payload.code === 'MEGA_NODE_FAILED'); + if (transient && attempt < retries) { + await this.sleep(Math.min(30_000, 1000 * 2 ** attempt), signal); + continue; + } + const error = leaseError('Direkt-Upload abgelehnt', reply); + if (transient || reply.status === 507) error.transientNetwork = true; + throw error; + } + } +} + +async function postChunk({ url, cipher, signal, timeoutMs, onSent }) { + const res = await request(url, { + method: 'POST', + dispatcher: megaAgent, + signal, + body: bodyStream(cipher, onSent), + headers: { 'Content-Type': 'application/octet-stream', 'Content-Length': String(cipher.length) }, + headersTimeout: timeoutMs, + bodyTimeout: timeoutMs, + }); + const body = Buffer.from(await res.body.arrayBuffer()); + return { status: res.statusCode, body }; +} + +async function uploadViaLease({ apiKey, baseUrl, dispatcher, userAgent, filePath, fileName, fileSize, progressCb, signal, throttle, options = {}, sleep, now }) { + const settings = { ...DEFAULTS, ...options }; + const throttled = Boolean(throttle) && !(typeof throttle.maxBps === 'number' && throttle.maxBps <= 0); + const minStartRate = throttled ? 0 : settings.minStartRate; + const api = new LeaseClient({ apiKey, baseUrl, dispatcher, userAgent, sleep, now }); + const started = performance.now(); + const lease = await api.call('POST', '/upload/init', { + filename: fileName, + size: fileSize, + mode: 'rename', + client: { tool: 'multi-hoster-uploader', version: options.appVersion || null, host: os.hostname(), settings: { connections: settings.connections, chunkSize: settings.chunkSize } }, + }, { signal, retries: 2 }); + const leaseId = lease.leaseId; + const events = []; + const event = (type, data) => events.push({ kind: 'event', type, t: Date.now(), ms: Math.round(performance.now() - started), ...data }); + const handle = await fs.promises.open(filePath, 'r'); + const pool = cryptoPool(); + const slots = chunkSlots(); + const fullHash = crypto.createHash('sha256'); + const avoidHosts = new Set(); + let hashedUpTo = 0; + let doneBytes = 0; + let slowRestarts = 0; + let retries = 0; + const report = (partAcked) => { if (progressCb) progressCb(Math.min(fileSize, doneBytes + partAcked), fileSize); }; + + async function sendPart(part, target) { + const key = d64(target.fileKey); + const aesKey = key.subarray(0, 16); + const nonce = key.subarray(16, 24); + const plan = planChunks(part.size, settings.chunkSize); + const macs = new Array(plan.chunks.length); + const partHash = crypto.createHash('sha256'); + const partAbort = new AbortController(); + const partSignal = signal ? AbortSignal.any([signal, partAbort.signal]) : partAbort.signal; + const local = new Semaphore(settings.connections); + const uploadStarted = performance.now(); + const history = []; + let sent = 0; + let acked = 0; + let completionHandle = null; + let partError = null; + const fail = (error) => { if (!partError) { partError = error; partAbort.abort(error); } }; + const monitor = setInterval(() => { + const at = performance.now(); + history.push({ at, sent }); + while (history.length && at - history[0].at > settings.windowMs * 2) history.shift(); + const from = history.find(item => at - item.at <= settings.windowMs * 1.25 + 250) || history[0]; + const seconds = (at - from.at) / 1000; + if (seconds < settings.windowMs / 2000) return; + const rate = (sent - from.sent) / seconds; + lineStats.active.set(leaseId, rate); + const aggregate = [...lineStats.active.values()].reduce((sum, value) => sum + value, 0); + lineStats.peakAggregate = Math.max(lineStats.peakAggregate, aggregate); + if (at - uploadStarted < settings.warmupMs || throttled) return; + if (slowRestarts >= settings.maxSlowRestarts || acked >= part.size * 0.15) return; + const lineFree = lineStats.active.size <= 1 || aggregate < lineStats.peakAggregate * 0.6; + const absoluteSlow = minStartRate > 0 && rate < minStartRate; + const relativeSlow = lineStats.bestRate > 0 && rate < lineStats.bestRate * settings.relativeStartRatio; + if ((absoluteSlow || relativeSlow) && lineFree) { + event('part_slow', { + part: part.index, host: target.host, rateBytesPerSecond: Math.round(rate), bestBytesPerSecond: Math.round(lineStats.bestRate), + rule: absoluteSlow ? 'slow-start' : 'slow-start-relative', + }); + fail(new PartFailure(`Clouddrop: Langsamer MEGA-Server ${target.host}`, { kind: 'slow' })); + } + }, 1000); + monitor.unref?.(); + const running = []; + try { + for (const chunk of plan.chunks) { + await local.acquire(partSignal).catch(error => fail(error)); + if (partError) break; + await slots.acquire(partSignal).catch(error => fail(error)); + if (partError) { local.release(); break; } + let plain = new ArrayBuffer(chunk.length); + const view = Buffer.from(plain, 0, chunk.length); + try { + await readFully(handle, view, chunk.length, part.offset + chunk.offset); + } catch (error) { + slots.release(); + local.release(); + fail(error); + break; + } + partHash.update(view); + if (part.offset + chunk.offset === hashedUpTo) { + fullHash.update(view); + hashedUpTo += chunk.length; + } + running.push((async () => { + try { + const encrypted = await pool.encrypt({ aesKey, nonce, offset: chunk.offset, data: plain, length: chunk.length, segments: chunk.segments }); + plain = encrypted.plain; + for (let tries = 1; ; tries++) { + if (partSignal.aborted) throw partSignal.reason; + if (throttle) await throttle.consume(chunk.length, partSignal); + let result = null; + let failure = null; + let sentThisTry = 0; + try { + result = await postChunk({ + url: `${target.uploadUrl}/${chunk.offset}`, + cipher: encrypted.cipher, + signal: partSignal, + timeoutMs: settings.requestTimeoutMs, + onSent: (bytes) => { sent += bytes; sentThisTry += bytes; }, + }); + } catch (error) { + if (partSignal.aborted) throw partSignal.reason; + failure = { kind: 'network', message: error.code || error.message }; + } + if (!failure && result.status !== 200) failure = { kind: 'http', message: `HTTP ${result.status}` }; + const code = !failure && result.body.length ? megaCode(result.body) : null; + if (!failure && code !== null) failure = { kind: 'mega', code, message: `MEGA ${code}` }; + if (!failure) { + if (result.body.length) completionHandle = e64(result.body); + macs[chunk.index] = encrypted.macs; + acked += chunk.length; + report(acked); + return; + } + sent -= sentThisTry; + retries++; + event('chunk_retry', { part: part.index, chunk: chunk.index, try: tries, ...failure }); + if (failure.kind === 'mega' && !RETRYABLE_MEGA_CODES.has(failure.code)) throw new PartFailure(`Clouddrop: MEGA-Fehler ${failure.code}`, failure); + if (tries > settings.maxChunkRetries) throw new PartFailure(`Clouddrop: Anfrage an MEGA nach ${tries} Versuchen fehlgeschlagen`, failure); + await delay(Math.min(30_000, 500 * 2 ** tries), undefined, { signal: partSignal }); + } + } catch (error) { + fail(error); + } finally { + slots.release(); + local.release(); + } + })()); + } + await Promise.all(running); + } finally { + clearInterval(monitor); + lineStats.active.delete(leaseId); + } + if (partError) { partError.ackedBytes = acked; throw partError; } + if (!completionHandle) throw new PartFailure('Clouddrop: MEGA hat keine Abschluss-Kennung geliefert'); + return { + completionHandle, + mac: e64(condenseMacs(aesKey, macs, { trailingMac: plan.trailingMac, nonce })), + sha256: partHash.digest('hex'), + uploadMs: performance.now() - uploadStarted, + }; + } + + async function completePart(part, body) { + for (let i = 0; ; i++) { + try { + return await api.call('POST', `/upload/${leaseId}/parts/${part.index}/complete`, body, { signal }); + } catch (error) { + if (error.code === 'PART_BUSY' && i < 24) { await api.sleep(5000, signal); continue; } + throw error; + } + } + } + + try { + for (const part of lease.parts) { + let reason = null; + for (let attempt = 1; ; attempt++) { + const body = { avoidHosts: [...avoidHosts] }; + if (reason) body.reason = reason; + const target = await api.call('POST', `/upload/${leaseId}/parts/${part.index}/target`, body, { signal, retries: 2 }); + target.host = new URL(target.uploadUrl).host; + event('target', { part: part.index, attempt: target.attempt, host: target.host, server: target.timings }); + try { + const sentPart = await sendPart(part, target); + await completePart(part, { + attempt: target.attempt, + completionHandle: sentPart.completionHandle, + mac: sentPart.mac, + sha256: sentPart.sha256, + bytes: part.size, + stats: { uploadMs: Math.round(sentPart.uploadMs), host: target.host }, + }); + doneBytes += part.size; + report(0); + if (!throttled && part.size >= 64 * MIB && sentPart.uploadMs > 0) { + lineStats.bestRate = Math.max(lineStats.bestRate, part.size / (sentPart.uploadMs / 1000)); + } + event('part_done', { part: part.index, attempt: target.attempt, bytes: part.size, uploadMs: Math.round(sentPart.uploadMs), bytesPerSecond: Math.round(part.size / (sentPart.uploadMs / 1000)), host: target.host }); + break; + } catch (error) { + if (signal && signal.aborted) throw error; + const slow = error instanceof PartFailure && error.details.kind === 'slow'; + const retryable = error instanceof PartFailure || error.code === 'MEGA_NODE_FAILED'; + if (slow) { slowRestarts++; avoidHosts.add(target.host); } + report(0); + reason = String(error.message || error).slice(0, 180); + event(slow ? 'part_retarget_slow' : 'part_failed', { part: part.index, attempt: target.attempt, error: reason }); + if (!retryable || attempt >= settings.maxPartAttempts) { + if (retryable) error.transientNetwork = true; + throw error; + } + if (!slow) await api.sleep(2000 * attempt, signal); + } + } + } + const final = await api.call('POST', `/upload/${leaseId}/finalize`, { sha256: fullHash.digest('hex') }, { signal }); + const durationMs = performance.now() - started; + events.push({ + kind: 'summary', t: Date.now(), durationMs: Math.round(durationMs), ackedBytes: fileSize, + averageBytesPerSecond: Math.round(fileSize / (durationMs / 1000)), retries, errors: 0, slowRetargets: slowRestarts, + settings: { connections: settings.connections, chunkSize: settings.chunkSize, partSize: lease.partBytes, parallelParts: 1 }, + host: { name: os.hostname(), platform: process.platform, cpus: os.cpus().length, node: process.version }, + }); + api.call('POST', `/upload/${leaseId}/telemetry`, { records: events }, { retries: 1 }).catch(() => {}); + return final.fileId; + } catch (error) { + if (error && typeof error === 'object') error.leaseStarted = true; + events.push({ kind: 'event', type: 'failed', t: Date.now(), error: String(error.message || error).slice(0, 200) }); + api.call('POST', `/upload/${leaseId}/telemetry`, { records: events }, { retries: 0 }).catch(() => {}); + api.call('POST', `/upload/${leaseId}/abort`, { reason: String(error.message || error).slice(0, 180) }, { retries: 1 }).catch(() => {}); + throw error; + } finally { + await handle.close().catch(() => {}); + } +} + +module.exports = { uploadViaLease, DEFAULTS, PartFailure, _megaAgent: megaAgent, _lineStats: lineStats }; diff --git a/lib/clouddrop-upload.js b/lib/clouddrop-upload.js index 7ec7ff2..10eefb5 100644 --- a/lib/clouddrop-upload.js +++ b/lib/clouddrop-upload.js @@ -4,6 +4,8 @@ const path = require('path'); const crypto = require('crypto'); const { setTimeout: delay } = require('node:timers/promises'); const { request, Agent } = require('undici'); +const { uploadViaLease } = require('./clouddrop-lease'); +const { version: APP_VERSION } = require('../package.json'); const BASE_URL = 'https://clouddrop.cc'; const API_BASE = `${BASE_URL}/api/cloud`; @@ -34,6 +36,10 @@ const JOB_POLL_MAX_INTERVAL_MS = 30_000; const JOB_POLL_ERROR_MAX_DELAY_MS = 60_000; const JOB_POLL_MAX_MS = 24 * 60 * 60_000; const EXPRESS_REUPLOAD_ERROR = /^dead:(?:express_[a-z_]+|relay_chunks_missing)$/; +const LEASE_ENABLED_CACHE_MS = 5 * 60_000; +const LEASE_DISABLED_CACHE_MS = 60_000; +const LEASE_FALLBACK_CODES = new Set(['MEGA_LEASE_DISABLED']); +const leaseAvailability = new Map(); // Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under // the server's per-IP limit of 100 concurrent connections (cd_conn). @@ -97,6 +103,62 @@ class ClouddropUploader { this.dispatcher = options.dispatcher || clouddropAgent; this.sleep = options.sleep || ((ms, signal) => delay(ms, undefined, { signal })); this.now = options.now || Date.now; + this.leaseOptions = options.leaseOptions || {}; + this.leaseDisabled = options.lease === false || process.env.MHU_CLOUDDROP_LEASE === '0'; + } + + _leaseCacheKey() { + return sha256Hex(Buffer.from(this.apiKey)).slice(0, 24); + } + + async _leaseEnabled(signal) { + if (this.leaseDisabled) return false; + const key = this._leaseCacheKey(); + const cached = leaseAvailability.get(key); + if (cached && cached.until > this.now()) return cached.enabled; + let enabled = false; + try { + const res = await request(`${API_BASE}/mega-lease/config`, { + method: 'GET', + dispatcher: this.dispatcher, + signal, + headers: this._headers(), + headersTimeout: 15_000, + bodyTimeout: 15_000 + }); + const reply = await this._readReply(res); + enabled = reply.ok && Boolean(reply.payload) && reply.payload.enabled === true; + } catch (error) { + if (signal && signal.aborted) throw error; + } + leaseAvailability.set(key, { enabled, until: this.now() + (enabled ? LEASE_ENABLED_CACHE_MS : LEASE_DISABLED_CACHE_MS) }); + return enabled; + } + + async _uploadLease(filePath, fileName, fileSize, progressCb, signal, throttle) { + try { + return await uploadViaLease({ + apiKey: this.apiKey, + baseUrl: BASE_URL, + dispatcher: this.dispatcher, + userAgent: USER_AGENT, + filePath, + fileName, + fileSize, + progressCb, + signal, + throttle, + sleep: this.sleep, + now: this.now, + options: { appVersion: APP_VERSION, ...this.leaseOptions } + }); + } catch (error) { + if (error && (LEASE_FALLBACK_CODES.has(error.code) || error.status === 404) && !error.leaseStarted) { + leaseAvailability.set(this._leaseCacheKey(), { enabled: false, until: this.now() + LEASE_DISABLED_CACHE_MS }); + return null; + } + throw error; + } } _headers(extra) { @@ -178,6 +240,10 @@ class ClouddropUploader { if (fileSize <= SIMPLE_UPLOAD_LIMIT) { return shareResult(await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle)); } + if (await this._leaseEnabled(signal)) { + const fileId = await this._uploadLease(filePath, fileName, fileSize, progressCb, signal, throttle); + if (fileId) return shareResult(fileId); + } const outcome = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle); return typeof outcome === 'string' ? shareResult(outcome) : outcome; } diff --git a/renderer/i18n.js b/renderer/i18n.js index e8f4f5a..4daf051 100644 --- a/renderer/i18n.js +++ b/renderer/i18n.js @@ -828,6 +828,7 @@ ['Verlauf und Aufbewahrung konnten nicht konsistent gespeichert werden', 'History and retention could not be saved consistently'], ['Kein Ordnerpfad angegeben', 'No folder path was provided'], ['Clouddrop: Datei ist leer', 'Clouddrop: The file is empty'], + ['Clouddrop: MEGA hat keine Abschluss-Kennung geliefert', 'Clouddrop: MEGA did not return a completion handle'], ['Clouddrop: Keine fileId in Upload-Antwort', 'Clouddrop: No fileId in the upload response'], ['Clouddrop: Keine sessionId von /upload/init', 'Clouddrop: No sessionId from /upload/init'], ['Clouddrop: Keine fileId im Job-Ergebnis', 'Clouddrop: No fileId in the job result'], @@ -1042,6 +1043,12 @@ [/^Quelldatei-Schutz konnte nicht vorbereitet werden: (.+)$/, 'Source file protection could not be prepared: $1'], [/^Clouddrop: API-Antwort war kein JSON (.+)$/, 'Clouddrop: API response was not JSON $1'], [/^Clouddrop: Datei nicht lesbar: (.+)$/, 'Clouddrop: File cannot be read: $1'], + [/^Clouddrop: Datei endet unerwartet bei Byte (.+)$/, 'Clouddrop: The file ends unexpectedly at byte $1'], + [/^Clouddrop: Direkt-Upload abgelehnt: (.+)$/, 'Clouddrop: Direct upload rejected: $1'], + [/^Clouddrop: Direkt-Upload abgelehnt \(HTTP (\d{3})\)$/, 'Clouddrop: Direct upload rejected (HTTP $1)'], + [/^Clouddrop: Langsamer MEGA-Server (.+)$/, 'Clouddrop: Slow MEGA server $1'], + [/^Clouddrop: MEGA-Fehler (.+)$/, 'Clouddrop: MEGA error $1'], + [/^Clouddrop: Anfrage an MEGA nach (\d+) Versuchen fehlgeschlagen$/, 'Clouddrop: Request to MEGA failed after $1 attempts'], [/^Clouddrop: Verarbeitung fehlgeschlagen \((.+)\)$/,'Clouddrop: Processing failed ($1)'], [/^Doodstream: konnte Upload-Server nicht ermitteln \(Endpoint geƤndert\?\)\. (.+)$/, 'Doodstream: Could not determine the upload server (endpoint changed?). $1'], [/^Doodstream Upload fehlgeschlagen: (.+)$/, 'Doodstream upload failed: $1'],