Files above 16 MB use CloudDrop's mega-lease protocol when the server has it switched on: the server picks the MEGA account and upload address, the uploader encrypts the file with MEGA's scheme in worker threads and sends it straight to MEGA, then the server registers the finished file. Each file stays whole in one account. Throttling, progress and cancellation work as before; without the server feature, or with MHU_CLOUDDROP_LEASE=0, the existing upload path is used unchanged. A start far below this machine's earlier lease throughput is restarted on another MEGA server when CloudDrop uploads are not just sharing a busy line. New error messages have English translations. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01E7HXdgTtkBuSJPpiTRCGa8
This commit is contained in:
1 parent
8de07502db
commit
702376f80d
5 files changed
+694
No files matched your search
@@ -0,0 +1,122 @@
|
|||||||
|
const { createCipheriv } = require('node:crypto');
|
||||||
|
|
||||||
|
const FIRST_SEGMENT = 128 * 1024;
|
||||||
|
const MAX_SEGMENT = 1024 * 1024;
|
||||||
|
const MAC_SLICE = 256 * 1024;
|
||||||
|
|
||||||
|
function segmentLengths(size) {
|
||||||
|
const lengths = [];
|
||||||
|
let position = 0;
|
||||||
|
let next = FIRST_SEGMENT;
|
||||||
|
while (position < size) {
|
||||||
|
const length = Math.min(next, size - position);
|
||||||
|
lengths.push(length);
|
||||||
|
position += length;
|
||||||
|
if (next < MAX_SEGMENT) next = Math.min(MAX_SEGMENT, next + FIRST_SEGMENT);
|
||||||
|
}
|
||||||
|
return lengths;
|
||||||
|
}
|
||||||
|
|
||||||
|
function endsOnSegmentBoundary(size) {
|
||||||
|
let position = 0;
|
||||||
|
let next = FIRST_SEGMENT;
|
||||||
|
while (position + next < size) {
|
||||||
|
position += next;
|
||||||
|
if (next < MAX_SEGMENT) next = Math.min(MAX_SEGMENT, next + FIRST_SEGMENT);
|
||||||
|
}
|
||||||
|
return Math.ceil((size - position) / 16) * 16 === next;
|
||||||
|
}
|
||||||
|
|
||||||
|
function planChunks(size, targetBytes) {
|
||||||
|
if (!Number.isSafeInteger(size) || size <= 0) throw new TypeError('size must be a positive integer');
|
||||||
|
const target = Math.max(FIRST_SEGMENT, Math.floor(targetBytes));
|
||||||
|
const segments = segmentLengths(size);
|
||||||
|
const chunks = [];
|
||||||
|
let offset = 0;
|
||||||
|
let current = null;
|
||||||
|
for (const length of segments) {
|
||||||
|
if (!current) current = { index: chunks.length, offset, length: 0, segments: [] };
|
||||||
|
current.segments.push(length);
|
||||||
|
current.length += length;
|
||||||
|
offset += length;
|
||||||
|
if (current.length >= target) {
|
||||||
|
chunks.push(current);
|
||||||
|
current = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (current) chunks.push(current);
|
||||||
|
return { chunks, trailingMac: endsOnSegmentBoundary(size) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function ctrIv(nonce, offset) {
|
||||||
|
if (offset % 16 !== 0) throw new RangeError('chunk offset must be a multiple of 16');
|
||||||
|
const iv = Buffer.alloc(16);
|
||||||
|
nonce.copy(iv, 0, 0, 8);
|
||||||
|
iv.writeBigUInt64BE(BigInt(offset / 16), 8);
|
||||||
|
return iv;
|
||||||
|
}
|
||||||
|
|
||||||
|
function encryptChunk({ aesKey, nonce, offset, data, segments }) {
|
||||||
|
const key = Buffer.from(aesKey);
|
||||||
|
const iv8 = Buffer.from(nonce);
|
||||||
|
const plain = Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength);
|
||||||
|
const cipher = createCipheriv('aes-128-ctr', key, ctrIv(iv8, offset)).update(plain);
|
||||||
|
const macIv = Buffer.alloc(16);
|
||||||
|
iv8.copy(macIv, 0, 0, 8);
|
||||||
|
iv8.copy(macIv, 8, 0, 8);
|
||||||
|
const macs = Buffer.alloc(16 * segments.length);
|
||||||
|
let position = 0;
|
||||||
|
for (let s = 0; s < segments.length; s++) {
|
||||||
|
const end = position + segments[s];
|
||||||
|
const cbc = createCipheriv('aes-128-cbc', key, macIv).setAutoPadding(false);
|
||||||
|
const fullEnd = position + Math.floor(segments[s] / 16) * 16;
|
||||||
|
let last = null;
|
||||||
|
for (let cursor = position; cursor < fullEnd;) {
|
||||||
|
const sliceEnd = Math.min(fullEnd, cursor + MAC_SLICE);
|
||||||
|
const out = cbc.update(plain.subarray(cursor, sliceEnd));
|
||||||
|
last = out.subarray(out.length - 16);
|
||||||
|
cursor = sliceEnd;
|
||||||
|
}
|
||||||
|
if (fullEnd < end) {
|
||||||
|
const padded = Buffer.alloc(16);
|
||||||
|
plain.copy(padded, 0, fullEnd, end);
|
||||||
|
last = cbc.update(padded);
|
||||||
|
}
|
||||||
|
last.copy(macs, s * 16);
|
||||||
|
position = end;
|
||||||
|
}
|
||||||
|
if (position !== plain.length) throw new RangeError('segment lengths do not cover the chunk');
|
||||||
|
return { cipher, macs };
|
||||||
|
}
|
||||||
|
|
||||||
|
function condenseMacs(aesKey, segmentMacs, { trailingMac = false, nonce = null } = {}) {
|
||||||
|
const ecb = createCipheriv('aes-128-ecb', Buffer.from(aesKey), null).setAutoPadding(false);
|
||||||
|
const items = [...segmentMacs];
|
||||||
|
if (trailingMac) {
|
||||||
|
const iv = Buffer.alloc(16);
|
||||||
|
Buffer.from(nonce).copy(iv, 0, 0, 8);
|
||||||
|
Buffer.from(nonce).copy(iv, 8, 0, 8);
|
||||||
|
items.push(iv);
|
||||||
|
}
|
||||||
|
let mac = Buffer.alloc(16, 0);
|
||||||
|
for (const item of items) {
|
||||||
|
for (let offset = 0; offset < item.length; offset += 16) {
|
||||||
|
for (let j = 0; j < 16; j++) mac[j] ^= item[offset + j];
|
||||||
|
mac = ecb.update(mac);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const out = Buffer.alloc(8);
|
||||||
|
out.writeInt32BE(mac.readInt32BE(0) ^ mac.readInt32BE(4), 0);
|
||||||
|
out.writeInt32BE(mac.readInt32BE(8) ^ mac.readInt32BE(12), 4);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function e64(buffer) {
|
||||||
|
return Buffer.from(buffer).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function d64(value) {
|
||||||
|
return Buffer.from(String(value).replace(/-/g, '+').replace(/_/g, '/'), 'base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { segmentLengths, endsOnSegmentBoundary, planChunks, encryptChunk, condenseMacs, e64, d64 };
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
const { parentPort } = require('node:worker_threads');
|
||||||
|
const { performance } = require('node:perf_hooks');
|
||||||
|
const { encryptChunk } = require('./clouddrop-lease-crypto');
|
||||||
|
|
||||||
|
function transferable(buffer) {
|
||||||
|
if (buffer.byteOffset === 0 && buffer.buffer.byteLength === buffer.length) return buffer.buffer;
|
||||||
|
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
parentPort.on('message', (job) => {
|
||||||
|
const started = performance.now();
|
||||||
|
try {
|
||||||
|
const { cipher, macs } = encryptChunk({
|
||||||
|
aesKey: Buffer.from(job.aesKey),
|
||||||
|
nonce: Buffer.from(job.nonce),
|
||||||
|
offset: job.offset,
|
||||||
|
data: Buffer.from(job.data, 0, job.length),
|
||||||
|
segments: job.segments,
|
||||||
|
});
|
||||||
|
const cipherBuffer = transferable(cipher);
|
||||||
|
const macBuffer = transferable(macs);
|
||||||
|
parentPort.postMessage({ id: job.id, cipher: cipherBuffer, macs: macBuffer, plain: job.data, ms: performance.now() - started },
|
||||||
|
[cipherBuffer, macBuffer, job.data]);
|
||||||
|
} catch (error) {
|
||||||
|
parentPort.postMessage({ id: job.id, error: error && error.message ? error.message : String(error), plain: job.data }, [job.data]);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,472 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const os = require('os');
|
||||||
|
const path = require('path');
|
||||||
|
const crypto = require('crypto');
|
||||||
|
const { Readable } = require('node:stream');
|
||||||
|
const { Worker } = require('node:worker_threads');
|
||||||
|
const { performance } = require('node:perf_hooks');
|
||||||
|
const { setTimeout: delay } = require('node:timers/promises');
|
||||||
|
const { request, Agent } = require('undici');
|
||||||
|
const { planChunks, condenseMacs, e64, d64 } = require('./clouddrop-lease-crypto');
|
||||||
|
|
||||||
|
const MIB = 1024 * 1024;
|
||||||
|
const LEASE_PATH = '/api/cloud/mega-lease';
|
||||||
|
const RETRYABLE_MEGA_CODES = new Set([-3, -4, -6]);
|
||||||
|
const DEFAULTS = Object.freeze({
|
||||||
|
connections: 12,
|
||||||
|
chunkSize: 8 * MIB,
|
||||||
|
minStartRate: 0,
|
||||||
|
relativeStartRatio: 0.25,
|
||||||
|
maxSlowRestarts: 2,
|
||||||
|
maxPartAttempts: 4,
|
||||||
|
maxChunkRetries: 6,
|
||||||
|
warmupMs: 5000,
|
||||||
|
windowMs: 4000,
|
||||||
|
requestTimeoutMs: 120000,
|
||||||
|
apiTimeoutMs: 180000,
|
||||||
|
maxInflightChunks: 64,
|
||||||
|
});
|
||||||
|
|
||||||
|
const megaAgent = new Agent({ connections: 64, pipelining: 1, keepAliveTimeout: 30_000, keepAliveMaxTimeout: 60_000 });
|
||||||
|
|
||||||
|
class Semaphore {
|
||||||
|
constructor(count) {
|
||||||
|
this.count = count;
|
||||||
|
this.waiters = [];
|
||||||
|
}
|
||||||
|
async acquire(signal) {
|
||||||
|
if (this.count > 0) { this.count--; return; }
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
const entry = { resolve, reject };
|
||||||
|
this.waiters.push(entry);
|
||||||
|
if (signal) {
|
||||||
|
const onAbort = () => {
|
||||||
|
const index = this.waiters.indexOf(entry);
|
||||||
|
if (index >= 0) this.waiters.splice(index, 1);
|
||||||
|
reject(signal.reason || new Error('Aborted'));
|
||||||
|
};
|
||||||
|
if (signal.aborted) onAbort();
|
||||||
|
else signal.addEventListener('abort', onAbort, { once: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
release() {
|
||||||
|
const next = this.waiters.shift();
|
||||||
|
if (next) next.resolve();
|
||||||
|
else this.count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class CryptoPool {
|
||||||
|
constructor(size) {
|
||||||
|
this.size = size;
|
||||||
|
this.workers = [];
|
||||||
|
this.idle = [];
|
||||||
|
this.queue = [];
|
||||||
|
this.pending = new Map();
|
||||||
|
this.nextId = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
_spawn() {
|
||||||
|
const worker = new Worker(path.join(__dirname, 'clouddrop-lease-worker.js'));
|
||||||
|
worker.on('message', (message) => this._onMessage(worker, message));
|
||||||
|
worker.on('error', (error) => this._onError(worker, error));
|
||||||
|
worker.on('exit', () => {
|
||||||
|
this.workers = this.workers.filter(item => item !== worker);
|
||||||
|
this.idle = this.idle.filter(item => item !== worker);
|
||||||
|
});
|
||||||
|
worker.unref();
|
||||||
|
this.workers.push(worker);
|
||||||
|
this.idle.push(worker);
|
||||||
|
}
|
||||||
|
|
||||||
|
encrypt(job) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const id = this.nextId++;
|
||||||
|
this.pending.set(id, { resolve, reject });
|
||||||
|
this.queue.push({ id, ...job });
|
||||||
|
if (!this.idle.length && this.workers.length < this.size) this._spawn();
|
||||||
|
this._drain();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
_drain() {
|
||||||
|
while (this.idle.length && this.queue.length) {
|
||||||
|
const worker = this.idle.pop();
|
||||||
|
const job = this.queue.shift();
|
||||||
|
worker.currentJob = job.id;
|
||||||
|
worker.postMessage(job, [job.data]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_onMessage(worker, message) {
|
||||||
|
worker.currentJob = null;
|
||||||
|
this.idle.push(worker);
|
||||||
|
const entry = this.pending.get(message.id);
|
||||||
|
this.pending.delete(message.id);
|
||||||
|
if (entry) {
|
||||||
|
if (message.error) {
|
||||||
|
const error = new Error(message.error);
|
||||||
|
error.plain = message.plain;
|
||||||
|
entry.reject(error);
|
||||||
|
} else {
|
||||||
|
entry.resolve({ cipher: Buffer.from(message.cipher), macs: Buffer.from(message.macs), plain: message.plain });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this._drain();
|
||||||
|
}
|
||||||
|
|
||||||
|
_onError(worker, error) {
|
||||||
|
const id = worker.currentJob;
|
||||||
|
if (id && this.pending.has(id)) {
|
||||||
|
this.pending.get(id).reject(error);
|
||||||
|
this.pending.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let sharedPool = null;
|
||||||
|
let sharedSlots = null;
|
||||||
|
const lineStats = { bestRate: 0, peakAggregate: 0, active: new Map() };
|
||||||
|
|
||||||
|
function cryptoPool() {
|
||||||
|
if (!sharedPool) sharedPool = new CryptoPool(Math.max(1, Math.min(6, os.cpus().length - 1)));
|
||||||
|
return sharedPool;
|
||||||
|
}
|
||||||
|
|
||||||
|
function chunkSlots() {
|
||||||
|
if (!sharedSlots) sharedSlots = new Semaphore(DEFAULTS.maxInflightChunks);
|
||||||
|
return sharedSlots;
|
||||||
|
}
|
||||||
|
|
||||||
|
class PartFailure extends Error {
|
||||||
|
constructor(message, details = {}) {
|
||||||
|
super(message);
|
||||||
|
this.details = details;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function megaCode(body) {
|
||||||
|
const text = body.toString('utf8').trim();
|
||||||
|
return /^-\d{1,3}$/.test(text) ? Number(text) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function bodyStream(buffer, onSent) {
|
||||||
|
return Readable.from((function* slices() {
|
||||||
|
for (let offset = 0; offset < buffer.length; offset += MIB) {
|
||||||
|
const slice = buffer.subarray(offset, Math.min(offset + MIB, buffer.length));
|
||||||
|
onSent(slice.length);
|
||||||
|
yield slice;
|
||||||
|
}
|
||||||
|
})(), { objectMode: false });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readFully(handle, target, length, position) {
|
||||||
|
let done = 0;
|
||||||
|
while (done < length) {
|
||||||
|
const { bytesRead } = await handle.read(target, done, length - done, position + done);
|
||||||
|
if (bytesRead <= 0) throw new Error(`Clouddrop: Datei endet unerwartet bei Byte ${position + done}`);
|
||||||
|
done += bytesRead;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function leaseError(prefix, reply) {
|
||||||
|
const payload = reply.payload;
|
||||||
|
const error = new Error(`Clouddrop: ${prefix}${payload && payload.error ? `: ${payload.error}` : ` (HTTP ${reply.status})`}`);
|
||||||
|
error.status = reply.status;
|
||||||
|
if (payload && typeof payload.code === 'string') error.code = payload.code;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
class LeaseClient {
|
||||||
|
constructor({ apiKey, baseUrl, dispatcher, userAgent, sleep, now }) {
|
||||||
|
this.apiKey = apiKey;
|
||||||
|
this.base = `${baseUrl}${LEASE_PATH}`;
|
||||||
|
this.dispatcher = dispatcher;
|
||||||
|
this.userAgent = userAgent;
|
||||||
|
this.sleep = sleep || ((ms, signal) => delay(ms, undefined, { signal }));
|
||||||
|
this.now = now || Date.now;
|
||||||
|
}
|
||||||
|
|
||||||
|
async call(method, route, body, { signal, retries = 4, timeoutMs = DEFAULTS.apiTimeoutMs } = {}) {
|
||||||
|
for (let attempt = 0; ; attempt++) {
|
||||||
|
let reply;
|
||||||
|
try {
|
||||||
|
const res = await request(`${this.base}${route}`, {
|
||||||
|
method,
|
||||||
|
dispatcher: this.dispatcher,
|
||||||
|
signal,
|
||||||
|
headers: {
|
||||||
|
'Authorization': `Bearer ${this.apiKey}`,
|
||||||
|
'User-Agent': this.userAgent,
|
||||||
|
'Accept': 'application/json',
|
||||||
|
...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
|
||||||
|
},
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
headersTimeout: timeoutMs,
|
||||||
|
bodyTimeout: timeoutMs,
|
||||||
|
});
|
||||||
|
const text = await res.body.text();
|
||||||
|
let payload = null;
|
||||||
|
try { payload = text ? JSON.parse(text) : null; } catch {}
|
||||||
|
reply = { status: res.statusCode, payload };
|
||||||
|
} catch (error) {
|
||||||
|
if (signal && signal.aborted) throw error;
|
||||||
|
if (attempt >= retries) { error.transientNetwork = true; throw error; }
|
||||||
|
await this.sleep(Math.min(30_000, 1000 * 2 ** attempt), signal);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (reply.status >= 200 && reply.status < 300) return reply.payload;
|
||||||
|
const transient = reply.status >= 500 && reply.status !== 507 && !(reply.payload && reply.payload.code === 'MEGA_NODE_FAILED');
|
||||||
|
if (transient && attempt < retries) {
|
||||||
|
await this.sleep(Math.min(30_000, 1000 * 2 ** attempt), signal);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const error = leaseError('Direkt-Upload abgelehnt', reply);
|
||||||
|
if (transient || reply.status === 507) error.transientNetwork = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function postChunk({ url, cipher, signal, timeoutMs, onSent }) {
|
||||||
|
const res = await request(url, {
|
||||||
|
method: 'POST',
|
||||||
|
dispatcher: megaAgent,
|
||||||
|
signal,
|
||||||
|
body: bodyStream(cipher, onSent),
|
||||||
|
headers: { 'Content-Type': 'application/octet-stream', 'Content-Length': String(cipher.length) },
|
||||||
|
headersTimeout: timeoutMs,
|
||||||
|
bodyTimeout: timeoutMs,
|
||||||
|
});
|
||||||
|
const body = Buffer.from(await res.body.arrayBuffer());
|
||||||
|
return { status: res.statusCode, body };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadViaLease({ apiKey, baseUrl, dispatcher, userAgent, filePath, fileName, fileSize, progressCb, signal, throttle, options = {}, sleep, now }) {
|
||||||
|
const settings = { ...DEFAULTS, ...options };
|
||||||
|
const throttled = Boolean(throttle) && !(typeof throttle.maxBps === 'number' && throttle.maxBps <= 0);
|
||||||
|
const minStartRate = throttled ? 0 : settings.minStartRate;
|
||||||
|
const api = new LeaseClient({ apiKey, baseUrl, dispatcher, userAgent, sleep, now });
|
||||||
|
const started = performance.now();
|
||||||
|
const lease = await api.call('POST', '/upload/init', {
|
||||||
|
filename: fileName,
|
||||||
|
size: fileSize,
|
||||||
|
mode: 'rename',
|
||||||
|
client: { tool: 'multi-hoster-uploader', version: options.appVersion || null, host: os.hostname(), settings: { connections: settings.connections, chunkSize: settings.chunkSize } },
|
||||||
|
}, { signal, retries: 2 });
|
||||||
|
const leaseId = lease.leaseId;
|
||||||
|
const events = [];
|
||||||
|
const event = (type, data) => events.push({ kind: 'event', type, t: Date.now(), ms: Math.round(performance.now() - started), ...data });
|
||||||
|
const handle = await fs.promises.open(filePath, 'r');
|
||||||
|
const pool = cryptoPool();
|
||||||
|
const slots = chunkSlots();
|
||||||
|
const fullHash = crypto.createHash('sha256');
|
||||||
|
const avoidHosts = new Set();
|
||||||
|
let hashedUpTo = 0;
|
||||||
|
let doneBytes = 0;
|
||||||
|
let slowRestarts = 0;
|
||||||
|
let retries = 0;
|
||||||
|
const report = (partAcked) => { if (progressCb) progressCb(Math.min(fileSize, doneBytes + partAcked), fileSize); };
|
||||||
|
|
||||||
|
async function sendPart(part, target) {
|
||||||
|
const key = d64(target.fileKey);
|
||||||
|
const aesKey = key.subarray(0, 16);
|
||||||
|
const nonce = key.subarray(16, 24);
|
||||||
|
const plan = planChunks(part.size, settings.chunkSize);
|
||||||
|
const macs = new Array(plan.chunks.length);
|
||||||
|
const partHash = crypto.createHash('sha256');
|
||||||
|
const partAbort = new AbortController();
|
||||||
|
const partSignal = signal ? AbortSignal.any([signal, partAbort.signal]) : partAbort.signal;
|
||||||
|
const local = new Semaphore(settings.connections);
|
||||||
|
const uploadStarted = performance.now();
|
||||||
|
const history = [];
|
||||||
|
let sent = 0;
|
||||||
|
let acked = 0;
|
||||||
|
let completionHandle = null;
|
||||||
|
let partError = null;
|
||||||
|
const fail = (error) => { if (!partError) { partError = error; partAbort.abort(error); } };
|
||||||
|
const monitor = setInterval(() => {
|
||||||
|
const at = performance.now();
|
||||||
|
history.push({ at, sent });
|
||||||
|
while (history.length && at - history[0].at > settings.windowMs * 2) history.shift();
|
||||||
|
const from = history.find(item => at - item.at <= settings.windowMs * 1.25 + 250) || history[0];
|
||||||
|
const seconds = (at - from.at) / 1000;
|
||||||
|
if (seconds < settings.windowMs / 2000) return;
|
||||||
|
const rate = (sent - from.sent) / seconds;
|
||||||
|
lineStats.active.set(leaseId, rate);
|
||||||
|
const aggregate = [...lineStats.active.values()].reduce((sum, value) => sum + value, 0);
|
||||||
|
lineStats.peakAggregate = Math.max(lineStats.peakAggregate, aggregate);
|
||||||
|
if (at - uploadStarted < settings.warmupMs || throttled) return;
|
||||||
|
if (slowRestarts >= settings.maxSlowRestarts || acked >= part.size * 0.15) return;
|
||||||
|
const lineFree = lineStats.active.size <= 1 || aggregate < lineStats.peakAggregate * 0.6;
|
||||||
|
const absoluteSlow = minStartRate > 0 && rate < minStartRate;
|
||||||
|
const relativeSlow = lineStats.bestRate > 0 && rate < lineStats.bestRate * settings.relativeStartRatio;
|
||||||
|
if ((absoluteSlow || relativeSlow) && lineFree) {
|
||||||
|
event('part_slow', {
|
||||||
|
part: part.index, host: target.host, rateBytesPerSecond: Math.round(rate), bestBytesPerSecond: Math.round(lineStats.bestRate),
|
||||||
|
rule: absoluteSlow ? 'slow-start' : 'slow-start-relative',
|
||||||
|
});
|
||||||
|
fail(new PartFailure(`Clouddrop: Langsamer MEGA-Server ${target.host}`, { kind: 'slow' }));
|
||||||
|
}
|
||||||
|
}, 1000);
|
||||||
|
monitor.unref?.();
|
||||||
|
const running = [];
|
||||||
|
try {
|
||||||
|
for (const chunk of plan.chunks) {
|
||||||
|
await local.acquire(partSignal).catch(error => fail(error));
|
||||||
|
if (partError) break;
|
||||||
|
await slots.acquire(partSignal).catch(error => fail(error));
|
||||||
|
if (partError) { local.release(); break; }
|
||||||
|
let plain = new ArrayBuffer(chunk.length);
|
||||||
|
const view = Buffer.from(plain, 0, chunk.length);
|
||||||
|
try {
|
||||||
|
await readFully(handle, view, chunk.length, part.offset + chunk.offset);
|
||||||
|
} catch (error) {
|
||||||
|
slots.release();
|
||||||
|
local.release();
|
||||||
|
fail(error);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
partHash.update(view);
|
||||||
|
if (part.offset + chunk.offset === hashedUpTo) {
|
||||||
|
fullHash.update(view);
|
||||||
|
hashedUpTo += chunk.length;
|
||||||
|
}
|
||||||
|
running.push((async () => {
|
||||||
|
try {
|
||||||
|
const encrypted = await pool.encrypt({ aesKey, nonce, offset: chunk.offset, data: plain, length: chunk.length, segments: chunk.segments });
|
||||||
|
plain = encrypted.plain;
|
||||||
|
for (let tries = 1; ; tries++) {
|
||||||
|
if (partSignal.aborted) throw partSignal.reason;
|
||||||
|
if (throttle) await throttle.consume(chunk.length, partSignal);
|
||||||
|
let result = null;
|
||||||
|
let failure = null;
|
||||||
|
let sentThisTry = 0;
|
||||||
|
try {
|
||||||
|
result = await postChunk({
|
||||||
|
url: `${target.uploadUrl}/${chunk.offset}`,
|
||||||
|
cipher: encrypted.cipher,
|
||||||
|
signal: partSignal,
|
||||||
|
timeoutMs: settings.requestTimeoutMs,
|
||||||
|
onSent: (bytes) => { sent += bytes; sentThisTry += bytes; },
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (partSignal.aborted) throw partSignal.reason;
|
||||||
|
failure = { kind: 'network', message: error.code || error.message };
|
||||||
|
}
|
||||||
|
if (!failure && result.status !== 200) failure = { kind: 'http', message: `HTTP ${result.status}` };
|
||||||
|
const code = !failure && result.body.length ? megaCode(result.body) : null;
|
||||||
|
if (!failure && code !== null) failure = { kind: 'mega', code, message: `MEGA ${code}` };
|
||||||
|
if (!failure) {
|
||||||
|
if (result.body.length) completionHandle = e64(result.body);
|
||||||
|
macs[chunk.index] = encrypted.macs;
|
||||||
|
acked += chunk.length;
|
||||||
|
report(acked);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sent -= sentThisTry;
|
||||||
|
retries++;
|
||||||
|
event('chunk_retry', { part: part.index, chunk: chunk.index, try: tries, ...failure });
|
||||||
|
if (failure.kind === 'mega' && !RETRYABLE_MEGA_CODES.has(failure.code)) throw new PartFailure(`Clouddrop: MEGA-Fehler ${failure.code}`, failure);
|
||||||
|
if (tries > settings.maxChunkRetries) throw new PartFailure(`Clouddrop: Anfrage an MEGA nach ${tries} Versuchen fehlgeschlagen`, failure);
|
||||||
|
await delay(Math.min(30_000, 500 * 2 ** tries), undefined, { signal: partSignal });
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
fail(error);
|
||||||
|
} finally {
|
||||||
|
slots.release();
|
||||||
|
local.release();
|
||||||
|
}
|
||||||
|
})());
|
||||||
|
}
|
||||||
|
await Promise.all(running);
|
||||||
|
} finally {
|
||||||
|
clearInterval(monitor);
|
||||||
|
lineStats.active.delete(leaseId);
|
||||||
|
}
|
||||||
|
if (partError) { partError.ackedBytes = acked; throw partError; }
|
||||||
|
if (!completionHandle) throw new PartFailure('Clouddrop: MEGA hat keine Abschluss-Kennung geliefert');
|
||||||
|
return {
|
||||||
|
completionHandle,
|
||||||
|
mac: e64(condenseMacs(aesKey, macs, { trailingMac: plan.trailingMac, nonce })),
|
||||||
|
sha256: partHash.digest('hex'),
|
||||||
|
uploadMs: performance.now() - uploadStarted,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function completePart(part, body) {
|
||||||
|
for (let i = 0; ; i++) {
|
||||||
|
try {
|
||||||
|
return await api.call('POST', `/upload/${leaseId}/parts/${part.index}/complete`, body, { signal });
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === 'PART_BUSY' && i < 24) { await api.sleep(5000, signal); continue; }
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const part of lease.parts) {
|
||||||
|
let reason = null;
|
||||||
|
for (let attempt = 1; ; attempt++) {
|
||||||
|
const body = { avoidHosts: [...avoidHosts] };
|
||||||
|
if (reason) body.reason = reason;
|
||||||
|
const target = await api.call('POST', `/upload/${leaseId}/parts/${part.index}/target`, body, { signal, retries: 2 });
|
||||||
|
target.host = new URL(target.uploadUrl).host;
|
||||||
|
event('target', { part: part.index, attempt: target.attempt, host: target.host, server: target.timings });
|
||||||
|
try {
|
||||||
|
const sentPart = await sendPart(part, target);
|
||||||
|
await completePart(part, {
|
||||||
|
attempt: target.attempt,
|
||||||
|
completionHandle: sentPart.completionHandle,
|
||||||
|
mac: sentPart.mac,
|
||||||
|
sha256: sentPart.sha256,
|
||||||
|
bytes: part.size,
|
||||||
|
stats: { uploadMs: Math.round(sentPart.uploadMs), host: target.host },
|
||||||
|
});
|
||||||
|
doneBytes += part.size;
|
||||||
|
report(0);
|
||||||
|
if (!throttled && part.size >= 64 * MIB && sentPart.uploadMs > 0) {
|
||||||
|
lineStats.bestRate = Math.max(lineStats.bestRate, part.size / (sentPart.uploadMs / 1000));
|
||||||
|
}
|
||||||
|
event('part_done', { part: part.index, attempt: target.attempt, bytes: part.size, uploadMs: Math.round(sentPart.uploadMs), bytesPerSecond: Math.round(part.size / (sentPart.uploadMs / 1000)), host: target.host });
|
||||||
|
break;
|
||||||
|
} catch (error) {
|
||||||
|
if (signal && signal.aborted) throw error;
|
||||||
|
const slow = error instanceof PartFailure && error.details.kind === 'slow';
|
||||||
|
const retryable = error instanceof PartFailure || error.code === 'MEGA_NODE_FAILED';
|
||||||
|
if (slow) { slowRestarts++; avoidHosts.add(target.host); }
|
||||||
|
report(0);
|
||||||
|
reason = String(error.message || error).slice(0, 180);
|
||||||
|
event(slow ? 'part_retarget_slow' : 'part_failed', { part: part.index, attempt: target.attempt, error: reason });
|
||||||
|
if (!retryable || attempt >= settings.maxPartAttempts) {
|
||||||
|
if (retryable) error.transientNetwork = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (!slow) await api.sleep(2000 * attempt, signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const final = await api.call('POST', `/upload/${leaseId}/finalize`, { sha256: fullHash.digest('hex') }, { signal });
|
||||||
|
const durationMs = performance.now() - started;
|
||||||
|
events.push({
|
||||||
|
kind: 'summary', t: Date.now(), durationMs: Math.round(durationMs), ackedBytes: fileSize,
|
||||||
|
averageBytesPerSecond: Math.round(fileSize / (durationMs / 1000)), retries, errors: 0, slowRetargets: slowRestarts,
|
||||||
|
settings: { connections: settings.connections, chunkSize: settings.chunkSize, partSize: lease.partBytes, parallelParts: 1 },
|
||||||
|
host: { name: os.hostname(), platform: process.platform, cpus: os.cpus().length, node: process.version },
|
||||||
|
});
|
||||||
|
api.call('POST', `/upload/${leaseId}/telemetry`, { records: events }, { retries: 1 }).catch(() => {});
|
||||||
|
return final.fileId;
|
||||||
|
} catch (error) {
|
||||||
|
if (error && typeof error === 'object') error.leaseStarted = true;
|
||||||
|
events.push({ kind: 'event', type: 'failed', t: Date.now(), error: String(error.message || error).slice(0, 200) });
|
||||||
|
api.call('POST', `/upload/${leaseId}/telemetry`, { records: events }, { retries: 0 }).catch(() => {});
|
||||||
|
api.call('POST', `/upload/${leaseId}/abort`, { reason: String(error.message || error).slice(0, 180) }, { retries: 1 }).catch(() => {});
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
await handle.close().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { uploadViaLease, DEFAULTS, PartFailure, _megaAgent: megaAgent, _lineStats: lineStats };
|
||||||
@@ -4,6 +4,8 @@ const path = require('path');
|
|||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const { setTimeout: delay } = require('node:timers/promises');
|
const { setTimeout: delay } = require('node:timers/promises');
|
||||||
const { request, Agent } = require('undici');
|
const { request, Agent } = require('undici');
|
||||||
|
const { uploadViaLease } = require('./clouddrop-lease');
|
||||||
|
const { version: APP_VERSION } = require('../package.json');
|
||||||
|
|
||||||
const BASE_URL = 'https://clouddrop.cc';
|
const BASE_URL = 'https://clouddrop.cc';
|
||||||
const API_BASE = `${BASE_URL}/api/cloud`;
|
const API_BASE = `${BASE_URL}/api/cloud`;
|
||||||
@@ -34,6 +36,10 @@ const JOB_POLL_MAX_INTERVAL_MS = 30_000;
|
|||||||
const JOB_POLL_ERROR_MAX_DELAY_MS = 60_000;
|
const JOB_POLL_ERROR_MAX_DELAY_MS = 60_000;
|
||||||
const JOB_POLL_MAX_MS = 24 * 60 * 60_000;
|
const JOB_POLL_MAX_MS = 24 * 60 * 60_000;
|
||||||
const EXPRESS_REUPLOAD_ERROR = /^dead:(?:express_[a-z_]+|relay_chunks_missing)$/;
|
const EXPRESS_REUPLOAD_ERROR = /^dead:(?:express_[a-z_]+|relay_chunks_missing)$/;
|
||||||
|
const LEASE_ENABLED_CACHE_MS = 5 * 60_000;
|
||||||
|
const LEASE_DISABLED_CACHE_MS = 60_000;
|
||||||
|
const LEASE_FALLBACK_CODES = new Set(['MEGA_LEASE_DISABLED']);
|
||||||
|
const leaseAvailability = new Map();
|
||||||
|
|
||||||
// Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under
|
// Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under
|
||||||
// the server's per-IP limit of 100 concurrent connections (cd_conn).
|
// the server's per-IP limit of 100 concurrent connections (cd_conn).
|
||||||
@@ -97,6 +103,62 @@ class ClouddropUploader {
|
|||||||
this.dispatcher = options.dispatcher || clouddropAgent;
|
this.dispatcher = options.dispatcher || clouddropAgent;
|
||||||
this.sleep = options.sleep || ((ms, signal) => delay(ms, undefined, { signal }));
|
this.sleep = options.sleep || ((ms, signal) => delay(ms, undefined, { signal }));
|
||||||
this.now = options.now || Date.now;
|
this.now = options.now || Date.now;
|
||||||
|
this.leaseOptions = options.leaseOptions || {};
|
||||||
|
this.leaseDisabled = options.lease === false || process.env.MHU_CLOUDDROP_LEASE === '0';
|
||||||
|
}
|
||||||
|
|
||||||
|
_leaseCacheKey() {
|
||||||
|
return sha256Hex(Buffer.from(this.apiKey)).slice(0, 24);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _leaseEnabled(signal) {
|
||||||
|
if (this.leaseDisabled) return false;
|
||||||
|
const key = this._leaseCacheKey();
|
||||||
|
const cached = leaseAvailability.get(key);
|
||||||
|
if (cached && cached.until > this.now()) return cached.enabled;
|
||||||
|
let enabled = false;
|
||||||
|
try {
|
||||||
|
const res = await request(`${API_BASE}/mega-lease/config`, {
|
||||||
|
method: 'GET',
|
||||||
|
dispatcher: this.dispatcher,
|
||||||
|
signal,
|
||||||
|
headers: this._headers(),
|
||||||
|
headersTimeout: 15_000,
|
||||||
|
bodyTimeout: 15_000
|
||||||
|
});
|
||||||
|
const reply = await this._readReply(res);
|
||||||
|
enabled = reply.ok && Boolean(reply.payload) && reply.payload.enabled === true;
|
||||||
|
} catch (error) {
|
||||||
|
if (signal && signal.aborted) throw error;
|
||||||
|
}
|
||||||
|
leaseAvailability.set(key, { enabled, until: this.now() + (enabled ? LEASE_ENABLED_CACHE_MS : LEASE_DISABLED_CACHE_MS) });
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
async _uploadLease(filePath, fileName, fileSize, progressCb, signal, throttle) {
|
||||||
|
try {
|
||||||
|
return await uploadViaLease({
|
||||||
|
apiKey: this.apiKey,
|
||||||
|
baseUrl: BASE_URL,
|
||||||
|
dispatcher: this.dispatcher,
|
||||||
|
userAgent: USER_AGENT,
|
||||||
|
filePath,
|
||||||
|
fileName,
|
||||||
|
fileSize,
|
||||||
|
progressCb,
|
||||||
|
signal,
|
||||||
|
throttle,
|
||||||
|
sleep: this.sleep,
|
||||||
|
now: this.now,
|
||||||
|
options: { appVersion: APP_VERSION, ...this.leaseOptions }
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (error && (LEASE_FALLBACK_CODES.has(error.code) || error.status === 404) && !error.leaseStarted) {
|
||||||
|
leaseAvailability.set(this._leaseCacheKey(), { enabled: false, until: this.now() + LEASE_DISABLED_CACHE_MS });
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_headers(extra) {
|
_headers(extra) {
|
||||||
@@ -178,6 +240,10 @@ class ClouddropUploader {
|
|||||||
if (fileSize <= SIMPLE_UPLOAD_LIMIT) {
|
if (fileSize <= SIMPLE_UPLOAD_LIMIT) {
|
||||||
return shareResult(await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle));
|
return shareResult(await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle));
|
||||||
}
|
}
|
||||||
|
if (await this._leaseEnabled(signal)) {
|
||||||
|
const fileId = await this._uploadLease(filePath, fileName, fileSize, progressCb, signal, throttle);
|
||||||
|
if (fileId) return shareResult(fileId);
|
||||||
|
}
|
||||||
const outcome = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle);
|
const outcome = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle);
|
||||||
return typeof outcome === 'string' ? shareResult(outcome) : outcome;
|
return typeof outcome === 'string' ? shareResult(outcome) : outcome;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -828,6 +828,7 @@
|
|||||||
['Verlauf und Aufbewahrung konnten nicht konsistent gespeichert werden', 'History and retention could not be saved consistently'],
|
['Verlauf und Aufbewahrung konnten nicht konsistent gespeichert werden', 'History and retention could not be saved consistently'],
|
||||||
['Kein Ordnerpfad angegeben', 'No folder path was provided'],
|
['Kein Ordnerpfad angegeben', 'No folder path was provided'],
|
||||||
['Clouddrop: Datei ist leer', 'Clouddrop: The file is empty'],
|
['Clouddrop: Datei ist leer', 'Clouddrop: The file is empty'],
|
||||||
|
['Clouddrop: MEGA hat keine Abschluss-Kennung geliefert', 'Clouddrop: MEGA did not return a completion handle'],
|
||||||
['Clouddrop: Keine fileId in Upload-Antwort', 'Clouddrop: No fileId in the upload response'],
|
['Clouddrop: Keine fileId in Upload-Antwort', 'Clouddrop: No fileId in the upload response'],
|
||||||
['Clouddrop: Keine sessionId von /upload/init', 'Clouddrop: No sessionId from /upload/init'],
|
['Clouddrop: Keine sessionId von /upload/init', 'Clouddrop: No sessionId from /upload/init'],
|
||||||
['Clouddrop: Keine fileId im Job-Ergebnis', 'Clouddrop: No fileId in the job result'],
|
['Clouddrop: Keine fileId im Job-Ergebnis', 'Clouddrop: No fileId in the job result'],
|
||||||
@@ -1042,6 +1043,12 @@
|
|||||||
[/^Quelldatei-Schutz konnte nicht vorbereitet werden: (.+)$/, 'Source file protection could not be prepared: $1'],
|
[/^Quelldatei-Schutz konnte nicht vorbereitet werden: (.+)$/, 'Source file protection could not be prepared: $1'],
|
||||||
[/^Clouddrop: API-Antwort war kein JSON (.+)$/, 'Clouddrop: API response was not JSON $1'],
|
[/^Clouddrop: API-Antwort war kein JSON (.+)$/, 'Clouddrop: API response was not JSON $1'],
|
||||||
[/^Clouddrop: Datei nicht lesbar: (.+)$/, 'Clouddrop: File cannot be read: $1'],
|
[/^Clouddrop: Datei nicht lesbar: (.+)$/, 'Clouddrop: File cannot be read: $1'],
|
||||||
|
[/^Clouddrop: Datei endet unerwartet bei Byte (.+)$/, 'Clouddrop: The file ends unexpectedly at byte $1'],
|
||||||
|
[/^Clouddrop: Direkt-Upload abgelehnt: (.+)$/, 'Clouddrop: Direct upload rejected: $1'],
|
||||||
|
[/^Clouddrop: Direkt-Upload abgelehnt \(HTTP (\d{3})\)$/, 'Clouddrop: Direct upload rejected (HTTP $1)'],
|
||||||
|
[/^Clouddrop: Langsamer MEGA-Server (.+)$/, 'Clouddrop: Slow MEGA server $1'],
|
||||||
|
[/^Clouddrop: MEGA-Fehler (.+)$/, 'Clouddrop: MEGA error $1'],
|
||||||
|
[/^Clouddrop: Anfrage an MEGA nach (\d+) Versuchen fehlgeschlagen$/, 'Clouddrop: Request to MEGA failed after $1 attempts'],
|
||||||
[/^Clouddrop: Verarbeitung fehlgeschlagen \((.+)\)$/,'Clouddrop: Processing failed ($1)'],
|
[/^Clouddrop: Verarbeitung fehlgeschlagen \((.+)\)$/,'Clouddrop: Processing failed ($1)'],
|
||||||
[/^Doodstream: konnte Upload-Server nicht ermitteln \(Endpoint geändert\?\)\. (.+)$/, 'Doodstream: Could not determine the upload server (endpoint changed?). $1'],
|
[/^Doodstream: konnte Upload-Server nicht ermitteln \(Endpoint geändert\?\)\. (.+)$/, 'Doodstream: Could not determine the upload server (endpoint changed?). $1'],
|
||||||
[/^Doodstream Upload fehlgeschlagen: (.+)$/, 'Doodstream upload failed: $1'],
|
[/^Doodstream Upload fehlgeschlagen: (.+)$/, 'Doodstream upload failed: $1'],
|
||||||
|
|||||||
Reference in new issue
Block a user