This commit is contained in:
1 parent
a5d6a1d5b1
commit
54750bda8f
4 files changed
+510
-31
No files matched your search
+379
-27
@@ -2,6 +2,7 @@ const fs = require('fs');
|
||||
const { statUploadFile, readUploadFile } = require('./upload-file-io');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { setTimeout: delay } = require('node:timers/promises');
|
||||
const { request, Agent } = require('undici');
|
||||
|
||||
const BASE_URL = 'https://clouddrop.cc';
|
||||
@@ -15,6 +16,24 @@ const INIT_TIMEOUT = 60_000;
|
||||
const CHUNK_TIMEOUT = 30 * 60_000; // 30 min per chunk
|
||||
const COMPLETE_TIMEOUT = 5 * 60_000;
|
||||
const SIMPLE_UPLOAD_TIMEOUT = 30 * 60_000;
|
||||
const STATUS_TIMEOUT = 60_000;
|
||||
const JOB_POLL_TIMEOUT = 60_000;
|
||||
|
||||
const RETRY_WINDOW_MS = 10 * 60_000;
|
||||
const RETRY_BASE_DELAY_MS = 1_000;
|
||||
const RETRY_MAX_DELAY_MS = 60_000;
|
||||
const RETRYABLE_STATUSES = new Set([429, 502, 503, 507]);
|
||||
const RETRYABLE_CHUNK_CODES = new Set(['CHUNK_IN_PROGRESS', 'CHUNK_HASH_MISMATCH', 'CHUNK_INCOMPLETE']);
|
||||
const TICKET_CODES = new Set(['TICKET_EXPIRED', 'TICKET_INVALID']);
|
||||
const RESTART_CODES = new Set(['UPLOAD_SESSION_GONE', 'TICKET_SESSION_MISMATCH', 'EXPRESS_RELAY_REVOKED']);
|
||||
const MAX_FILE_RESTARTS = 2;
|
||||
const MAX_TICKET_REFRESHES = 3;
|
||||
const MAX_MISSING_CHUNK_RESENDS = 3;
|
||||
const JOB_POLL_INTERVAL_MS = 5_000;
|
||||
const JOB_POLL_MAX_INTERVAL_MS = 30_000;
|
||||
const JOB_POLL_ERROR_MAX_DELAY_MS = 60_000;
|
||||
const JOB_POLL_MAX_MS = 24 * 60 * 60_000;
|
||||
const EXPRESS_REUPLOAD_ERROR = /^dead:(?:express_[a-z_]+|relay_chunks_missing)$/;
|
||||
|
||||
// Cap concurrent TCP connections to clouddrop.cc at 50 to stay well under
|
||||
// the server's per-IP limit of 100 concurrent connections (cd_conn).
|
||||
@@ -26,14 +45,58 @@ const clouddropAgent = new Agent({
|
||||
keepAliveMaxTimeout: 60_000
|
||||
});
|
||||
|
||||
function isExpressSession(initPayload) {
|
||||
return initPayload.completed === true
|
||||
|| (typeof initPayload.uploadBaseUrl === 'string' && initPayload.uploadBaseUrl !== '');
|
||||
}
|
||||
|
||||
function expressBaseUrl(value) {
|
||||
let url = null;
|
||||
try { url = new URL(String(value)); } catch {}
|
||||
if (!url || url.protocol !== 'https:' || url.username || url.password || url.search || url.hash
|
||||
|| !/(^|\.)clouddrop\.cc$/i.test(url.hostname)) {
|
||||
throw new Error('Clouddrop: Ungültige Express-Upload-Adresse');
|
||||
}
|
||||
return url.href.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
function retryAfterMs(headers) {
|
||||
const raw = headers ? headers['retry-after'] : undefined;
|
||||
const value = Array.isArray(raw) ? raw[0] : raw;
|
||||
if (value === undefined || value === null || String(value).trim() === '') return 0;
|
||||
const seconds = Number(value);
|
||||
const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(value) - Date.now();
|
||||
return Number.isFinite(ms) ? Math.min(Math.max(0, ms), RETRY_WINDOW_MS) : 0;
|
||||
}
|
||||
|
||||
function sha256Hex(buffer) {
|
||||
return crypto.createHash('sha256').update(buffer).digest('hex');
|
||||
}
|
||||
|
||||
function shareResult(fileId) {
|
||||
return {
|
||||
download_url: `${BASE_URL}/share/${fileId}`,
|
||||
embed_url: null,
|
||||
file_code: fileId
|
||||
};
|
||||
}
|
||||
|
||||
function markRestart(error) {
|
||||
error.restartFile = true;
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clouddrop.cc uploader — uses API Key (Bearer) authentication.
|
||||
* Files > 16 MB use the chunked protocol, smaller files use simple upload.
|
||||
* After upload, a share link is created and returned as download_url.
|
||||
*/
|
||||
class ClouddropUploader {
|
||||
constructor(apiKey) {
|
||||
constructor(apiKey, options = {}) {
|
||||
this.apiKey = String(apiKey || '').trim();
|
||||
this.dispatcher = options.dispatcher || clouddropAgent;
|
||||
this.sleep = options.sleep || ((ms, signal) => delay(ms, undefined, { signal }));
|
||||
this.now = options.now || Date.now;
|
||||
}
|
||||
|
||||
_headers(extra) {
|
||||
@@ -61,6 +124,46 @@ class ClouddropUploader {
|
||||
return payload;
|
||||
}
|
||||
|
||||
async _readReply(res) {
|
||||
const text = await res.body.text();
|
||||
let payload = null;
|
||||
try { payload = text ? JSON.parse(text) : {}; } catch {}
|
||||
return {
|
||||
ok: res.statusCode >= 200 && res.statusCode < 300,
|
||||
status: res.statusCode,
|
||||
headers: res.headers,
|
||||
payload: payload && typeof payload === 'object' ? payload : null,
|
||||
text
|
||||
};
|
||||
}
|
||||
|
||||
_replyError(reply) {
|
||||
const payload = reply.payload;
|
||||
const error = payload
|
||||
? new Error(`Clouddrop: ${payload.error || payload.message || `HTTP ${reply.status}`}`)
|
||||
: new Error(`Clouddrop: API-Antwort war kein JSON (HTTP ${reply.status}): ${reply.text.slice(0, 200)}`);
|
||||
error.status = reply.status;
|
||||
if (payload && typeof payload.code === 'string') error.code = payload.code;
|
||||
return error;
|
||||
}
|
||||
|
||||
_retryBudget(signal) {
|
||||
let firstFailureAt = 0;
|
||||
let failures = 0;
|
||||
return async (error, minimumDelayMs = 0) => {
|
||||
if (signal && signal.aborted) throw error;
|
||||
const now = this.now();
|
||||
if (!firstFailureAt) firstFailureAt = now;
|
||||
if (now - firstFailureAt >= RETRY_WINDOW_MS) {
|
||||
error.transientNetwork = true;
|
||||
throw error;
|
||||
}
|
||||
const backoff = Math.min(RETRY_BASE_DELAY_MS * 2 ** failures, RETRY_MAX_DELAY_MS);
|
||||
failures += 1;
|
||||
await this.sleep(Math.max(backoff, minimumDelayMs), signal);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file. Returns { download_url, embed_url, file_code }.
|
||||
*/
|
||||
@@ -72,18 +175,11 @@ class ClouddropUploader {
|
||||
catch { throw new Error(`Clouddrop: Datei nicht lesbar: ${fileName}`); }
|
||||
if (fileSize <= 0) throw new Error('Clouddrop: Datei ist leer');
|
||||
|
||||
let fileId;
|
||||
if (fileSize <= SIMPLE_UPLOAD_LIMIT) {
|
||||
fileId = await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle);
|
||||
} else {
|
||||
fileId = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle);
|
||||
return shareResult(await this._uploadSimple(filePath, fileName, fileSize, progressCb, signal, throttle));
|
||||
}
|
||||
|
||||
return {
|
||||
download_url: `${BASE_URL}/share/${fileId}`,
|
||||
embed_url: null,
|
||||
file_code: fileId
|
||||
};
|
||||
const outcome = await this._uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle);
|
||||
return typeof outcome === 'string' ? shareResult(outcome) : outcome;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -118,7 +214,7 @@ class ClouddropUploader {
|
||||
|
||||
const res = await request(`${API_BASE}/upload?mode=rename`, {
|
||||
method: 'POST',
|
||||
dispatcher: clouddropAgent,
|
||||
dispatcher: this.dispatcher,
|
||||
body: generate(),
|
||||
signal,
|
||||
headers: this._headers({
|
||||
@@ -134,22 +230,44 @@ class ClouddropUploader {
|
||||
return payload.fileId;
|
||||
}
|
||||
|
||||
async _uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle) {
|
||||
for (let restarts = 0; ; restarts++) {
|
||||
const initPayload = await this._initSession(fileName, fileSize, restarts > 0, signal);
|
||||
if (!isExpressSession(initPayload)) {
|
||||
return this._uploadChunksToOvh(initPayload, filePath, fileSize, progressCb, signal, throttle);
|
||||
}
|
||||
try {
|
||||
return await this._uploadExpress(initPayload, filePath, fileSize, progressCb, signal, throttle);
|
||||
} catch (error) {
|
||||
if (!error || error.restartFile !== true || (signal && signal.aborted)) throw error;
|
||||
if (restarts >= MAX_FILE_RESTARTS) {
|
||||
error.transientNetwork = true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _initSession(fileName, fileSize, forceNew, signal) {
|
||||
const body = { filename: fileName, size: fileSize, parentId: null, relayCapable: true };
|
||||
if (forceNew) body.forceNew = true;
|
||||
const initRes = await request(`${API_BASE}/upload/init`, {
|
||||
method: 'POST',
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers({ 'Content-Type': 'application/json' }),
|
||||
body: JSON.stringify(body),
|
||||
headersTimeout: INIT_TIMEOUT,
|
||||
bodyTimeout: INIT_TIMEOUT
|
||||
});
|
||||
return this._parseJsonResponse(initRes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunked upload for files > 16 MB.
|
||||
* Flow: POST /upload/init → PUT /upload/:sessionId/chunk/:n (0-based) → POST /upload/:sessionId/complete
|
||||
*/
|
||||
async _uploadChunked(filePath, fileName, fileSize, progressCb, signal, throttle) {
|
||||
// 1. Init session
|
||||
const initRes = await request(`${API_BASE}/upload/init`, {
|
||||
method: 'POST',
|
||||
dispatcher: clouddropAgent,
|
||||
signal,
|
||||
headers: this._headers({ 'Content-Type': 'application/json' }),
|
||||
body: JSON.stringify({ filename: fileName, size: fileSize, parentId: null }),
|
||||
headersTimeout: INIT_TIMEOUT,
|
||||
bodyTimeout: INIT_TIMEOUT
|
||||
});
|
||||
const initPayload = await this._parseJsonResponse(initRes);
|
||||
async _uploadChunksToOvh(initPayload, filePath, fileSize, progressCb, signal, throttle) {
|
||||
const sessionId = initPayload.sessionId;
|
||||
const chunkSize = initPayload.chunkSize || CHUNK_SIZE;
|
||||
const totalChunks = initPayload.totalChunks || Math.ceil(fileSize / chunkSize);
|
||||
@@ -178,7 +296,7 @@ class ClouddropUploader {
|
||||
|
||||
const chunkRes = await request(`${CHUNK_UPLOAD_BASE}/upload/${sessionId}/chunk/${i}`, {
|
||||
method: 'PUT',
|
||||
dispatcher: clouddropAgent,
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
body,
|
||||
headers: this._headers({
|
||||
@@ -204,7 +322,7 @@ class ClouddropUploader {
|
||||
try {
|
||||
const completeRes = await request(`${API_BASE}/upload/${sessionId}/complete`, {
|
||||
method: 'POST',
|
||||
dispatcher: clouddropAgent,
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers({ 'Content-Type': 'application/json' }),
|
||||
body: '{}',
|
||||
@@ -218,6 +336,240 @@ class ClouddropUploader {
|
||||
}
|
||||
}
|
||||
|
||||
async _uploadExpress(initPayload, filePath, fileSize, progressCb, signal, throttle) {
|
||||
const sessionId = initPayload.sessionId;
|
||||
if (!sessionId) throw new Error('Clouddrop: Keine sessionId von /upload/init');
|
||||
const pending = this._pendingJob(initPayload.jobId || sessionId);
|
||||
if (initPayload.completed === true) return pending;
|
||||
|
||||
const route = {
|
||||
sessionId,
|
||||
baseUrl: expressBaseUrl(initPayload.uploadBaseUrl),
|
||||
ticket: typeof initPayload.uploadTicket === 'string' ? initPayload.uploadTicket : ''
|
||||
};
|
||||
if (!route.ticket) await this._refreshTicket(route, signal);
|
||||
const chunkSize = initPayload.chunkSize || CHUNK_SIZE;
|
||||
const totalChunks = initPayload.totalChunks || Math.ceil(fileSize / chunkSize);
|
||||
|
||||
const fh = await fs.promises.open(filePath, 'r');
|
||||
const buffer = Buffer.allocUnsafe(chunkSize);
|
||||
const readChunk = async (index) => {
|
||||
const length = Math.min(chunkSize, fileSize - index * chunkSize);
|
||||
await fh.read(buffer, 0, length, index * chunkSize);
|
||||
return buffer.subarray(0, length);
|
||||
};
|
||||
try {
|
||||
const fileHash = crypto.createHash('sha256');
|
||||
let bytesSent = 0;
|
||||
for (let index = 0; index < totalChunks; index++) {
|
||||
if (signal && signal.aborted) throw new Error('Aborted');
|
||||
const body = await readChunk(index);
|
||||
fileHash.update(body);
|
||||
if (await this._putExpressChunk(route, index, body, signal, throttle) === 'sealed') return pending;
|
||||
bytesSent += body.length;
|
||||
if (progressCb) progressCb(bytesSent, fileSize);
|
||||
}
|
||||
const clientSha256 = fileHash.digest('hex');
|
||||
for (let resends = 0; ; resends++) {
|
||||
const reply = await this._postComplete(sessionId, clientSha256, signal);
|
||||
if (reply.ok) return pending;
|
||||
const error = this._replyError(reply);
|
||||
const missing = reply.payload && Array.isArray(reply.payload.missing) ? reply.payload.missing : [];
|
||||
const resendable = error.code === 'EXPRESS_CHUNK_ACKS_INCOMPLETE'
|
||||
&& missing.length > 0
|
||||
&& missing.every(index => Number.isInteger(index) && index >= 0 && index < totalChunks);
|
||||
if (resendable && resends < MAX_MISSING_CHUNK_RESENDS) {
|
||||
for (const index of missing) {
|
||||
if (await this._putExpressChunk(route, index, await readChunk(index), signal, throttle) === 'sealed') return pending;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (reply.status === 410 || RESTART_CODES.has(error.code)) throw markRestart(error);
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
try { await fh.close(); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
async _putExpressChunk(route, index, body, signal, throttle) {
|
||||
const chunkSha256 = sha256Hex(body);
|
||||
const retry = this._retryBudget(signal);
|
||||
let ticketRefreshes = 0;
|
||||
for (;;) {
|
||||
if (throttle) await throttle.consume(body.length, signal);
|
||||
let reply;
|
||||
try {
|
||||
const res = await request(`${route.baseUrl}/upload/${route.sessionId}/chunk/${index}`, {
|
||||
method: 'PUT',
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
body,
|
||||
headers: {
|
||||
'Authorization': `Bearer ${route.ticket}`,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Content-Length': String(body.length),
|
||||
'X-Chunk-SHA256': chunkSha256
|
||||
},
|
||||
headersTimeout: CHUNK_TIMEOUT,
|
||||
bodyTimeout: CHUNK_TIMEOUT
|
||||
});
|
||||
reply = await this._readReply(res);
|
||||
} catch (error) {
|
||||
await retry(error);
|
||||
continue;
|
||||
}
|
||||
if (reply.ok) return 'stored';
|
||||
const error = this._replyError(reply);
|
||||
if (error.code === 'UPLOAD_SESSION_SEALED') return 'sealed';
|
||||
if (reply.status === 401 && TICKET_CODES.has(error.code)) {
|
||||
if (ticketRefreshes >= MAX_TICKET_REFRESHES) throw markRestart(error);
|
||||
ticketRefreshes += 1;
|
||||
await this._refreshTicket(route, signal);
|
||||
continue;
|
||||
}
|
||||
if (error.code === 'CHUNK_ALREADY_COMMITTED') {
|
||||
error.fileRejected = true;
|
||||
throw error;
|
||||
}
|
||||
if (reply.status === 410 || RESTART_CODES.has(error.code)) throw markRestart(error);
|
||||
if (RETRYABLE_STATUSES.has(reply.status) || RETRYABLE_CHUNK_CODES.has(error.code)) {
|
||||
await retry(error, retryAfterMs(reply.headers));
|
||||
continue;
|
||||
}
|
||||
error.transientNetwork = true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async _refreshTicket(route, signal) {
|
||||
const retry = this._retryBudget(signal);
|
||||
for (;;) {
|
||||
let reply;
|
||||
try {
|
||||
const res = await request(`${API_BASE}/upload/${route.sessionId}/status`, {
|
||||
method: 'GET',
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers(),
|
||||
headersTimeout: STATUS_TIMEOUT,
|
||||
bodyTimeout: STATUS_TIMEOUT
|
||||
});
|
||||
reply = await this._readReply(res);
|
||||
} catch (error) {
|
||||
await retry(error);
|
||||
continue;
|
||||
}
|
||||
if (reply.ok && reply.payload) {
|
||||
const { uploadTicket, uploadBaseUrl } = reply.payload;
|
||||
if (typeof uploadTicket !== 'string' || uploadTicket === '') {
|
||||
throw markRestart(new Error('Clouddrop: Kein Express-Upload-Ticket erhalten'));
|
||||
}
|
||||
if (uploadBaseUrl) route.baseUrl = expressBaseUrl(uploadBaseUrl);
|
||||
route.ticket = uploadTicket;
|
||||
return;
|
||||
}
|
||||
const error = this._replyError(reply);
|
||||
if (reply.status === 404 || reply.status === 410 || RESTART_CODES.has(error.code)) throw markRestart(error);
|
||||
if (RETRYABLE_STATUSES.has(reply.status)) {
|
||||
await retry(error, retryAfterMs(reply.headers));
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async _postComplete(sessionId, clientSha256, signal) {
|
||||
const retry = this._retryBudget(signal);
|
||||
for (;;) {
|
||||
let reply;
|
||||
try {
|
||||
const res = await request(`${API_BASE}/upload/${sessionId}/complete`, {
|
||||
method: 'POST',
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers({ 'Content-Type': 'application/json' }),
|
||||
body: JSON.stringify({ clientSha256 }),
|
||||
headersTimeout: COMPLETE_TIMEOUT,
|
||||
bodyTimeout: COMPLETE_TIMEOUT
|
||||
});
|
||||
reply = await this._readReply(res);
|
||||
} catch (error) {
|
||||
await retry(error);
|
||||
continue;
|
||||
}
|
||||
if (!RETRYABLE_STATUSES.has(reply.status)) return reply;
|
||||
await retry(this._replyError(reply), retryAfterMs(reply.headers));
|
||||
}
|
||||
}
|
||||
|
||||
_pendingJob(jobId) {
|
||||
return { clouddropJob: { jobId, wait: (signal, onState) => this.waitForJob(jobId, signal, onState) } };
|
||||
}
|
||||
|
||||
async waitForJob(jobId, signal, onState) {
|
||||
const startedAt = this.now();
|
||||
let interval = JOB_POLL_INTERVAL_MS;
|
||||
let failures = 0;
|
||||
for (;;) {
|
||||
let payload = null;
|
||||
try {
|
||||
const res = await request(`${API_BASE}/upload/jobs/${encodeURIComponent(jobId)}`, {
|
||||
method: 'GET',
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers(),
|
||||
headersTimeout: JOB_POLL_TIMEOUT,
|
||||
bodyTimeout: JOB_POLL_TIMEOUT
|
||||
});
|
||||
const reply = await this._readReply(res);
|
||||
if (reply.ok) payload = reply.payload;
|
||||
} catch (error) {
|
||||
if (signal && signal.aborted) throw error;
|
||||
}
|
||||
let waitMs;
|
||||
if (payload) {
|
||||
const result = this._jobOutcome(payload);
|
||||
if (result) return result;
|
||||
if (onState) onState({ retrying: payload.retrying === true, phase: payload.relayPhase || payload.phase || null });
|
||||
failures = 0;
|
||||
waitMs = interval;
|
||||
interval = Math.min(Math.round(interval * 1.5), JOB_POLL_MAX_INTERVAL_MS);
|
||||
} else {
|
||||
failures += 1;
|
||||
waitMs = Math.min(JOB_POLL_INTERVAL_MS * 2 ** failures, JOB_POLL_ERROR_MAX_DELAY_MS);
|
||||
}
|
||||
const remaining = JOB_POLL_MAX_MS - (this.now() - startedAt);
|
||||
if (remaining <= 0) {
|
||||
const error = new Error('Status unklar: Upload nach 24 Stunden nicht abgeschlossen, Quelldatei bleibt erhalten');
|
||||
error.code = 'CLOUDDROP_JOB_UNCLEAR';
|
||||
throw error;
|
||||
}
|
||||
await this.sleep(Math.min(waitMs, remaining), signal);
|
||||
}
|
||||
}
|
||||
|
||||
_jobOutcome(payload) {
|
||||
const status = payload.status;
|
||||
const terminal = typeof payload.terminal === 'boolean'
|
||||
? payload.terminal
|
||||
: status === 'done' || status === 'failed';
|
||||
if (!terminal) return null;
|
||||
if (status === 'done') {
|
||||
const fileId = payload.result && typeof payload.result.fileId === 'string' ? payload.result.fileId : '';
|
||||
if (!fileId) throw new Error('Clouddrop: Keine fileId im Job-Ergebnis');
|
||||
return shareResult(fileId);
|
||||
}
|
||||
const reason = typeof payload.error === 'string' && payload.error ? payload.error : String(status || 'unknown');
|
||||
const error = new Error(`Clouddrop: Verarbeitung fehlgeschlagen (${reason})`);
|
||||
error.code = 'CLOUDDROP_JOB_FAILED';
|
||||
error.jobError = reason;
|
||||
error.expressReupload = EXPRESS_REUPLOAD_ERROR.test(reason);
|
||||
throw error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight auth check — GET /api/cloud/files (list root, small response).
|
||||
*/
|
||||
@@ -225,7 +577,7 @@ class ClouddropUploader {
|
||||
if (!this.apiKey) throw new Error('Clouddrop: API-Key fehlt');
|
||||
const res = await request(`${API_BASE}/files/?limit=1`, {
|
||||
method: 'GET',
|
||||
dispatcher: clouddropAgent,
|
||||
dispatcher: this.dispatcher,
|
||||
signal,
|
||||
headers: this._headers(),
|
||||
headersTimeout: 15_000,
|
||||
|
||||
+120
-2
@@ -433,7 +433,6 @@ class UploadManager extends EventEmitter {
|
||||
const batchId = `batch-${Date.now()}`;
|
||||
const results = new Map(); // filePath -> { name, size, results: [] }
|
||||
this._batchResults = results;
|
||||
this._batchTaskCount = tasks.length;
|
||||
this._additionalPromises = []; // Track jobs added mid-batch via addJobs()
|
||||
this._batchTaskCount = tasks.length;
|
||||
|
||||
@@ -529,6 +528,7 @@ class UploadManager extends EventEmitter {
|
||||
let lastError = null;
|
||||
let lastFailureDetails = null;
|
||||
let finalAttempt = 0;
|
||||
let deferredUpload = null;
|
||||
|
||||
const recordFinalResult = (status, payload = {}) => {
|
||||
if (status === 'aborted' && jobAbortController.signal.reason === 'automation-pause' && !this.cancelledJobIds.has(jobId) && !batchSignal.aborted) payload = { ...payload, error: 'Warteschlange angehalten' };
|
||||
@@ -576,6 +576,53 @@ class UploadManager extends EventEmitter {
|
||||
});
|
||||
};
|
||||
|
||||
const releaseSlots = () => {
|
||||
if (globalSlotAcquired && globalSemaphore) globalSemaphore.release();
|
||||
if (hosterSlotAcquired) hosterSemaphore.release();
|
||||
globalSlotAcquired = false;
|
||||
hosterSlotAcquired = false;
|
||||
};
|
||||
|
||||
const acquireSlots = async () => {
|
||||
await hosterSemaphore.acquire(signal);
|
||||
hosterSlotAcquired = true;
|
||||
if (globalSemaphore) {
|
||||
await globalSemaphore.acquire(signal);
|
||||
globalSlotAcquired = true;
|
||||
}
|
||||
};
|
||||
|
||||
const finishDeferredUpload = async (deferred) => {
|
||||
try {
|
||||
const result = await this._awaitDeferredUpload(task, deferred, {
|
||||
uploadId, jobId, fileName, fileSize, signal, fileProbe: deferred.fileProbe, maxAttempts, releaseSlots, acquireSlots
|
||||
});
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
emitFinalStatus('done', {
|
||||
result,
|
||||
speedKbs: deferred.speedKbs,
|
||||
elapsed: Math.round((Date.now() - deferred.startedAt) / 1000),
|
||||
attempt: deferred.attempt
|
||||
});
|
||||
recordFinalResult('done', { result, attempt: deferred.attempt });
|
||||
this._emitAccountSucceeded(task);
|
||||
} catch (err) {
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) {
|
||||
const error = 'Abgebrochen';
|
||||
emitFinalStatus('aborted', { error, attempt: deferred.attempt });
|
||||
recordFinalResult('aborted', { error, attempt: deferred.attempt });
|
||||
return;
|
||||
}
|
||||
const error = err && err.message ? err.message : 'Unbekannter Fehler';
|
||||
this._rotLog('deferred-upload-failure', {
|
||||
jobId, hoster: task.hoster, accountId: task.accountId, fileName, error,
|
||||
code: err && err.code ? err.code : null
|
||||
});
|
||||
emitFinalStatus('error', { error, attempt: deferred.attempt });
|
||||
recordFinalResult('error', { error, attempt: deferred.attempt });
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) {
|
||||
const error = 'Abgebrochen';
|
||||
@@ -815,6 +862,11 @@ class UploadManager extends EventEmitter {
|
||||
this.sessionBytes += fileSize;
|
||||
this.activeJobs.delete(uploadId);
|
||||
|
||||
if (result.clouddropJob) {
|
||||
deferredUpload = { result, speedKbs: currentSpeedKbs, startedAt: jobStart, attempt, throttle, fileProbe };
|
||||
break;
|
||||
}
|
||||
|
||||
emitFinalStatus('done', {
|
||||
result,
|
||||
speedKbs: currentSpeedKbs,
|
||||
@@ -910,6 +962,8 @@ class UploadManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
if (deferredUpload) return await finishDeferredUpload(deferredUpload);
|
||||
|
||||
const wasStopped = this.stopAfterActive && !signal.aborted;
|
||||
const wasAborted = signal.aborted || this.cancelledJobIds.has(jobId);
|
||||
if (wasStopped || wasAborted) {
|
||||
@@ -1138,6 +1192,10 @@ class UploadManager extends EventEmitter {
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.activeJobs.delete(uploadId);
|
||||
this.sessionBytes += fileSize;
|
||||
if (result.clouddropJob) {
|
||||
deferredUpload = { result, speedKbs: currentSpeedKbs, startedAt: jobStart, attempt, throttle, fileProbe };
|
||||
break;
|
||||
}
|
||||
emitFinalStatus('done', { result, speedKbs: currentSpeedKbs, elapsed: Math.round((Date.now() - jobStart) / 1000), attempt });
|
||||
recordFinalResult('done', { result });
|
||||
this._emitAccountSucceeded(task);
|
||||
@@ -1171,8 +1229,11 @@ class UploadManager extends EventEmitter {
|
||||
if (attempt >= maxAttempts) break;
|
||||
}
|
||||
}
|
||||
if (deferredUpload) break;
|
||||
}
|
||||
|
||||
if (deferredUpload) return await finishDeferredUpload(deferredUpload);
|
||||
|
||||
const stoppedLate = this.stopAfterActive && !signal.aborted;
|
||||
const abortedLate = signal.aborted || this.cancelledJobIds.has(jobId);
|
||||
if (stoppedLate || abortedLate) {
|
||||
@@ -1348,9 +1409,67 @@ class UploadManager extends EventEmitter {
|
||||
|
||||
async _executeUpload(task, progressCb, signal, throttle, fileProbe) {
|
||||
const result = await this._executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe);
|
||||
if (task.hoster === 'clouddrop.cc' && result && result.clouddropJob) return result;
|
||||
return assertUploadConfirmation(result, task.hoster);
|
||||
}
|
||||
|
||||
async _awaitDeferredUpload(task, deferred, ctx) {
|
||||
const { uploadId, jobId, fileName, fileSize, signal, fileProbe, maxAttempts, releaseSlots, acquireSlots } = ctx;
|
||||
let pendingJob = deferred.result.clouddropJob;
|
||||
let reuploaded = false;
|
||||
for (;;) {
|
||||
releaseSlots();
|
||||
const emitServerState = (state) => this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId,
|
||||
jobId, status: 'uploading', progress: 1, bytesUploaded: fileSize, bytesTotal: fileSize,
|
||||
speedKbs: 0, elapsed: 0, remaining: 0, error: null, result: null,
|
||||
statusDetail: state && state.retrying ? 'Wird automatisch wiederholt' : 'Verarbeitung auf Clouddrop',
|
||||
attempt: deferred.attempt, maxAttempts
|
||||
});
|
||||
emitServerState(null);
|
||||
try {
|
||||
return assertUploadConfirmation(await pendingJob.wait(signal, emitServerState), task.hoster);
|
||||
} catch (err) {
|
||||
if (reuploaded || !err || err.expressReupload !== true || signal.aborted) throw err;
|
||||
reuploaded = true;
|
||||
this._rotLog('express-reupload', {
|
||||
jobId, hoster: task.hoster, accountId: task.accountId, fileName, error: err.message
|
||||
});
|
||||
this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId,
|
||||
jobId, status: 'retrying', progress: 0, bytesUploaded: 0, bytesTotal: fileSize,
|
||||
speedKbs: 0, elapsed: 0, remaining: 0,
|
||||
error: 'Express-Upload fehlgeschlagen, neuer Upload aus der Quelldatei', result: null, attempt: 1, maxAttempts: 1
|
||||
});
|
||||
await acquireSlots();
|
||||
const result = await this._reuploadFromSource(task, { uploadId, jobId, fileName, fileSize, signal, fileProbe, throttle: deferred.throttle });
|
||||
if (!result.clouddropJob) return result;
|
||||
pendingJob = result.clouddropJob;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async _reuploadFromSource(task, ctx) {
|
||||
const { uploadId, jobId, fileName, fileSize, signal, fileProbe, throttle } = ctx;
|
||||
const activeEntry = { jobId, speedKbs: 0, bytesUploaded: 0, hoster: task.hoster };
|
||||
this.activeJobs.set(uploadId, activeEntry);
|
||||
const progressCb = (bytesUploaded, bytesTotal) => {
|
||||
this._recordTransferred(activeEntry, bytesUploaded, fileSize);
|
||||
this._emitProgress(uploadId, fileName, task.hoster, { accountId: task.accountId,
|
||||
jobId, status: 'uploading',
|
||||
progress: bytesTotal > 0 ? Math.min(1, bytesUploaded / bytesTotal) : 0,
|
||||
bytesUploaded, bytesTotal, speedKbs: 0, elapsed: 0, remaining: 0,
|
||||
error: null, result: null, attempt: 1, maxAttempts: 1
|
||||
});
|
||||
};
|
||||
try {
|
||||
const result = await this._executeUpload(task, progressCb, signal, throttle, fileProbe);
|
||||
if (signal.aborted || this.cancelledJobIds.has(jobId)) throw new Error('Aborted');
|
||||
this.sessionBytes += fileSize;
|
||||
return result;
|
||||
} finally {
|
||||
this.activeJobs.delete(uploadId);
|
||||
}
|
||||
}
|
||||
|
||||
async _executeUploadUnchecked(task, progressCb, signal, throttle, fileProbe) {
|
||||
if (task.hoster === 'vidmoly.me' && task.username) {
|
||||
const vidmoly = new VidmolyUploader();
|
||||
@@ -1568,7 +1687,6 @@ class UploadManager extends EventEmitter {
|
||||
this._additionalPromises.push(this._runJob(task, results, signal));
|
||||
this._batchTaskCount++;
|
||||
addResult.added++;
|
||||
this._batchTaskCount++;
|
||||
}
|
||||
return addResult;
|
||||
}
|
||||
|
||||
+3
-2
@@ -3528,7 +3528,7 @@ function getStatusText(job) {
|
||||
case 'preview': text = job.interrupted ? 'Unterbrochen · bereit zum Fortsetzen' : 'Bereit'; break;
|
||||
case 'queued': text = 'Wartet'; break;
|
||||
case 'getting-server': text = `Server…${accSuffix}`; break;
|
||||
case 'uploading': text = `Upload${accSuffix}`; break;
|
||||
case 'uploading': text = job.statusDetail ? `${localizeUiText(job.statusDetail)}${accSuffix}` : `Upload${accSuffix}`; break;
|
||||
case 'retrying': {
|
||||
const base = `Retry ${job.attempt}/${job.maxAttempts}${accSuffix}`;
|
||||
text = shortErr ? `${base}: ${shortErr}` : base;
|
||||
@@ -3536,7 +3536,7 @@ function getStatusText(job) {
|
||||
}
|
||||
case 'done': text = 'Fertig'; break;
|
||||
case 'aborted': text = shortErr || 'Abgebrochen'; break;
|
||||
case 'error': text = shortErr ? (/^Fehlgeschlagen(?::|$)/.test(shortErr) ? shortErr : `Fehlgeschlagen: ${shortErr}`) : 'Fehlgeschlagen'; break;
|
||||
case 'error': text = shortErr ? (/^(?:Fehlgeschlagen|Status unklar)(?::|$)/.test(shortErr) ? shortErr : `Fehlgeschlagen: ${shortErr}`) : 'Fehlgeschlagen'; break;
|
||||
case 'skipped': text = shortErr ? `Übersprungen: ${shortErr}` : 'Übersprungen'; break;
|
||||
default: text = job.status;
|
||||
}
|
||||
@@ -5148,6 +5148,7 @@ function _handleProgressImpl(data) {
|
||||
job.attempt = data.attempt || 0;
|
||||
job.maxAttempts = data.maxAttempts || 0;
|
||||
job.progress = data.progress || 0;
|
||||
job.statusDetail = data.statusDetail || null;
|
||||
// Track which account the backend is currently using so the status cell
|
||||
// can display "Primär" vs "Fallback #N" during rotation.
|
||||
if (data.accountId) job.accountId = data.accountId;
|
||||
|
||||
@@ -830,6 +830,13 @@
|
||||
['Clouddrop: Datei ist leer', 'Clouddrop: The file is empty'],
|
||||
['Clouddrop: Keine fileId in Upload-Antwort', 'Clouddrop: No fileId in the upload response'],
|
||||
['Clouddrop: Keine sessionId von /upload/init', 'Clouddrop: No sessionId from /upload/init'],
|
||||
['Clouddrop: Keine fileId im Job-Ergebnis', 'Clouddrop: No fileId in the job result'],
|
||||
['Clouddrop: Ungültige Express-Upload-Adresse', 'Clouddrop: Invalid Express upload address'],
|
||||
['Clouddrop: Kein Express-Upload-Ticket erhalten', 'Clouddrop: No Express upload ticket received'],
|
||||
['Status unklar: Upload nach 24 Stunden nicht abgeschlossen, Quelldatei bleibt erhalten', 'Status unclear: upload not finished after 24 hours, source file is kept'],
|
||||
['Express-Upload fehlgeschlagen, neuer Upload aus der Quelldatei', 'Express upload failed, uploading again from the source file'],
|
||||
['Verarbeitung auf Clouddrop', 'Processing on Clouddrop'],
|
||||
['Wird automatisch wiederholt', 'Retrying automatically'],
|
||||
['Doodstream: sess_id nicht gefunden nach Login', 'Doodstream: sess_id was not found after login'],
|
||||
['Kein Upload-Server erhalten. API-Key prüfen.', 'No upload server was returned. Check the API key.'],
|
||||
['Online-Sicherung enthält keine gültigen Einstellungen', 'The online backup does not contain valid settings'],
|
||||
@@ -1032,6 +1039,7 @@
|
||||
[/^Quelldatei-Schutz konnte nicht vorbereitet werden: (.+)$/, 'Source file protection could not be prepared: $1'],
|
||||
[/^Clouddrop: API-Antwort war kein JSON (.+)$/, 'Clouddrop: API response was not JSON $1'],
|
||||
[/^Clouddrop: Datei nicht lesbar: (.+)$/, 'Clouddrop: File cannot be read: $1'],
|
||||
[/^Clouddrop: Verarbeitung fehlgeschlagen \((.+)\)$/,'Clouddrop: Processing failed ($1)'],
|
||||
[/^Doodstream: konnte Upload-Server nicht ermitteln \(Endpoint geändert\?\)\. (.+)$/, 'Doodstream: Could not determine the upload server (endpoint changed?). $1'],
|
||||
[/^Doodstream Upload fehlgeschlagen: (.+)$/, 'Doodstream upload failed: $1'],
|
||||
[/^DoodStream: Upload-Seite antwortete mit HTTP (\d{3})$/, 'DoodStream: The upload page responded with HTTP $1'],
|
||||
|
||||
Reference in new issue
Block a user