Fix blocking upload logging and recover DoodStream socket failures; release 2.1.67
CI / verify (push) Canceled after 0s
CI / verify (push) Canceled after 0s
This commit is contained in:
1 parent
5a0cad0ca5
commit
9b06b4f852
10 files changed
+254
-114
No files matched your search
+60
-53
@@ -10,7 +10,7 @@ const UPLOAD_TIMEOUT = 1800000; // 30 min
|
||||
|
||||
// Cap doodstream's per-hoster debug log alongside the main log files so
|
||||
// dev-mode sessions don't accumulate gigabytes of upload trace.
|
||||
const { maybeRotateLogFile } = require('./log-rotation');
|
||||
const { maybeRotateLogFileAsync } = require('./log-rotation');
|
||||
const _DOODSTREAM_LOG_MAX_BYTES = 10 * 1024 * 1024;
|
||||
const _DOODSTREAM_LOG_MAX_BACKUPS = 1;
|
||||
|
||||
@@ -29,15 +29,18 @@ function _doodstreamLogPath() {
|
||||
}
|
||||
|
||||
let _debugVerbose = false;
|
||||
let _debugWrite = Promise.resolve();
|
||||
function setDebugVerbose(v) { _debugVerbose = !!v; }
|
||||
|
||||
function _debugLog(msg, force = false) {
|
||||
if (!_debugVerbose && !force) return;
|
||||
try {
|
||||
const logPath = _doodstreamLogPath();
|
||||
maybeRotateLogFile(logPath, _DOODSTREAM_LOG_MAX_BYTES, _DOODSTREAM_LOG_MAX_BACKUPS);
|
||||
const ts = new Date().toISOString();
|
||||
fs.appendFileSync(logPath, `[${ts}] ${msg}\n`);
|
||||
_debugWrite = _debugWrite.then(async () => {
|
||||
await maybeRotateLogFileAsync(logPath, _DOODSTREAM_LOG_MAX_BYTES, _DOODSTREAM_LOG_MAX_BACKUPS);
|
||||
await fs.promises.appendFile(logPath, `[${ts}] ${msg}\n`);
|
||||
}).catch(() => {});
|
||||
} catch {}
|
||||
}
|
||||
|
||||
@@ -415,12 +418,15 @@ class DoodstreamUploader {
|
||||
const totalSize = preambleBuf.length + fileSize + epilogueBuf.length;
|
||||
|
||||
let bytesRead = 0;
|
||||
const attempt = new AbortController();
|
||||
const uploadSignal = signal ? AbortSignal.any([signal, attempt.signal]) : attempt.signal;
|
||||
|
||||
async function* generate() {
|
||||
yield preambleBuf;
|
||||
for await (const chunk of readUploadFile(filePath, signal)) {
|
||||
if (signal && signal.aborted) throw new Error('Aborted');
|
||||
if (throttle) await throttle.consume(chunk.length, signal);
|
||||
for await (const chunk of readUploadFile(filePath, uploadSignal)) {
|
||||
uploadSignal.throwIfAborted();
|
||||
if (throttle) await throttle.consume(chunk.length, uploadSignal);
|
||||
uploadSignal.throwIfAborted();
|
||||
bytesRead += chunk.length;
|
||||
yield chunk;
|
||||
if (progressCb) progressCb(bytesRead, fileSize);
|
||||
@@ -428,56 +434,57 @@ class DoodstreamUploader {
|
||||
yield epilogueBuf;
|
||||
}
|
||||
|
||||
let uploadRes;
|
||||
try {
|
||||
uploadRes = await request(uploadUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||
'Content-Length': String(totalSize),
|
||||
'User-Agent': USER_AGENT,
|
||||
'Cookie': this._cookieHeader()
|
||||
},
|
||||
body: generate(),
|
||||
signal,
|
||||
bodyTimeout: UPLOAD_TIMEOUT,
|
||||
headersTimeout: 60000
|
||||
});
|
||||
} catch (err) {
|
||||
// Label which phase failed so a future "fetch failed"/"terminated" is
|
||||
// attributable to the big upload POST vs the small bookend requests. The
|
||||
// original message is preserved as a substring so upload-manager's
|
||||
// transient classification still matches. NOTE: undici may surface
|
||||
// "terminated"/"other side closed", which are not yet in that transient
|
||||
// list — revisit if logs show them.
|
||||
const mb = Math.round(bytesRead / 1048576);
|
||||
throw new Error(`Doodstream Upload-POST (${mb} MB an ${uploadUrl}): ${err && err.message ? err.message : err}`);
|
||||
}
|
||||
|
||||
const statusCode = uploadRes.statusCode;
|
||||
_debugLog(`Upload response status: ${statusCode}`);
|
||||
|
||||
// Handle redirects from upload server (undici doesn't follow them)
|
||||
if ([301, 302, 303, 307, 308].includes(statusCode)) {
|
||||
const location = uploadRes.headers['location'];
|
||||
try { await uploadRes.body.text(); } catch {}
|
||||
_debugLog(`Upload redirect to: ${location}`);
|
||||
if (location) {
|
||||
return this._handleUploadResult(location);
|
||||
let uploadRes;
|
||||
try {
|
||||
uploadRes = await request(uploadUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||
'Content-Length': String(totalSize),
|
||||
'User-Agent': USER_AGENT,
|
||||
'Cookie': this._cookieHeader()
|
||||
},
|
||||
body: generate(),
|
||||
signal: uploadSignal,
|
||||
bodyTimeout: UPLOAD_TIMEOUT,
|
||||
headersTimeout: 60000
|
||||
});
|
||||
} catch (err) {
|
||||
const mb = Math.round(bytesRead / 1048576);
|
||||
const node = new URL(uploadUrl).hostname;
|
||||
const error = new Error(`Doodstream Upload-POST (${mb} MB an ${node}): ${err && err.message ? err.message : err}`, { cause: err });
|
||||
if (err?.code) error.code = err.code;
|
||||
throw error;
|
||||
}
|
||||
|
||||
const statusCode = uploadRes.statusCode;
|
||||
_debugLog(`Upload response status: ${statusCode}`);
|
||||
|
||||
// Handle redirects from upload server (undici doesn't follow them)
|
||||
if ([301, 302, 303, 307, 308].includes(statusCode)) {
|
||||
const location = uploadRes.headers['location'];
|
||||
try { await uploadRes.body.text(); } catch {}
|
||||
_debugLog(`Upload redirect to: ${location}`);
|
||||
if (location) {
|
||||
return this._handleUploadResult(location);
|
||||
}
|
||||
}
|
||||
|
||||
const resText = await uploadRes.body.text();
|
||||
_debugLog(`Upload response body (first 500): ${resText.slice(0, 500)}`);
|
||||
|
||||
if (statusCode >= 400) {
|
||||
let payload;
|
||||
try { payload = JSON.parse(resText); } catch {}
|
||||
const msg = payload && payload.msg ? payload.msg : resText.slice(0, 200);
|
||||
throw new Error(`Doodstream Upload HTTP ${statusCode}: ${msg}`);
|
||||
}
|
||||
|
||||
return await this._parseUploadResponse(resText);
|
||||
} finally {
|
||||
attempt.abort();
|
||||
}
|
||||
|
||||
const resText = await uploadRes.body.text();
|
||||
_debugLog(`Upload response body (first 500): ${resText.slice(0, 500)}`);
|
||||
|
||||
if (statusCode >= 400) {
|
||||
let payload;
|
||||
try { payload = JSON.parse(resText); } catch {}
|
||||
const msg = payload && payload.msg ? payload.msg : resText.slice(0, 200);
|
||||
throw new Error(`Doodstream Upload HTTP ${statusCode}: ${msg}`);
|
||||
}
|
||||
|
||||
return this._parseUploadResponse(resText);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user