fix(webhook): retry+429+status handling, await before shutdown, error-path notify, abort/auto-retry suppress, Discord limits
This commit is contained in:
parent
8300d13817
commit
0e5eaa89e6
@ -1,5 +1,13 @@
|
|||||||
function isDiscordWebhook(url) {
|
function isDiscordWebhook(url) {
|
||||||
return /(^https?:\/\/)(ptb\.|canary\.)?(discord(app)?\.com)\/api\/webhooks\//i.test(String(url || ''));
|
return /^https?:\/\/(ptb\.|canary\.)?(discord(app)?\.com)\/api\/webhooks\/\d+\/[\w-]+/i.test(String(url || ''));
|
||||||
|
}
|
||||||
|
|
||||||
|
const DISCORD_CONTENT_LIMIT = 1900;
|
||||||
|
|
||||||
|
function clampDiscordContent(text) {
|
||||||
|
const s = String(text || '');
|
||||||
|
if (s.length <= DISCORD_CONTENT_LIMIT) return s;
|
||||||
|
return s.slice(0, DISCORD_CONTENT_LIMIT - 1) + '…';
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatDurationShort(sec) {
|
function formatDurationShort(sec) {
|
||||||
@ -57,16 +65,21 @@ function buildWebhookRequest(url, summary, meta) {
|
|||||||
|
|
||||||
let body;
|
let body;
|
||||||
if (isDiscordWebhook(url)) {
|
if (isDiscordWebhook(url)) {
|
||||||
const hosterLines = Object.entries(perHoster)
|
const headline = m.aborted ? 'Batch abgebrochen' : 'Batch fertig';
|
||||||
|
const hosterEntries = Object.entries(perHoster);
|
||||||
|
const MAX_HOSTER_LINES = 12;
|
||||||
|
let hosterLines = hosterEntries.slice(0, MAX_HOSTER_LINES)
|
||||||
.map(([h, b]) => `${h}: ${b.ok}/${b.ok + b.fail}`)
|
.map(([h, b]) => `${h}: ${b.ok}/${b.ok + b.fail}`)
|
||||||
.join(' · ');
|
.join(' · ');
|
||||||
|
if (hosterEntries.length > MAX_HOSTER_LINES) hosterLines += ` · …+${hosterEntries.length - MAX_HOSTER_LINES}`;
|
||||||
const lines = [
|
const lines = [
|
||||||
`**Multi-Hoster-Upload — Batch fertig**${m.machineName ? ` (${m.machineName})` : ''}`,
|
`**Multi-Hoster-Upload — ${headline}**${m.machineName ? ` (${m.machineName})` : ''}`,
|
||||||
`✅ ${succeeded} ok · ❌ ${failed} Fehler · 📦 ${total} gesamt · ⏱ ${duration}`
|
`✅ ${succeeded} ok · ❌ ${failed} Fehler · 📦 ${total} gesamt · ⏱ ${duration}`
|
||||||
];
|
];
|
||||||
if (hosterLines) lines.push(hosterLines);
|
if (hosterLines) lines.push(hosterLines);
|
||||||
const mention = resolveDiscordMention(m.mention);
|
const mention = resolveDiscordMention(m.mention);
|
||||||
const payload = { content: (mention ? mention.token + ' ' : '') + lines.join('\n') };
|
const content = clampDiscordContent((mention ? mention.token + ' ' : '') + lines.join('\n'));
|
||||||
|
const payload = { content };
|
||||||
payload.allowed_mentions = mention ? mention.allowed : { parse: [] };
|
payload.allowed_mentions = mention ? mention.allowed : { parse: [] };
|
||||||
body = JSON.stringify(payload);
|
body = JSON.stringify(payload);
|
||||||
} else {
|
} else {
|
||||||
@ -79,6 +92,7 @@ function buildWebhookRequest(url, summary, meta) {
|
|||||||
succeeded,
|
succeeded,
|
||||||
failed,
|
failed,
|
||||||
durationSec: Math.round(Number(m.durationSec) || 0),
|
durationSec: Math.round(Number(m.durationSec) || 0),
|
||||||
|
aborted: !!m.aborted,
|
||||||
perHoster,
|
perHoster,
|
||||||
timestamp: m.timestamp || null
|
timestamp: m.timestamp || null
|
||||||
});
|
});
|
||||||
@ -92,4 +106,18 @@ function buildWebhookRequest(url, summary, meta) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { isDiscordWebhook, formatDurationShort, summarizePerHosterFromBatch, buildWebhookRequest, resolveDiscordMention };
|
function isAllAborted(summary) {
|
||||||
|
if (!summary || !Array.isArray(summary.files) || summary.files.length === 0) return false;
|
||||||
|
let sawResult = false;
|
||||||
|
for (const f of summary.files) {
|
||||||
|
if (!f || !Array.isArray(f.results)) continue;
|
||||||
|
for (const r of f.results) {
|
||||||
|
if (!r) continue;
|
||||||
|
sawResult = true;
|
||||||
|
if (r.status !== 'aborted') return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sawResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { isDiscordWebhook, formatDurationShort, summarizePerHosterFromBatch, buildWebhookRequest, resolveDiscordMention, isAllAborted, clampDiscordContent, DISCORD_CONTENT_LIMIT };
|
||||||
|
|||||||
100
main.js
100
main.js
@ -17,7 +17,7 @@ const RemoteServer = require('./lib/remote-server');
|
|||||||
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
const { maybeRotateLogFile } = require('./lib/log-rotation');
|
||||||
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
const { hosterLogToFileEnabled } = require('./lib/log-policy');
|
||||||
const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle');
|
const { sanitizeConfig, buildSupportBundleText } = require('./lib/support-bundle');
|
||||||
const { buildWebhookRequest } = require('./lib/webhook-notify');
|
const { buildWebhookRequest, isAllAborted } = require('./lib/webhook-notify');
|
||||||
|
|
||||||
let mainWindow;
|
let mainWindow;
|
||||||
let _lastImportPath = null;
|
let _lastImportPath = null;
|
||||||
@ -280,30 +280,66 @@ function stopNetworkMonitor() {
|
|||||||
debugLog('network-monitor: stopped');
|
debugLog('network-monitor: stopped');
|
||||||
}
|
}
|
||||||
|
|
||||||
function sendBatchWebhook(summary, durationSec) {
|
function _sleepMs(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
||||||
|
|
||||||
|
async function _postWebhookWithRetry(req, maxAttempts) {
|
||||||
|
let lastErr = null;
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(req.url, {
|
||||||
|
method: req.method,
|
||||||
|
headers: req.headers,
|
||||||
|
body: req.body,
|
||||||
|
signal: AbortSignal.timeout(10_000)
|
||||||
|
});
|
||||||
|
if (res.status === 429) {
|
||||||
|
let waitMs = 2000 * attempt;
|
||||||
|
try {
|
||||||
|
const ra = res.headers.get('retry-after');
|
||||||
|
if (ra) waitMs = Math.min(60_000, Math.max(waitMs, Math.ceil(parseFloat(ra) * 1000)));
|
||||||
|
} catch {}
|
||||||
|
debugLog(`webhook: 429 rate-limited, retrying in ${waitMs}ms (attempt ${attempt}/${maxAttempts})`);
|
||||||
|
if (attempt < maxAttempts) { await _sleepMs(waitMs); continue; }
|
||||||
|
return { ok: false, status: 429 };
|
||||||
|
}
|
||||||
|
if (res.status >= 200 && res.status < 300) {
|
||||||
|
return { ok: true, status: res.status };
|
||||||
|
}
|
||||||
|
if (res.status >= 400 && res.status < 500) {
|
||||||
|
debugLog(`webhook: client error HTTP ${res.status} — not retrying (check URL/payload)`);
|
||||||
|
return { ok: false, status: res.status };
|
||||||
|
}
|
||||||
|
debugLog(`webhook: server error HTTP ${res.status} (attempt ${attempt}/${maxAttempts})`);
|
||||||
|
lastErr = new Error(`HTTP ${res.status}`);
|
||||||
|
} catch (err) {
|
||||||
|
lastErr = err;
|
||||||
|
debugLog(`webhook: send error: ${err && err.message ? err.message : err} (attempt ${attempt}/${maxAttempts})`);
|
||||||
|
}
|
||||||
|
if (attempt < maxAttempts) await _sleepMs(2000 * attempt);
|
||||||
|
}
|
||||||
|
return { ok: false, error: lastErr && lastErr.message ? lastErr.message : String(lastErr) };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendBatchWebhook(summary, durationSec, extra) {
|
||||||
try {
|
try {
|
||||||
const gs = configStore.load().globalSettings || {};
|
const gs = configStore.load().globalSettings || {};
|
||||||
const url = String(gs.webhookUrl || '').trim();
|
const url = String(gs.webhookUrl || '').trim();
|
||||||
if (!url || !/^https?:\/\//i.test(url)) return;
|
if (!url || !/^https?:\/\//i.test(url)) return { ok: false, skipped: true };
|
||||||
const req = buildWebhookRequest(url, summary, {
|
const req = buildWebhookRequest(url, summary, {
|
||||||
durationSec,
|
durationSec: Math.max(0, Number(durationSec) || 0),
|
||||||
appVersion: app.getVersion(),
|
appVersion: app.getVersion(),
|
||||||
machineName: require('os').hostname(),
|
machineName: require('os').hostname() || 'unknown-host',
|
||||||
mention: gs.webhookMention || '',
|
mention: gs.webhookMention || '',
|
||||||
|
aborted: !!(extra && extra.aborted),
|
||||||
timestamp: new Date().toISOString()
|
timestamp: new Date().toISOString()
|
||||||
});
|
});
|
||||||
fetch(req.url, {
|
const result = await _postWebhookWithRetry(req, 3);
|
||||||
method: req.method,
|
if (result.ok) debugLog(`webhook: sent batch-done notification (HTTP ${result.status})`);
|
||||||
headers: req.headers,
|
else debugLog(`webhook: gave up after retries (${result.status || result.error || 'unknown'})`);
|
||||||
body: req.body,
|
return result;
|
||||||
signal: AbortSignal.timeout(10_000)
|
|
||||||
}).then((res) => {
|
|
||||||
debugLog(`webhook: sent batch-done notification (HTTP ${res.status})`);
|
|
||||||
}).catch((err) => {
|
|
||||||
debugLog(`webhook: send failed: ${err && err.message ? err.message : err}`);
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
debugLog(`webhook: build failed: ${err && err.message ? err.message : err}`);
|
debugLog(`webhook: build failed: ${err && err.message ? err.message : err}`);
|
||||||
|
return { ok: false, error: err && err.message ? err.message : String(err) };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1520,6 +1556,7 @@ ipcMain.handle('start-upload', (_event, payload) => {
|
|||||||
const files = payload && Array.isArray(payload.files) ? payload.files : [];
|
const files = payload && Array.isArray(payload.files) ? payload.files : [];
|
||||||
const hosters = payload && Array.isArray(payload.hosters) ? payload.hosters : [];
|
const hosters = payload && Array.isArray(payload.hosters) ? payload.hosters : [];
|
||||||
const jobs = payload && Array.isArray(payload.jobs) ? payload.jobs : [];
|
const jobs = payload && Array.isArray(payload.jobs) ? payload.jobs : [];
|
||||||
|
const isAutoRetry = !!(payload && payload.isAutoRetry);
|
||||||
|
|
||||||
// At 500+ jobs JSON.stringify blew up the debug log with MB-sized lines
|
// At 500+ jobs JSON.stringify blew up the debug log with MB-sized lines
|
||||||
// per start-upload and added noticeable delay — log counts only.
|
// per start-upload and added noticeable delay — log counts only.
|
||||||
@ -1700,12 +1737,20 @@ ipcMain.handle('start-upload', (_event, payload) => {
|
|||||||
const _batchDurationSec = _thisManager && _thisManager.startTime
|
const _batchDurationSec = _thisManager && _thisManager.startTime
|
||||||
? Math.round((Date.now() - _thisManager.startTime) / 1000)
|
? Math.round((Date.now() - _thisManager.startTime) / 1000)
|
||||||
: 0;
|
: 0;
|
||||||
sendBatchWebhook(summary, _batchDurationSec);
|
|
||||||
try { await configStore.appendHistory(summary); } catch (err) {
|
try { await configStore.appendHistory(summary); } catch (err) {
|
||||||
debugLog(`appendHistory failed: ${err.message}`);
|
debugLog(`appendHistory failed: ${err.message}`);
|
||||||
}
|
}
|
||||||
safeSend('upload-batch-done', summary);
|
safeSend('upload-batch-done', summary);
|
||||||
|
|
||||||
|
const fullyAborted = isAllAborted(summary);
|
||||||
|
if (isAutoRetry) {
|
||||||
|
debugLog('webhook: skipped — auto-retry round (initial batch already notified)');
|
||||||
|
} else if (fullyAborted) {
|
||||||
|
debugLog('webhook: skipped — batch fully aborted/cancelled');
|
||||||
|
} else {
|
||||||
|
await sendBatchWebhook(summary, _batchDurationSec, { aborted: fullyAborted });
|
||||||
|
}
|
||||||
|
|
||||||
// Shutdown after finish
|
// Shutdown after finish
|
||||||
handleShutdownAfterFinish();
|
handleShutdownAfterFinish();
|
||||||
if (uploadManager === _thisManager) { uploadManager = null; globalThis._mhuUploadManagerRef = null; }
|
if (uploadManager === _thisManager) { uploadManager = null; globalThis._mhuUploadManagerRef = null; }
|
||||||
@ -1724,16 +1769,19 @@ ipcMain.handle('start-upload', (_event, payload) => {
|
|||||||
primeOverrides: Array.from(_sessionAccountOverrides.entries())
|
primeOverrides: Array.from(_sessionAccountOverrides.entries())
|
||||||
}).catch((err) => {
|
}).catch((err) => {
|
||||||
debugLog(`startBatch REJECTED: ${err && err.stack ? err.stack : err}`);
|
debugLog(`startBatch REJECTED: ${err && err.stack ? err.stack : err}`);
|
||||||
// Forward error to renderer as batch-done with failure
|
stopNetworkMonitor();
|
||||||
safeSend('upload-batch-done', {
|
const errorSummary = {
|
||||||
id: 'error',
|
id: 'error',
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
total: tasks.length,
|
total: tasks.length,
|
||||||
succeeded: 0,
|
succeeded: 0,
|
||||||
failed: tasks.length,
|
failed: tasks.length,
|
||||||
files: [],
|
files: [],
|
||||||
error: err ? err.message : 'Unbekannter Fehler'
|
error: err ? err.message : 'Unbekannter Fehler'
|
||||||
});
|
};
|
||||||
|
safeSend('upload-batch-done', errorSummary);
|
||||||
|
if (!isAutoRetry) sendBatchWebhook(errorSummary, 0);
|
||||||
|
if (uploadManager === _thisManager) { uploadManager = null; globalThis._mhuUploadManagerRef = null; }
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@ -1817,6 +1817,7 @@ async function startUpload(opts) {
|
|||||||
|
|
||||||
const uploadPayload = {
|
const uploadPayload = {
|
||||||
hosters,
|
hosters,
|
||||||
|
isAutoRetry: !!(opts && opts._autoRetry),
|
||||||
jobs: jobsToStart.map((job) => ({
|
jobs: jobsToStart.map((job) => ({
|
||||||
id: job.id,
|
id: job.id,
|
||||||
file: job.file,
|
file: job.file,
|
||||||
@ -3270,6 +3271,7 @@ async function saveSettings(options = {}) {
|
|||||||
autoRetryRounds: Math.max(0, Math.min(5, parseInt(document.getElementById('autoRetryRoundsInput')?.value || '0', 10) || 0)),
|
autoRetryRounds: Math.max(0, Math.min(5, parseInt(document.getElementById('autoRetryRoundsInput')?.value || '0', 10) || 0)),
|
||||||
autoRetryDelayMin: Math.max(1, Math.min(120, parseInt(document.getElementById('autoRetryDelayMinInput')?.value || '5', 10) || 5)),
|
autoRetryDelayMin: Math.max(1, Math.min(120, parseInt(document.getElementById('autoRetryDelayMinInput')?.value || '5', 10) || 5)),
|
||||||
folderMonitor: {
|
folderMonitor: {
|
||||||
|
...((config.globalSettings || {}).folderMonitor || {}),
|
||||||
enabled: !!document.getElementById('fmEnabledInput')?.checked,
|
enabled: !!document.getElementById('fmEnabledInput')?.checked,
|
||||||
folderPath: (document.getElementById('fmFolderPathInput')?.value || '').trim(),
|
folderPath: (document.getElementById('fmFolderPathInput')?.value || '').trim(),
|
||||||
recursive: !!document.getElementById('fmRecursiveInput')?.checked,
|
recursive: !!document.getElementById('fmRecursiveInput')?.checked,
|
||||||
@ -3281,6 +3283,7 @@ async function saveSettings(options = {}) {
|
|||||||
hosters: Array.from(document.querySelectorAll('.fm-hoster-checkbox:checked')).map(el => el.dataset.fmHoster)
|
hosters: Array.from(document.querySelectorAll('.fm-hoster-checkbox:checked')).map(el => el.dataset.fmHoster)
|
||||||
},
|
},
|
||||||
remote: {
|
remote: {
|
||||||
|
...((config.globalSettings || {}).remote || {}),
|
||||||
enabled: !!document.getElementById('remoteEnabledInput')?.checked,
|
enabled: !!document.getElementById('remoteEnabledInput')?.checked,
|
||||||
port: Math.max(1024, Math.min(65535, parseInt(document.getElementById('remotePortInput')?.value || '9100', 10) || 9100)),
|
port: Math.max(1024, Math.min(65535, parseInt(document.getElementById('remotePortInput')?.value || '9100', 10) || 9100)),
|
||||||
token: (document.getElementById('remoteTokenInput')?.value || '').trim(),
|
token: (document.getElementById('remoteTokenInput')?.value || '').trim(),
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
const test = require('node:test');
|
const test = require('node:test');
|
||||||
const assert = require('node:assert');
|
const assert = require('node:assert');
|
||||||
const { isDiscordWebhook, formatDurationShort, summarizePerHosterFromBatch, buildWebhookRequest, resolveDiscordMention } = require('../lib/webhook-notify');
|
const { isDiscordWebhook, formatDurationShort, summarizePerHosterFromBatch, buildWebhookRequest, resolveDiscordMention, isAllAborted, clampDiscordContent, DISCORD_CONTENT_LIMIT } = require('../lib/webhook-notify');
|
||||||
|
|
||||||
const SAMPLE_SUMMARY = {
|
const SAMPLE_SUMMARY = {
|
||||||
total: 10,
|
total: 10,
|
||||||
@ -28,6 +28,46 @@ test('isDiscordWebhook recognizes discord URLs incl. ptb/canary/discordapp', ()
|
|||||||
assert.strictEqual(isDiscordWebhook(null), false);
|
assert.strictEqual(isDiscordWebhook(null), false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('isDiscordWebhook REJECTS incomplete discord URLs (no id/token)', () => {
|
||||||
|
assert.strictEqual(isDiscordWebhook('https://discord.com/api/webhooks/'), false);
|
||||||
|
assert.strictEqual(isDiscordWebhook('https://discord.com/api/webhooks'), false);
|
||||||
|
assert.strictEqual(isDiscordWebhook('https://discord.com/api/webhooks/123'), false);
|
||||||
|
assert.strictEqual(isDiscordWebhook('https://discord.com/api/webhooks/123/'), false);
|
||||||
|
assert.ok(isDiscordWebhook('https://discord.com/api/webhooks/123456789/aBc-_token123'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('clampDiscordContent caps to the Discord limit with ellipsis', () => {
|
||||||
|
const short = 'hello';
|
||||||
|
assert.strictEqual(clampDiscordContent(short), short);
|
||||||
|
const long = 'x'.repeat(5000);
|
||||||
|
const clamped = clampDiscordContent(long);
|
||||||
|
assert.ok(clamped.length <= DISCORD_CONTENT_LIMIT);
|
||||||
|
assert.ok(clamped.endsWith('…'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('buildWebhookRequest: many hosters does not exceed Discord limit', () => {
|
||||||
|
const files = [{ name: 'a.mkv', results: [] }];
|
||||||
|
for (let i = 0; i < 60; i++) files[0].results.push({ hoster: `hoster-with-a-really-long-name-${i}.example.com`, status: 'done' });
|
||||||
|
const req = buildWebhookRequest('https://discord.com/api/webhooks/1/x', { total: 60, succeeded: 60, failed: 0, files }, { durationSec: 60 });
|
||||||
|
const body = JSON.parse(req.body);
|
||||||
|
assert.ok(body.content.length <= DISCORD_CONTENT_LIMIT, `content ${body.content.length} must be <= ${DISCORD_CONTENT_LIMIT}`);
|
||||||
|
assert.match(body.content, /\+\d+/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('buildWebhookRequest: aborted meta changes the headline', () => {
|
||||||
|
const req = buildWebhookRequest('https://discord.com/api/webhooks/1/x', { total: 5, succeeded: 0, failed: 5, files: [] }, { aborted: true });
|
||||||
|
const body = JSON.parse(req.body);
|
||||||
|
assert.match(body.content, /Batch abgebrochen/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('isAllAborted: true only when every result is aborted', () => {
|
||||||
|
assert.strictEqual(isAllAborted({ files: [{ results: [{ status: 'aborted' }, { status: 'aborted' }] }] }), true);
|
||||||
|
assert.strictEqual(isAllAborted({ files: [{ results: [{ status: 'aborted' }, { status: 'done' }] }] }), false);
|
||||||
|
assert.strictEqual(isAllAborted({ files: [{ results: [{ status: 'error' }] }] }), false);
|
||||||
|
assert.strictEqual(isAllAborted({ files: [] }), false);
|
||||||
|
assert.strictEqual(isAllAborted(null), false);
|
||||||
|
});
|
||||||
|
|
||||||
test('formatDurationShort formats h/m/s tiers', () => {
|
test('formatDurationShort formats h/m/s tiers', () => {
|
||||||
assert.strictEqual(formatDurationShort(45), '45s');
|
assert.strictEqual(formatDurationShort(45), '45s');
|
||||||
assert.strictEqual(formatDurationShort(125), '2m 5s');
|
assert.strictEqual(formatDurationShort(125), '2m 5s');
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user