Keep account fallback healthy across concurrent upload failures
CI / verify (push) Canceled after 0s
CI / verify (push) Canceled after 0s
This commit is contained in:
1 parent
e9287545ee
commit
2ba21993c8
3 files changed
+50
-17
No files matched your search
@@ -24,6 +24,21 @@ function createAccountPicker({ hosters, hosterSettings, hasCreds, indices }) {
|
|||||||
return pick;
|
return pick;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function selectFallbackAccount({ accounts, hoster, currentAccountId, failedKeys, preferredAccountId, excludedIds, hasCreds = () => true }) {
|
||||||
|
if (!Array.isArray(accounts)) return null;
|
||||||
|
const usable = account => account && account.id && account.id !== currentAccountId
|
||||||
|
&& account.enabled !== false && hasCreds(hoster, account)
|
||||||
|
&& !failedKeys?.has(`${hoster}:${account.id}`) && !excludedIds?.has(account.id);
|
||||||
|
const preferred = accounts.find(account => account?.id === preferredAccountId && usable(account));
|
||||||
|
if (preferred) return preferred;
|
||||||
|
const currentIndex = accounts.findIndex(account => account?.id === currentAccountId);
|
||||||
|
for (let offset = 1; offset <= accounts.length; offset++) {
|
||||||
|
const account = accounts[(currentIndex + offset) % accounts.length];
|
||||||
|
if (usable(account)) return account;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function classifyAccountFailure(error) {
|
function classifyAccountFailure(error) {
|
||||||
if (!error || error.transientNetwork === true || error.hosterTransient === true || error.fileRejected === true) return 'none';
|
if (!error || error.transientNetwork === true || error.hosterTransient === true || error.fileRejected === true) return 'none';
|
||||||
if (error.otpRequired === true) return 'manual';
|
if (error.otpRequired === true) return 'manual';
|
||||||
@@ -175,5 +190,6 @@ module.exports = {
|
|||||||
classifyAccountFailure,
|
classifyAccountFailure,
|
||||||
createAccountCooldownController,
|
createAccountCooldownController,
|
||||||
createAccountPicker,
|
createAccountPicker,
|
||||||
|
selectFallbackAccount,
|
||||||
enabledAccountsFor
|
enabledAccountsFor
|
||||||
};
|
};
|
||||||
+20
-4
@@ -12,7 +12,7 @@ const Semaphore = require('./semaphore');
|
|||||||
const Throttle = require('./throttle');
|
const Throttle = require('./throttle');
|
||||||
const { probeFileHead } = require('./file-probe');
|
const { probeFileHead } = require('./file-probe');
|
||||||
const { normalizeFailureDetails } = require('./upload-diagnostics');
|
const { normalizeFailureDetails } = require('./upload-diagnostics');
|
||||||
const { classifyAccountFailure } = require('./account-rotation');
|
const { classifyAccountFailure, selectFallbackAccount } = require('./account-rotation');
|
||||||
|
|
||||||
const DEFAULT_SETTINGS = {
|
const DEFAULT_SETTINGS = {
|
||||||
retries: 3,
|
retries: 3,
|
||||||
@@ -97,6 +97,18 @@ class UploadManager extends EventEmitter {
|
|||||||
return this._accountOverrides.get(hoster) || null;
|
return this._accountOverrides.get(hoster) || null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_getFallbackAccount(hoster, currentAccountId, excludedIds) {
|
||||||
|
const override = this.getOverride(hoster);
|
||||||
|
return selectFallbackAccount({
|
||||||
|
accounts: this.accountPools[hoster] || (override ? [override] : []),
|
||||||
|
hoster,
|
||||||
|
currentAccountId,
|
||||||
|
failedKeys: this._failedAccounts,
|
||||||
|
preferredAccountId: override?.id,
|
||||||
|
excludedIds
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
getActiveJobCount() {
|
getActiveJobCount() {
|
||||||
return this.activeJobs.size;
|
return this.activeJobs.size;
|
||||||
}
|
}
|
||||||
@@ -123,7 +135,9 @@ class UploadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_emitAccountSucceeded(task) {
|
_emitAccountSucceeded(task) {
|
||||||
if (task?.hoster && task?.accountId) this.emit('account-succeeded', { hoster: task.hoster, accountId: task.accountId });
|
if (task?.hoster && task?.accountId && !this._failedAccounts.has(`${task.hoster}:${task.accountId}`)) {
|
||||||
|
this.emit('account-succeeded', { hoster: task.hoster, accountId: task.accountId });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// True if the hoster has a usable override stored that differs from the
|
// True if the hoster has a usable override stored that differs from the
|
||||||
@@ -604,7 +618,7 @@ class UploadManager extends EventEmitter {
|
|||||||
// with 500 queued jobs and 1 parallel slot, without this check every
|
// with 500 queued jobs and 1 parallel slot, without this check every
|
||||||
// job still hits the original dead account first.
|
// job still hits the original dead account first.
|
||||||
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) {
|
if (task.accountId && this._failedAccounts.has(task.hoster + ':' + task.accountId)) {
|
||||||
const override = this._accountOverrides.get(task.hoster);
|
const override = this._getFallbackAccount(task.hoster, task.accountId);
|
||||||
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
if (override && !this._failedAccounts.has(task.hoster + ':' + override.id)) {
|
||||||
this._rotLog('pre-job-swap', {
|
this._rotLog('pre-job-swap', {
|
||||||
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
jobId, hoster: task.hoster, fileName, fromAccountId: task.accountId, toAccountId: override.id
|
||||||
@@ -952,6 +966,7 @@ class UploadManager extends EventEmitter {
|
|||||||
recordFinalResult('error', { error });
|
recordFinalResult('error', { error });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const attemptedAccounts = new Set([task.accountId]);
|
||||||
while (task.accountId) {
|
while (task.accountId) {
|
||||||
if (signal.aborted || this.stopAfterActive) break;
|
if (signal.aborted || this.stopAfterActive) break;
|
||||||
// The rotated-to account failed with a file-class error (file
|
// The rotated-to account failed with a file-class error (file
|
||||||
@@ -995,7 +1010,7 @@ class UploadManager extends EventEmitter {
|
|||||||
lastError: lastError ? lastError.message : null
|
lastError: lastError ? lastError.message : null
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const override = this._accountOverrides.get(task.hoster);
|
const override = this._getFallbackAccount(task.hoster, task.accountId, attemptedAccounts);
|
||||||
if (!override) {
|
if (!override) {
|
||||||
this._rotLog('rotation-end', {
|
this._rotLog('rotation-end', {
|
||||||
jobId, hoster: task.hoster, fileName, reason: 'no-override-set',
|
jobId, hoster: task.hoster, fileName, reason: 'no-override-set',
|
||||||
@@ -1023,6 +1038,7 @@ class UploadManager extends EventEmitter {
|
|||||||
fromAccountId: task.accountId, toAccountId: override.id
|
fromAccountId: task.accountId, toAccountId: override.id
|
||||||
});
|
});
|
||||||
task.accountId = override.id;
|
task.accountId = override.id;
|
||||||
|
attemptedAccounts.add(override.id);
|
||||||
task.username = override.username;
|
task.username = override.username;
|
||||||
task.password = override.password;
|
task.password = override.password;
|
||||||
task.apiKey = override.apiKey;
|
task.apiKey = override.apiKey;
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ const VidmolyUploader = require('./lib/vidmoly-upload');
|
|||||||
const VoeUploader = require('./lib/voe-upload');
|
const VoeUploader = require('./lib/voe-upload');
|
||||||
const DoodstreamUploader = require('./lib/doodstream-upload');
|
const DoodstreamUploader = require('./lib/doodstream-upload');
|
||||||
const { createDoodstreamOtpCoordinator, selectUploadAuth } = require('./lib/account-auth');
|
const { createDoodstreamOtpCoordinator, selectUploadAuth } = require('./lib/account-auth');
|
||||||
const { createAccountCooldownController, createAccountPicker } = require('./lib/account-rotation');
|
const { createAccountCooldownController, createAccountPicker, selectFallbackAccount } = require('./lib/account-rotation');
|
||||||
const ClouddropUploader = require('./lib/clouddrop-upload');
|
const ClouddropUploader = require('./lib/clouddrop-upload');
|
||||||
const { checkForUpdate, prepareUpdate, launchPreparedUpdate, abortUpdate, createUpdateAnnouncementState } = require('./lib/updater');
|
const { checkForUpdate, prepareUpdate, launchPreparedUpdate, abortUpdate, createUpdateAnnouncementState } = require('./lib/updater');
|
||||||
const backupCrypto = require('./lib/backup-crypto');
|
const backupCrypto = require('./lib/backup-crypto');
|
||||||
@@ -1156,16 +1156,19 @@ function hosterAccountHasCreds(name, account) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function getNextFallbackAccount(config, hosterName, failedAccountId) {
|
function getNextFallbackAccount(config, hosterName, failedAccountId) {
|
||||||
const accounts = config.hosters[hosterName];
|
const failedKeys = new Set([
|
||||||
if (!Array.isArray(accounts)) return null;
|
..._accountCooldowns.activeKeys(),
|
||||||
const failedIndex = accounts.findIndex(a => a.id === failedAccountId);
|
...(uploadManager?.getFailedAccountKeys() || [])
|
||||||
if (failedIndex < 0) return null;
|
]);
|
||||||
for (let i = failedIndex + 1; i < accounts.length; i++) {
|
const preferred = uploadManager?.getOverride(hosterName) || _sessionAccountOverrides.get(hosterName);
|
||||||
if (accounts[i].enabled !== false && hosterAccountHasCreds(hosterName, accounts[i])) {
|
return selectFallbackAccount({
|
||||||
return accounts[i];
|
accounts: config.hosters?.[hosterName],
|
||||||
}
|
hoster: hosterName,
|
||||||
}
|
currentAccountId: failedAccountId,
|
||||||
return null;
|
failedKeys,
|
||||||
|
preferredAccountId: preferred?.id,
|
||||||
|
hasCreds: hosterAccountHasCreds
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildAccountPools(config) {
|
function buildAccountPools(config) {
|
||||||
@@ -1912,7 +1915,6 @@ ipcMain.handle('save-config', async (_event, config) => {
|
|||||||
if (sep < 0) continue;
|
if (sep < 0) continue;
|
||||||
const hoster = key.slice(0, sep);
|
const hoster = key.slice(0, sep);
|
||||||
const failedAccountId = key.slice(sep + 1);
|
const failedAccountId = key.slice(sep + 1);
|
||||||
if (uploadManager.getOverride(hoster)) continue; // already has a fallback
|
|
||||||
const fallback = getNextFallbackAccount(cfg, hoster, failedAccountId);
|
const fallback = getNextFallbackAccount(cfg, hoster, failedAccountId);
|
||||||
if (fallback) {
|
if (fallback) {
|
||||||
rotLog(`main: config-updated → late fallback ${fallback.id} for ${hoster} (was stuck on ${failedAccountId})`);
|
rotLog(`main: config-updated → late fallback ${fallback.id} for ${hoster} (was stuck on ${failedAccountId})`);
|
||||||
@@ -2396,7 +2398,6 @@ ipcMain.handle('start-upload', async (_event, payload) => {
|
|||||||
// _failedAccounts), so jobs still start on the primary as expected.
|
// _failedAccounts), so jobs still start on the primary as expected.
|
||||||
const hostersInBatch = new Set(tasks.map(t => t.hoster).filter(Boolean));
|
const hostersInBatch = new Set(tasks.map(t => t.hoster).filter(Boolean));
|
||||||
for (const hoster of hostersInBatch) {
|
for (const hoster of hostersInBatch) {
|
||||||
if (_sessionAccountOverrides.has(hoster)) continue; // already learned from past batch
|
|
||||||
const accounts = config.hosters && config.hosters[hoster];
|
const accounts = config.hosters && config.hosters[hoster];
|
||||||
if (!Array.isArray(accounts) || accounts.length < 2) continue;
|
if (!Array.isArray(accounts) || accounts.length < 2) continue;
|
||||||
const primary = accounts.find(a => a && a.enabled !== false && hosterAccountHasCreds(hoster, a));
|
const primary = accounts.find(a => a && a.enabled !== false && hosterAccountHasCreds(hoster, a));
|
||||||
|
|||||||
Reference in new issue
Block a user