Refresh stale queued link availability in the background
This commit is contained in:
@@ -194,6 +194,12 @@ const MAX_SAME_DIRECT_URL_ATTEMPTS = 3;
|
|||||||
const MAX_HTTP416_FRESH_RESTARTS = 2;
|
const MAX_HTTP416_FRESH_RESTARTS = 2;
|
||||||
const HTTP416_FRESH_RESTART_DELAY_MS = 8000;
|
const HTTP416_FRESH_RESTART_DELAY_MS = 8000;
|
||||||
const DOWNLOAD_LIVE_UPDATE_INTERVAL_MS = 750;
|
const DOWNLOAD_LIVE_UPDATE_INTERVAL_MS = 750;
|
||||||
|
const BACKGROUND_AVAILABILITY_INITIAL_DELAY_MS = 10000;
|
||||||
|
const BACKGROUND_AVAILABILITY_INTERVAL_MS = 30000;
|
||||||
|
const BACKGROUND_AVAILABILITY_STALE_MS = 30 * 60 * 1000;
|
||||||
|
const BACKGROUND_AVAILABILITY_BATCH_SIZE = 40;
|
||||||
|
const BACKGROUND_AVAILABILITY_CONCURRENCY = 4;
|
||||||
|
const BACKGROUND_AVAILABILITY_TIMEOUT_MS = 15000;
|
||||||
|
|
||||||
function getHttp416FreshRestartDelayMs(): number {
|
function getHttp416FreshRestartDelayMs(): number {
|
||||||
const fromEnv = Number(process.env.RD_HTTP416_FRESH_RESTART_DELAY_MS ?? NaN);
|
const fromEnv = Number(process.env.RD_HTTP416_FRESH_RESTART_DELAY_MS ?? NaN);
|
||||||
@@ -479,8 +485,16 @@ type DownloadManagerOptions = {
|
|||||||
onHistoryEntry?: HistoryEntryCallback;
|
onHistoryEntry?: HistoryEntryCallback;
|
||||||
enqueueNotification?: (event: NotificationEvent) => Promise<void>;
|
enqueueNotification?: (event: NotificationEvent) => Promise<void>;
|
||||||
protectEmptyClobber?: boolean;
|
protectEmptyClobber?: boolean;
|
||||||
|
enableBackgroundAvailabilityChecks?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type BackgroundAvailabilityHoster = "rapidgator" | "ddownload" | "onefichier";
|
||||||
|
|
||||||
|
type BackgroundAvailabilityResult =
|
||||||
|
| { hoster: "rapidgator"; value: NonNullable<Awaited<ReturnType<typeof checkRapidgatorOnline>>> }
|
||||||
|
| { hoster: "ddownload"; value: DdownloadCheckResult }
|
||||||
|
| { hoster: "onefichier"; value: OneFichierCheckResult };
|
||||||
|
|
||||||
type RunLifecycleContext = {
|
type RunLifecycleContext = {
|
||||||
id: string;
|
id: string;
|
||||||
startedAt: number;
|
startedAt: number;
|
||||||
@@ -2105,6 +2119,14 @@ export class DownloadManager extends EventEmitter {
|
|||||||
|
|
||||||
private sourceAvailabilityRecheckAt = new Map<string, number>();
|
private sourceAvailabilityRecheckAt = new Map<string, number>();
|
||||||
|
|
||||||
|
private backgroundAvailabilityTimer: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
|
private backgroundAvailabilityAbortController: AbortController | null = null;
|
||||||
|
|
||||||
|
private backgroundAvailabilityItemIds = new Set<string>();
|
||||||
|
|
||||||
|
private readonly backgroundAvailabilityChecksEnabled: boolean;
|
||||||
|
|
||||||
private packageDiskRetryAfterByPackage = new Map<string, number>();
|
private packageDiskRetryAfterByPackage = new Map<string, number>();
|
||||||
|
|
||||||
private diskWaitEvents: NonNullable<UiSnapshot["diskWaitEvents"]> = [];
|
private diskWaitEvents: NonNullable<UiSnapshot["diskWaitEvents"]> = [];
|
||||||
@@ -2139,6 +2161,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) {
|
public constructor(settings: AppSettings, session: SessionState, storagePaths: StoragePaths, options: DownloadManagerOptions = {}) {
|
||||||
super();
|
super();
|
||||||
this.settings = settings;
|
this.settings = settings;
|
||||||
|
this.backgroundAvailabilityChecksEnabled = options.enableBackgroundAvailabilityChecks ?? !process.env.VITEST;
|
||||||
const startedAt = nowMs();
|
const startedAt = nowMs();
|
||||||
this.appSessionStartedAt = startedAt;
|
this.appSessionStartedAt = startedAt;
|
||||||
this.runtimePersistedTotalMs = Math.max(0, Number(settings.totalRuntimeAllTimeMs || 0));
|
this.runtimePersistedTotalMs = Math.max(0, Number(settings.totalRuntimeAllTimeMs || 0));
|
||||||
@@ -2186,6 +2209,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.checkExistingRapidgatorLinks();
|
this.checkExistingRapidgatorLinks();
|
||||||
this.checkExistingDdownloadLinks();
|
this.checkExistingDdownloadLinks();
|
||||||
this.checkExistingOneFichierLinks();
|
this.checkExistingOneFichierLinks();
|
||||||
|
this.scheduleBackgroundAvailabilityCheck(BACKGROUND_AVAILABILITY_INITIAL_DELAY_MS);
|
||||||
void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (constructor): ${compactErrorText(err)}`));
|
void this.cleanupExistingExtractedArchives().catch((err) => logger.warn(`cleanupExistingExtractedArchives Fehler (constructor): ${compactErrorText(err)}`));
|
||||||
setRotationEventListener(() => {
|
setRotationEventListener(() => {
|
||||||
if (this.rotationListenerActive === false) {
|
if (this.rotationListenerActive === false) {
|
||||||
@@ -3960,11 +3984,13 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (item.status === "failed" && item.onlineStatus === "offline") {
|
if (item.status === "failed" && item.onlineStatus === "offline") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
item.onlineStatus = result.online ? "online" : "offline";
|
item.onlineStatus = result.online ? "online" : "offline";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
if (!result.online) {
|
if (!result.online) {
|
||||||
item.status = "failed";
|
item.status = "failed";
|
||||||
item.fullStatus = "Offline";
|
item.fullStatus = "Offline";
|
||||||
@@ -3989,6 +4015,172 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private scheduleBackgroundAvailabilityCheck(delayMs: number): void {
|
||||||
|
if (!this.backgroundAvailabilityChecksEnabled || !this.metadataChecksActive || this.backgroundAvailabilityTimer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.backgroundAvailabilityTimer = setTimeout(() => {
|
||||||
|
this.backgroundAvailabilityTimer = null;
|
||||||
|
void this.runBackgroundAvailabilityPass()
|
||||||
|
.catch((error) => logger.warn(`Hintergrund-Linkprüfung fehlgeschlagen: ${compactErrorText(error)}`))
|
||||||
|
.finally(() => this.scheduleBackgroundAvailabilityCheck(BACKGROUND_AVAILABILITY_INTERVAL_MS));
|
||||||
|
}, Math.max(0, delayMs));
|
||||||
|
this.backgroundAvailabilityTimer.unref?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
private stopBackgroundAvailabilityChecks(): void {
|
||||||
|
if (this.backgroundAvailabilityTimer) {
|
||||||
|
clearTimeout(this.backgroundAvailabilityTimer);
|
||||||
|
this.backgroundAvailabilityTimer = null;
|
||||||
|
}
|
||||||
|
for (const itemId of this.backgroundAvailabilityItemIds) {
|
||||||
|
const item = this.session.items[itemId];
|
||||||
|
if (item?.onlineStatus === "checking") item.onlineStatus = "online";
|
||||||
|
}
|
||||||
|
this.backgroundAvailabilityItemIds.clear();
|
||||||
|
this.backgroundAvailabilityAbortController?.abort("shutdown");
|
||||||
|
}
|
||||||
|
|
||||||
|
private backgroundAvailabilityHoster(item: DownloadItem): BackgroundAvailabilityHoster | null {
|
||||||
|
if (isRapidgatorSourceLink(item.url)) return "rapidgator";
|
||||||
|
if (isDdownloadLink(item.url)) return "ddownload";
|
||||||
|
if (isOneFichierLink(item.url)) return "onefichier";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private isBackgroundAvailabilityPending(item: DownloadItem): boolean {
|
||||||
|
const pkg = this.session.packages[item.packageId];
|
||||||
|
return Boolean(pkg
|
||||||
|
&& !pkg.cancelled
|
||||||
|
&& pkg.enabled
|
||||||
|
&& (item.status === "queued" || item.status === "reconnect_wait")
|
||||||
|
&& !this.activeTasks.has(item.id)
|
||||||
|
&& this.backgroundAvailabilityHoster(item));
|
||||||
|
}
|
||||||
|
|
||||||
|
private collectBackgroundAvailabilityCandidates(checkedAt: number): DownloadItem[] {
|
||||||
|
const staleBefore = checkedAt - BACKGROUND_AVAILABILITY_STALE_MS;
|
||||||
|
const candidates: DownloadItem[] = [];
|
||||||
|
for (const packageId of this.session.packageOrder) {
|
||||||
|
const pkg = this.session.packages[packageId];
|
||||||
|
if (!pkg || pkg.cancelled || !pkg.enabled) continue;
|
||||||
|
for (const itemId of pkg.itemIds) {
|
||||||
|
const item = this.session.items[itemId];
|
||||||
|
if (!item || item.onlineStatus !== "online" || !this.isBackgroundAvailabilityPending(item)) continue;
|
||||||
|
if (Number(item.onlineCheckedAt || 0) > staleBefore) continue;
|
||||||
|
candidates.push(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return candidates
|
||||||
|
.sort((left, right) => Number(left.onlineCheckedAt || 0) - Number(right.onlineCheckedAt || 0))
|
||||||
|
.slice(0, BACKGROUND_AVAILABILITY_BATCH_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async checkBackgroundAvailability(
|
||||||
|
item: DownloadItem,
|
||||||
|
hoster: BackgroundAvailabilityHoster,
|
||||||
|
signal: AbortSignal
|
||||||
|
): Promise<BackgroundAvailabilityResult | null> {
|
||||||
|
const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(BACKGROUND_AVAILABILITY_TIMEOUT_MS)]);
|
||||||
|
if (hoster === "rapidgator") {
|
||||||
|
const result = await checkRapidgatorOnline(item.url, requestSignal);
|
||||||
|
return result ? { hoster, value: result } : null;
|
||||||
|
}
|
||||||
|
if (hoster === "ddownload") {
|
||||||
|
const result = await checkDdownloadOnline(item.url, requestSignal);
|
||||||
|
return result ? { hoster, value: result } : null;
|
||||||
|
}
|
||||||
|
const results = await checkOneFichierLinks([item.url], requestSignal);
|
||||||
|
const result = results.get(item.url);
|
||||||
|
return result ? { hoster, value: result } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private applyBackgroundAvailabilityResult(
|
||||||
|
itemId: string,
|
||||||
|
result: BackgroundAvailabilityResult | null
|
||||||
|
): void {
|
||||||
|
const item = this.session.items[itemId];
|
||||||
|
if (!item) return;
|
||||||
|
if (!this.isBackgroundAvailabilityPending(item)) {
|
||||||
|
if (item.onlineStatus === "checking") item.onlineStatus = "online";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!result) {
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
|
if (item.onlineStatus === "checking") item.onlineStatus = "online";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (result.value.online) {
|
||||||
|
if (result.hoster === "rapidgator") {
|
||||||
|
this.applyRapidgatorCheckResult(item, result.value);
|
||||||
|
} else if (result.hoster === "ddownload") {
|
||||||
|
this.applyDdownloadCheckResult(item, result.value);
|
||||||
|
} else {
|
||||||
|
this.applyOneFichierCheckResult(item, result.value);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const pkg = this.session.packages[item.packageId];
|
||||||
|
if (!pkg) return;
|
||||||
|
const hosterLabel = result.hoster === "rapidgator" ? "Rapidgator" : result.hoster === "ddownload" ? "DDownload" : "1Fichier";
|
||||||
|
this.markItemOfflineAndSkipRelated(pkg, item, `Datei nicht gefunden auf ${hosterLabel}`, "hoster", "background");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runBackgroundAvailabilityPass(): Promise<void> {
|
||||||
|
if (!this.metadataChecksActive || this.backgroundAvailabilityAbortController) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const candidates = this.collectBackgroundAvailabilityCandidates(nowMs());
|
||||||
|
if (candidates.length === 0) return;
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
this.backgroundAvailabilityAbortController = controller;
|
||||||
|
const pendingByUrl = new Map<string, Promise<BackgroundAvailabilityResult | null>>();
|
||||||
|
for (const item of candidates) {
|
||||||
|
item.onlineStatus = "checking";
|
||||||
|
this.backgroundAvailabilityItemIds.add(item.id);
|
||||||
|
}
|
||||||
|
this.emitState();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runWithLimitedConcurrency(candidates, BACKGROUND_AVAILABILITY_CONCURRENCY, async (candidate) => {
|
||||||
|
const item = this.session.items[candidate.id];
|
||||||
|
if (!item || !this.isBackgroundAvailabilityPending(item) || controller.signal.aborted) return;
|
||||||
|
const hoster = this.backgroundAvailabilityHoster(item);
|
||||||
|
if (!hoster) return;
|
||||||
|
const key = `${hoster}:${item.url}`;
|
||||||
|
let pending = pendingByUrl.get(key);
|
||||||
|
if (!pending) {
|
||||||
|
pending = this.checkBackgroundAvailability(item, hoster, controller.signal);
|
||||||
|
pendingByUrl.set(key, pending);
|
||||||
|
}
|
||||||
|
let result: BackgroundAvailabilityResult | null = null;
|
||||||
|
try {
|
||||||
|
result = await pending;
|
||||||
|
} catch (error) {
|
||||||
|
if (!controller.signal.aborted) {
|
||||||
|
logger.warn(`Gedrosselte Link-Nachprüfung fehlgeschlagen: item=${item.fileName || item.id}, error=${compactErrorText(error)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!this.metadataChecksActive || controller.signal.aborted) return;
|
||||||
|
this.applyBackgroundAvailabilityResult(candidate.id, result);
|
||||||
|
this.persistSoon();
|
||||||
|
this.emitState();
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
if (this.backgroundAvailabilityAbortController === controller) {
|
||||||
|
this.backgroundAvailabilityAbortController = null;
|
||||||
|
}
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
const item = this.session.items[candidate.id];
|
||||||
|
if (item?.onlineStatus === "checking") item.onlineStatus = "online";
|
||||||
|
this.backgroundAvailabilityItemIds.delete(candidate.id);
|
||||||
|
}
|
||||||
|
this.persistSoon();
|
||||||
|
this.emitState();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async confirmSourceOfflineAfterFailure(
|
private async confirmSourceOfflineAfterFailure(
|
||||||
item: DownloadItem,
|
item: DownloadItem,
|
||||||
errorText: string,
|
errorText: string,
|
||||||
@@ -4018,14 +4210,17 @@ export class DownloadManager extends EventEmitter {
|
|||||||
const checkSignal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]);
|
const checkSignal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]);
|
||||||
if (isRapidgatorSourceLink(item.url)) {
|
if (isRapidgatorSourceLink(item.url)) {
|
||||||
const result = await checkRapidgatorOnline(item.url, checkSignal);
|
const result = await checkRapidgatorOnline(item.url, checkSignal);
|
||||||
|
if (result) item.onlineCheckedAt = checkedAt;
|
||||||
return result && !result.online ? "hoster" : null;
|
return result && !result.online ? "hoster" : null;
|
||||||
}
|
}
|
||||||
if (isDdownloadLink(item.url)) {
|
if (isDdownloadLink(item.url)) {
|
||||||
const result = await checkDdownloadOnline(item.url, checkSignal);
|
const result = await checkDdownloadOnline(item.url, checkSignal);
|
||||||
|
if (result) item.onlineCheckedAt = checkedAt;
|
||||||
return result && !result.online ? "hoster" : null;
|
return result && !result.online ? "hoster" : null;
|
||||||
}
|
}
|
||||||
const results = await checkOneFichierLinks([item.url], checkSignal);
|
const results = await checkOneFichierLinks([item.url], checkSignal);
|
||||||
const result = results.get(item.url);
|
const result = results.get(item.url);
|
||||||
|
if (result) item.onlineCheckedAt = checkedAt;
|
||||||
return result && !result.online ? "hoster" : null;
|
return result && !result.online ? "hoster" : null;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (signal.aborted) {
|
if (signal.aborted) {
|
||||||
@@ -4040,7 +4235,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
pkg: PackageEntry,
|
pkg: PackageEntry,
|
||||||
item: DownloadItem,
|
item: DownloadItem,
|
||||||
errorText: string,
|
errorText: string,
|
||||||
confirmedBy: "provider" | "hoster"
|
confirmedBy: "provider" | "hoster",
|
||||||
|
detectedDuring: "download" | "background" = "download"
|
||||||
): void {
|
): void {
|
||||||
const cleanError = errorText
|
const cleanError = errorText
|
||||||
.replace(/^Error:\s*/i, "")
|
.replace(/^Error:\s*/i, "")
|
||||||
@@ -4048,6 +4244,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
.trim();
|
.trim();
|
||||||
item.status = "failed";
|
item.status = "failed";
|
||||||
item.onlineStatus = "offline";
|
item.onlineStatus = "offline";
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
item.lastError = cleanError || "Quelllink ist nicht mehr verfügbar";
|
item.lastError = cleanError || "Quelllink ist nicht mehr verfügbar";
|
||||||
item.fullStatus = "Offline";
|
item.fullStatus = "Offline";
|
||||||
item.speedBps = 0;
|
item.speedBps = 0;
|
||||||
@@ -4055,7 +4252,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.retryAfterByItem.delete(item.id);
|
this.retryAfterByItem.delete(item.id);
|
||||||
this.retryStateByItem.delete(item.id);
|
this.retryStateByItem.delete(item.id);
|
||||||
this.sourceAvailabilityRecheckAt.delete(item.id);
|
this.sourceAvailabilityRecheckAt.delete(item.id);
|
||||||
this.recordRunOutcome(item.id, "failed");
|
if (this.runItemIds.has(item.id)) this.recordRunOutcome(item.id, "failed");
|
||||||
|
|
||||||
const packageItems = pkg.itemIds
|
const packageItems = pkg.itemIds
|
||||||
.map((itemId) => this.session.items[itemId])
|
.map((itemId) => this.session.items[itemId])
|
||||||
@@ -4086,12 +4283,15 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (!relatedActive) {
|
if (!relatedActive) {
|
||||||
this.releaseTargetPath(related.id);
|
this.releaseTargetPath(related.id);
|
||||||
}
|
}
|
||||||
this.recordRunOutcome(related.id, "cancelled");
|
if (this.runItemIds.has(related.id)) this.recordRunOutcome(related.id, "cancelled");
|
||||||
skippedItems.push(related);
|
skippedItems.push(related);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.logPackageForItem(item, "WARN", "Quelllink während des Downloads offline geworden", {
|
this.logPackageForItem(item, "WARN", detectedDuring === "background"
|
||||||
|
? "Quelllink bei Hintergrundprüfung offline geworden"
|
||||||
|
: "Quelllink während des Downloads offline geworden", {
|
||||||
confirmedBy,
|
confirmedBy,
|
||||||
|
detectedDuring,
|
||||||
offlineSkipScope: this.settings.offlineSkipScope,
|
offlineSkipScope: this.settings.offlineSkipScope,
|
||||||
skippedItems: skippedItems.length,
|
skippedItems: skippedItems.length,
|
||||||
skippedNames: skippedItems.map((entry) => entry.fileName).join(" | ")
|
skippedNames: skippedItems.map((entry) => entry.fileName).join(" | ")
|
||||||
@@ -4140,6 +4340,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (item.status === "failed" && item.onlineStatus === "offline") {
|
if (item.status === "failed" && item.onlineStatus === "offline") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
if (!result.online) {
|
if (!result.online) {
|
||||||
item.onlineStatus = "offline";
|
item.onlineStatus = "offline";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
@@ -4204,6 +4405,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
if (item.status === "failed" && item.onlineStatus === "offline") {
|
if (item.status === "failed" && item.onlineStatus === "offline") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
item.onlineCheckedAt = nowMs();
|
||||||
if (!result.online) {
|
if (!result.online) {
|
||||||
item.onlineStatus = "offline";
|
item.onlineStatus = "offline";
|
||||||
item.updatedAt = nowMs();
|
item.updatedAt = nowMs();
|
||||||
@@ -6935,6 +7137,7 @@ export class DownloadManager extends EventEmitter {
|
|||||||
this.updateStatisticsActivity(nowMs());
|
this.updateStatisticsActivity(nowMs());
|
||||||
this.rotationListenerActive = false;
|
this.rotationListenerActive = false;
|
||||||
this.metadataChecksActive = false;
|
this.metadataChecksActive = false;
|
||||||
|
this.stopBackgroundAvailabilityChecks();
|
||||||
this.clearPersistTimer();
|
this.clearPersistTimer();
|
||||||
if (this.stateEmitTimer) {
|
if (this.stateEmitTimer) {
|
||||||
clearTimeout(this.stateEmitTimer);
|
clearTimeout(this.stateEmitTimer);
|
||||||
|
|||||||
@@ -1047,6 +1047,9 @@ export function normalizeLoadedSession(raw: unknown): SessionState {
|
|||||||
lastError: asText(item.lastError),
|
lastError: asText(item.lastError),
|
||||||
fullStatus: asText(item.fullStatus),
|
fullStatus: asText(item.fullStatus),
|
||||||
onlineStatus: VALID_ONLINE_STATUSES.has(onlineStatusRaw) ? onlineStatusRaw as "online" | "offline" | "checking" : undefined,
|
onlineStatus: VALID_ONLINE_STATUSES.has(onlineStatusRaw) ? onlineStatusRaw as "online" | "offline" | "checking" : undefined,
|
||||||
|
onlineCheckedAt: item.onlineCheckedAt === undefined
|
||||||
|
? undefined
|
||||||
|
: clampNumber(item.onlineCheckedAt, 0, 0, now),
|
||||||
createdAt: clampNumber(item.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
|
createdAt: clampNumber(item.createdAt, now, 0, Number.MAX_SAFE_INTEGER),
|
||||||
updatedAt: clampNumber(item.updatedAt, now, 0, Number.MAX_SAFE_INTEGER)
|
updatedAt: clampNumber(item.updatedAt, now, 0, Number.MAX_SAFE_INTEGER)
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -502,6 +502,7 @@ export interface DownloadItem {
|
|||||||
createdAt: number;
|
createdAt: number;
|
||||||
updatedAt: number;
|
updatedAt: number;
|
||||||
onlineStatus?: "online" | "offline" | "checking";
|
onlineStatus?: "online" | "offline" | "checking";
|
||||||
|
onlineCheckedAt?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AudioStripFileResult {
|
export interface AudioStripFileResult {
|
||||||
|
|||||||
@@ -1534,6 +1534,157 @@ describe("download manager", () => {
|
|||||||
expect(items[2]).toMatchObject({ status: "completed", downloadedBytes: 16 });
|
expect(items[2]).toMatchObject({ status: "completed", downloadedBytes: 16 });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("refreshes a stale queued link in the background and applies the archive offline scope", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-background-availability-offline-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const manager = new DownloadManager({
|
||||||
|
...defaultSettings(),
|
||||||
|
offlineSkipScope: "archive",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract")
|
||||||
|
}, emptySession(), createStoragePaths(path.join(root, "state")));
|
||||||
|
const fileNames = [
|
||||||
|
"show.s01e02.part2.rar",
|
||||||
|
"show.s01e02.part1.rar",
|
||||||
|
"show.s01e02.part3.rar",
|
||||||
|
"show.s01e03.part1.rar"
|
||||||
|
];
|
||||||
|
manager.addPackages([{ name: "show-season", links: fileNames.map((fileName) => `https://example.test/${fileName}`) }]);
|
||||||
|
await waitFor(() => Object.values(manager.getSnapshot().session.items).every((item) => item.onlineStatus !== "checking"), 1_000);
|
||||||
|
|
||||||
|
const internal = manager as any;
|
||||||
|
const items = internal.session.packages[internal.session.packageOrder[0]].itemIds
|
||||||
|
.map((itemId: string) => internal.session.items[itemId]) as DownloadItem[];
|
||||||
|
const checkedAt = Date.now();
|
||||||
|
for (let index = 0; index < items.length; index += 1) {
|
||||||
|
items[index].url = `https://rapidgator.net/file/${String(index + 1).padStart(32, "0")}/${fileNames[index]}.html`;
|
||||||
|
items[index].onlineStatus = "online";
|
||||||
|
items[index].onlineCheckedAt = index === 0 ? checkedAt - 31 * 60 * 1000 : checkedAt;
|
||||||
|
items[index].totalBytes = 16;
|
||||||
|
}
|
||||||
|
|
||||||
|
const requested: string[] = [];
|
||||||
|
globalThis.fetch = (async (input: RequestInfo | URL): Promise<Response> => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
|
requested.push(url);
|
||||||
|
return new Response("File not found", { status: 404 });
|
||||||
|
}) as typeof fetch;
|
||||||
|
|
||||||
|
await internal.runBackgroundAvailabilityPass();
|
||||||
|
|
||||||
|
expect(requested).toEqual([items[0].url]);
|
||||||
|
expect(items[0]).toMatchObject({ status: "failed", onlineStatus: "offline", fullStatus: "Offline" });
|
||||||
|
expect(items[1]).toMatchObject({ status: "cancelled", fullStatus: "Übersprungen (Archivteil offline)" });
|
||||||
|
expect(items[2]).toMatchObject({ status: "cancelled", fullStatus: "Übersprungen (Archivteil offline)" });
|
||||||
|
expect(items[3]).toMatchObject({ status: "queued", onlineStatus: "online" });
|
||||||
|
expect(internal.runOutcomes.size).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the previous online state when a background check is not definitive", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-background-availability-ambiguous-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const manager = new DownloadManager(defaultSettings(), emptySession(), createStoragePaths(path.join(root, "state")));
|
||||||
|
manager.addPackages([{ name: "ambiguous", links: ["https://example.test/episode.mkv"] }]);
|
||||||
|
await waitFor(() => Object.values(manager.getSnapshot().session.items).every((item) => item.onlineStatus !== "checking"), 1_000);
|
||||||
|
|
||||||
|
const internal = manager as any;
|
||||||
|
const item = Object.values(internal.session.items)[0] as DownloadItem;
|
||||||
|
const previousCheckedAt = Date.now() - 31 * 60 * 1000;
|
||||||
|
item.url = "https://rapidgator.net/file/11111111111111111111111111111111/episode.mkv.html";
|
||||||
|
item.onlineStatus = "online";
|
||||||
|
item.onlineCheckedAt = previousCheckedAt;
|
||||||
|
item.totalBytes = 16;
|
||||||
|
let requestCount = 0;
|
||||||
|
globalThis.fetch = (async (): Promise<Response> => {
|
||||||
|
requestCount += 1;
|
||||||
|
return new Response("temporarily blocked", { status: 403 });
|
||||||
|
}) as typeof fetch;
|
||||||
|
|
||||||
|
await internal.runBackgroundAvailabilityPass();
|
||||||
|
await internal.runBackgroundAvailabilityPass();
|
||||||
|
|
||||||
|
expect(item).toMatchObject({ status: "queued", onlineStatus: "online" });
|
||||||
|
expect(item.onlineCheckedAt).toBeGreaterThan(previousCheckedAt);
|
||||||
|
expect(requestCount).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("limits each background pass to 40 links and four concurrent hoster requests", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-background-availability-limit-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const manager = new DownloadManager(defaultSettings(), emptySession(), createStoragePaths(path.join(root, "state")));
|
||||||
|
const fileNames = Array.from({ length: 45 }, (_, index) => `episode-${index}.mkv`);
|
||||||
|
manager.addPackages([{ name: "large-queue", links: fileNames.map((fileName) => `https://example.test/${fileName}`) }]);
|
||||||
|
await waitFor(() => Object.values(manager.getSnapshot().session.items).every((item) => item.onlineStatus !== "checking"), 1_000);
|
||||||
|
|
||||||
|
const internal = manager as any;
|
||||||
|
const items = internal.session.packages[internal.session.packageOrder[0]].itemIds
|
||||||
|
.map((itemId: string) => internal.session.items[itemId]) as DownloadItem[];
|
||||||
|
const staleBase = Date.now() - 2 * 60 * 60 * 1000;
|
||||||
|
for (let index = 0; index < items.length; index += 1) {
|
||||||
|
items[index].url = `https://rapidgator.net/file/${index.toString(16).padStart(32, "0")}/${fileNames[index]}.html`;
|
||||||
|
items[index].onlineStatus = "online";
|
||||||
|
items[index].onlineCheckedAt = staleBase + index;
|
||||||
|
items[index].totalBytes = 16;
|
||||||
|
}
|
||||||
|
|
||||||
|
const requested: string[] = [];
|
||||||
|
let active = 0;
|
||||||
|
let peak = 0;
|
||||||
|
globalThis.fetch = (async (input: RequestInfo | URL): Promise<Response> => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
|
||||||
|
requested.push(url);
|
||||||
|
active += 1;
|
||||||
|
peak = Math.max(peak, active);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
active -= 1;
|
||||||
|
const fileName = path.basename(new URL(url).pathname).replace(/\.html$/i, "");
|
||||||
|
return new Response(`<html><title>${fileName}</title><div>File size: <strong>16 B</strong></div></html>`, {
|
||||||
|
status: 200,
|
||||||
|
headers: { "Content-Type": "text/html" }
|
||||||
|
});
|
||||||
|
}) as typeof fetch;
|
||||||
|
|
||||||
|
await internal.runBackgroundAvailabilityPass();
|
||||||
|
|
||||||
|
expect(requested).toHaveLength(40);
|
||||||
|
expect(peak).toBeLessThanOrEqual(4);
|
||||||
|
expect(new Set(requested)).toEqual(new Set(items.slice(0, 40).map((item) => item.url)));
|
||||||
|
expect(items.slice(0, 40).every((item) => Number(item.onlineCheckedAt) > staleBase + 44)).toBe(true);
|
||||||
|
expect(items.slice(40).every((item, index) => item.onlineCheckedAt === staleBase + index + 40)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores a background checking marker to online during shutdown", async () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-background-availability-shutdown-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
defaultSettings(),
|
||||||
|
emptySession(),
|
||||||
|
createStoragePaths(path.join(root, "state")),
|
||||||
|
{ enableBackgroundAvailabilityChecks: true }
|
||||||
|
);
|
||||||
|
manager.addPackages([{ name: "shutdown", links: ["https://example.test/episode.mkv"] }]);
|
||||||
|
await waitFor(() => Object.values(manager.getSnapshot().session.items).every((item) => item.onlineStatus !== "checking"), 1_000);
|
||||||
|
|
||||||
|
const internal = manager as any;
|
||||||
|
const item = Object.values(internal.session.items)[0] as DownloadItem;
|
||||||
|
item.url = "https://rapidgator.net/file/22222222222222222222222222222222/episode.mkv.html";
|
||||||
|
item.onlineStatus = "online";
|
||||||
|
item.onlineCheckedAt = Date.now() - 31 * 60 * 1000;
|
||||||
|
item.totalBytes = 16;
|
||||||
|
globalThis.fetch = ((_input: RequestInfo | URL, init?: RequestInit): Promise<Response> => new Promise((_resolve, reject) => {
|
||||||
|
init?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true });
|
||||||
|
})) as typeof fetch;
|
||||||
|
|
||||||
|
expect(internal.backgroundAvailabilityTimer).not.toBeNull();
|
||||||
|
const pass = internal.runBackgroundAvailabilityPass();
|
||||||
|
await waitFor(() => item.onlineStatus === "checking", 1_000);
|
||||||
|
manager.prepareForShutdown();
|
||||||
|
await pass;
|
||||||
|
|
||||||
|
expect(item.onlineStatus).toBe("online");
|
||||||
|
expect(internal.backgroundAvailabilityTimer).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it("applies an imported settings snapshot without touching queued items or filesystem workflows", () => {
|
it("applies an imported settings snapshot without touching queued items or filesystem workflows", () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-settings-import-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-settings-import-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
|
|||||||
@@ -157,6 +157,57 @@ describe("settings storage", () => {
|
|||||||
expect(history?.provider).toBe("deepbrid");
|
expect(history?.provider).toBe("deepbrid");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves valid availability timestamps and clamps future values while loading a session", () => {
|
||||||
|
const validCheckedAt = Date.now() - 60_000;
|
||||||
|
const beforeNormalize = Date.now();
|
||||||
|
const normalized = normalizeLoadedSession({
|
||||||
|
...emptySession(),
|
||||||
|
packageOrder: ["pkg-availability"],
|
||||||
|
packages: {
|
||||||
|
"pkg-availability": {
|
||||||
|
id: "pkg-availability",
|
||||||
|
name: "Availability",
|
||||||
|
outputDir: "C:\\Downloads\\Availability",
|
||||||
|
extractDir: "C:\\Downloads\\Availability",
|
||||||
|
status: "queued",
|
||||||
|
itemIds: ["item-valid", "item-future"],
|
||||||
|
cancelled: false,
|
||||||
|
enabled: true,
|
||||||
|
createdAt: 1,
|
||||||
|
updatedAt: 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
items: {
|
||||||
|
"item-valid": {
|
||||||
|
id: "item-valid",
|
||||||
|
packageId: "pkg-availability",
|
||||||
|
url: "https://example.test/valid.bin",
|
||||||
|
status: "queued",
|
||||||
|
fileName: "valid.bin",
|
||||||
|
onlineStatus: "online",
|
||||||
|
onlineCheckedAt: validCheckedAt,
|
||||||
|
createdAt: 1,
|
||||||
|
updatedAt: 2
|
||||||
|
},
|
||||||
|
"item-future": {
|
||||||
|
id: "item-future",
|
||||||
|
packageId: "pkg-availability",
|
||||||
|
url: "https://example.test/future.bin",
|
||||||
|
status: "queued",
|
||||||
|
fileName: "future.bin",
|
||||||
|
onlineStatus: "online",
|
||||||
|
onlineCheckedAt: Date.now() + 60_000,
|
||||||
|
createdAt: 1,
|
||||||
|
updatedAt: 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(normalized.items["item-valid"].onlineCheckedAt).toBe(validCheckedAt);
|
||||||
|
expect(normalized.items["item-future"].onlineCheckedAt).toBeGreaterThanOrEqual(beforeNormalize);
|
||||||
|
expect(normalized.items["item-future"].onlineCheckedAt).toBeLessThanOrEqual(Date.now());
|
||||||
|
});
|
||||||
|
|
||||||
it.each([undefined, "megadebrid", "realdebrid", "invalid-provider"])("normalizes svc-deepbrid status with provider %s deterministically to Deepbrid", (provider) => {
|
it.each([undefined, "megadebrid", "realdebrid", "invalid-provider"])("normalizes svc-deepbrid status with provider %s deterministically to Deepbrid", (provider) => {
|
||||||
const key = "fixture-deepbrid-status-key-4pQ5";
|
const key = "fixture-deepbrid-status-key-4pQ5";
|
||||||
const normalized = normalizeSettings({
|
const normalized = normalizeSettings({
|
||||||
|
|||||||
Reference in New Issue
Block a user