Fix disk-backpressure stalls and improve episode-token parsing
This commit is contained in:
parent
a6c65acfcb
commit
7dc12aca0c
@ -38,6 +38,8 @@ type ActiveTask = {
|
|||||||
stallRetries?: number;
|
stallRetries?: number;
|
||||||
genericErrorRetries?: number;
|
genericErrorRetries?: number;
|
||||||
unrestrictRetries?: number;
|
unrestrictRetries?: number;
|
||||||
|
blockedOnDiskWrite?: boolean;
|
||||||
|
blockedOnDiskSince?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
const DEFAULT_DOWNLOAD_STALL_TIMEOUT_MS = 10000;
|
const DEFAULT_DOWNLOAD_STALL_TIMEOUT_MS = 10000;
|
||||||
@ -324,7 +326,7 @@ function toWindowsLongPathIfNeeded(filePath: string): string {
|
|||||||
|
|
||||||
const SCENE_RELEASE_FOLDER_RE = /-(?:4sf|4sj)$/i;
|
const SCENE_RELEASE_FOLDER_RE = /-(?:4sf|4sj)$/i;
|
||||||
const SCENE_GROUP_SUFFIX_RE = /-(?=[A-Za-z0-9]{2,}$)(?=[A-Za-z0-9]*[A-Z])[A-Za-z0-9]+$/;
|
const SCENE_GROUP_SUFFIX_RE = /-(?=[A-Za-z0-9]{2,}$)(?=[A-Za-z0-9]*[A-Z])[A-Za-z0-9]+$/;
|
||||||
const SCENE_EPISODE_RE = /(?:^|[._\-\s])s(\d{1,2})e(\d{1,3})(?:[._\-\s]|$)/i;
|
const SCENE_EPISODE_RE = /(?:^|[._\-\s])s(\d{1,2})e(\d{1,3})(?:e(\d{1,3}))?(?:[._\-\s]|$)/i;
|
||||||
const SCENE_SEASON_ONLY_RE = /(^|[._\-\s])s\d{1,2}(?=[._\-\s]|$)/i;
|
const SCENE_SEASON_ONLY_RE = /(^|[._\-\s])s\d{1,2}(?=[._\-\s]|$)/i;
|
||||||
const SCENE_SEASON_CAPTURE_RE = /(?:^|[._\-\s])s(\d{1,2})(?=[._\-\s]|$)/i;
|
const SCENE_SEASON_CAPTURE_RE = /(?:^|[._\-\s])s(\d{1,2})(?=[._\-\s]|$)/i;
|
||||||
const SCENE_EPISODE_ONLY_RE = /(?:^|[._\-\s])e(?:p(?:isode)?)?\s*0*(\d{1,3})(?:[._\-\s]|$)/i;
|
const SCENE_EPISODE_ONLY_RE = /(?:^|[._\-\s])e(?:p(?:isode)?)?\s*0*(\d{1,3})(?:[._\-\s]|$)/i;
|
||||||
@ -395,7 +397,14 @@ export function extractEpisodeToken(fileName: string): string | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return `S${String(season).padStart(2, "0")}E${String(episode).padStart(2, "0")}`;
|
let token = `S${String(season).padStart(2, "0")}E${String(episode).padStart(2, "0")}`;
|
||||||
|
if (match[3]) {
|
||||||
|
const episode2 = Number(match[3]);
|
||||||
|
if (Number.isFinite(episode2) && episode2 > 0) {
|
||||||
|
token += `E${String(episode2).padStart(2, "0")}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return token;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractSeasonToken(fileName: string): string | null {
|
function extractSeasonToken(fileName: string): string | null {
|
||||||
@ -542,7 +551,7 @@ export function applyEpisodeTokenToFolderName(folderName: string, episodeToken:
|
|||||||
return episodeToken;
|
return episodeToken;
|
||||||
}
|
}
|
||||||
|
|
||||||
const episodeRe = /(^|[._\-\s])s\d{1,2}e\d{1,3}(?=[._\-\s]|$)/i;
|
const episodeRe = /(^|[._\-\s])s\d{1,2}e\d{1,3}(?:e\d{1,3})?(?=[._\-\s]|$)/i;
|
||||||
if (episodeRe.test(trimmed)) {
|
if (episodeRe.test(trimmed)) {
|
||||||
return trimmed.replace(episodeRe, `$1${episodeToken}`);
|
return trimmed.replace(episodeRe, `$1${episodeToken}`);
|
||||||
}
|
}
|
||||||
@ -3316,10 +3325,15 @@ export class DownloadManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let stalledCount = 0;
|
let stalledCount = 0;
|
||||||
|
let diskBlockedCount = 0;
|
||||||
for (const active of this.activeTasks.values()) {
|
for (const active of this.activeTasks.values()) {
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (active.blockedOnDiskWrite) {
|
||||||
|
diskBlockedCount += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const item = this.session.items[active.itemId];
|
const item = this.session.items[active.itemId];
|
||||||
if (item && (item.status === "downloading" || item.status === "validating")) {
|
if (item && (item.status === "downloading" || item.status === "validating")) {
|
||||||
stalledCount += 1;
|
stalledCount += 1;
|
||||||
@ -3330,11 +3344,14 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten`);
|
logger.warn(`Globaler Download-Stall erkannt (${Math.floor((now - this.lastGlobalProgressAt) / 1000)}s ohne Fortschritt), ${stalledCount} Task(s) neu starten, diskBlocked=${diskBlockedCount}`);
|
||||||
for (const active of this.activeTasks.values()) {
|
for (const active of this.activeTasks.values()) {
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (active.blockedOnDiskWrite) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const item = this.session.items[active.itemId];
|
const item = this.session.items[active.itemId];
|
||||||
if (item && (item.status === "downloading" || item.status === "validating")) {
|
if (item && (item.status === "downloading" || item.status === "validating")) {
|
||||||
active.abortReason = "stall";
|
active.abortReason = "stall";
|
||||||
@ -3552,7 +3569,9 @@ export class DownloadManager extends EventEmitter {
|
|||||||
abortController: new AbortController(),
|
abortController: new AbortController(),
|
||||||
abortReason: "none",
|
abortReason: "none",
|
||||||
resumable: true,
|
resumable: true,
|
||||||
nonResumableCounted: false
|
nonResumableCounted: false,
|
||||||
|
blockedOnDiskWrite: false,
|
||||||
|
blockedOnDiskSince: 0
|
||||||
};
|
};
|
||||||
this.activeTasks.set(itemId, active);
|
this.activeTasks.set(itemId, active);
|
||||||
this.emitState();
|
this.emitState();
|
||||||
@ -4253,7 +4272,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
: 170;
|
: 170;
|
||||||
let lastUiEmitAt = 0;
|
let lastUiEmitAt = 0;
|
||||||
const stallTimeoutMs = getDownloadStallTimeoutMs();
|
const stallTimeoutMs = getDownloadStallTimeoutMs();
|
||||||
const drainTimeoutMs = Math.max(4000, Math.min(45000, stallTimeoutMs > 0 ? stallTimeoutMs : 15000));
|
const drainTimeoutMs = Math.max(30000, Math.min(300000, stallTimeoutMs > 0 ? stallTimeoutMs * 12 : 120000));
|
||||||
|
let lastDiskBusyEmitAt = 0;
|
||||||
|
|
||||||
const waitDrain = (): Promise<void> => new Promise((resolve, reject) => {
|
const waitDrain = (): Promise<void> => new Promise((resolve, reject) => {
|
||||||
if (active.abortController.signal.aborted) {
|
if (active.abortController.signal.aborted) {
|
||||||
@ -4261,15 +4281,27 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
active.blockedOnDiskWrite = true;
|
||||||
|
active.blockedOnDiskSince = nowMs();
|
||||||
|
if (item.status !== "paused" && !this.session.paused) {
|
||||||
|
const nowTick = nowMs();
|
||||||
|
if (nowTick - lastDiskBusyEmitAt >= 1200) {
|
||||||
|
item.status = "downloading";
|
||||||
|
item.speedBps = 0;
|
||||||
|
item.fullStatus = `Warte auf Festplatte (${providerLabel(item.provider)})`;
|
||||||
|
item.updatedAt = nowTick;
|
||||||
|
this.emitState();
|
||||||
|
lastDiskBusyEmitAt = nowTick;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let settled = false;
|
let settled = false;
|
||||||
let timeoutId: NodeJS.Timeout | null = setTimeout(() => {
|
let timeoutId: NodeJS.Timeout | null = setTimeout(() => {
|
||||||
if (settled) {
|
if (settled) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
settled = true;
|
settled = true;
|
||||||
stream.off("drain", onDrain);
|
cleanup();
|
||||||
stream.off("error", onError);
|
|
||||||
active.abortController.signal.removeEventListener("abort", onAbort);
|
|
||||||
// Do NOT abort the controller here – drain timeout means disk is slow,
|
// Do NOT abort the controller here – drain timeout means disk is slow,
|
||||||
// not network stall. Rejecting without abort lets the inner retry loop
|
// not network stall. Rejecting without abort lets the inner retry loop
|
||||||
// handle it (resume download) instead of escalating to processItem's
|
// handle it (resume download) instead of escalating to processItem's
|
||||||
@ -4282,6 +4314,8 @@ export class DownloadManager extends EventEmitter {
|
|||||||
clearTimeout(timeoutId);
|
clearTimeout(timeoutId);
|
||||||
timeoutId = null;
|
timeoutId = null;
|
||||||
}
|
}
|
||||||
|
active.blockedOnDiskWrite = false;
|
||||||
|
active.blockedOnDiskSince = 0;
|
||||||
stream.off("drain", onDrain);
|
stream.off("drain", onDrain);
|
||||||
stream.off("error", onError);
|
stream.off("error", onError);
|
||||||
active.abortController.signal.removeEventListener("abort", onAbort);
|
active.abortController.signal.removeEventListener("abort", onAbort);
|
||||||
@ -4336,6 +4370,21 @@ export class DownloadManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const nowTick = nowMs();
|
const nowTick = nowMs();
|
||||||
|
if (active.blockedOnDiskWrite) {
|
||||||
|
if (item.status === "paused" || this.session.paused) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (nowTick - lastIdleEmitAt >= idlePulseMs) {
|
||||||
|
item.status = "downloading";
|
||||||
|
item.speedBps = 0;
|
||||||
|
item.fullStatus = `Warte auf Festplatte (${providerLabel(item.provider)})`;
|
||||||
|
item.updatedAt = nowTick;
|
||||||
|
this.emitState();
|
||||||
|
lastIdleEmitAt = nowTick;
|
||||||
|
lastDiskBusyEmitAt = nowTick;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (nowTick - lastDataAt < idlePulseMs) {
|
if (nowTick - lastDataAt < idlePulseMs) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -62,6 +62,22 @@ describe("extractEpisodeToken", () => {
|
|||||||
it("extracts from episode token at end of string", () => {
|
it("extracts from episode token at end of string", () => {
|
||||||
expect(extractEpisodeToken("show.s02e03")).toBe("S02E03");
|
expect(extractEpisodeToken("show.s02e03")).toBe("S02E03");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("extracts double episode token s01e01e02", () => {
|
||||||
|
expect(extractEpisodeToken("tvr-mammon-s01e01e02-720p")).toBe("S01E01E02");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts double episode with dot separators", () => {
|
||||||
|
expect(extractEpisodeToken("Show.S01E03E04.720p")).toBe("S01E03E04");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts double episode at end of string", () => {
|
||||||
|
expect(extractEpisodeToken("show.s02e05e06")).toBe("S02E05E06");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("extracts double episode with single-digit numbers", () => {
|
||||||
|
expect(extractEpisodeToken("show-s1e1e2-720p")).toBe("S01E01E02");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("applyEpisodeTokenToFolderName", () => {
|
describe("applyEpisodeTokenToFolderName", () => {
|
||||||
@ -96,6 +112,21 @@ describe("applyEpisodeTokenToFolderName", () => {
|
|||||||
it("is case-insensitive for -4SF/-4SJ suffix", () => {
|
it("is case-insensitive for -4SF/-4SJ suffix", () => {
|
||||||
expect(applyEpisodeTokenToFolderName("Show.720p-4SF", "S01E01")).toBe("Show.720p.S01E01-4SF");
|
expect(applyEpisodeTokenToFolderName("Show.720p-4SF", "S01E01")).toBe("Show.720p.S01E01-4SF");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("applies double episode token to season-only folder", () => {
|
||||||
|
expect(applyEpisodeTokenToFolderName("Mammon.S01.German.1080P.Bluray.x264-SMAHD", "S01E01E02"))
|
||||||
|
.toBe("Mammon.S01E01E02.German.1080P.Bluray.x264-SMAHD");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("replaces existing double episode in folder with new token", () => {
|
||||||
|
expect(applyEpisodeTokenToFolderName("Show.S01E01E02.720p-4sf", "S01E03E04"))
|
||||||
|
.toBe("Show.S01E03E04.720p-4sf");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("replaces existing single episode in folder with double episode token", () => {
|
||||||
|
expect(applyEpisodeTokenToFolderName("Show.S01E01.720p-4sf", "S01E01E02"))
|
||||||
|
.toBe("Show.S01E01E02.720p-4sf");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("sourceHasRpToken", () => {
|
describe("sourceHasRpToken", () => {
|
||||||
@ -556,4 +587,37 @@ describe("buildAutoRenameBaseNameFromFolders", () => {
|
|||||||
);
|
);
|
||||||
expect(result).toBe("Cheat.der.Betrug.S01E01.GERMAN.720p.WEB.h264-tmsf");
|
expect(result).toBe("Cheat.der.Betrug.S01E01.GERMAN.720p.WEB.h264-tmsf");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renames double episode file into season folder (Mammon style)", () => {
|
||||||
|
const result = buildAutoRenameBaseNameFromFoldersWithOptions(
|
||||||
|
[
|
||||||
|
"Mammon.S01.German.1080P.Bluray.x264-SMAHD"
|
||||||
|
],
|
||||||
|
"tvr-mammon-s01e01e02-720p",
|
||||||
|
{ forceEpisodeForSeasonFolder: true }
|
||||||
|
);
|
||||||
|
expect(result).toBe("Mammon.S01E01E02.German.1080P.Bluray.x264-SMAHD");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renames second double episode file correctly", () => {
|
||||||
|
const result = buildAutoRenameBaseNameFromFoldersWithOptions(
|
||||||
|
[
|
||||||
|
"Mammon.S01.German.1080P.Bluray.x264-SMAHD"
|
||||||
|
],
|
||||||
|
"tvr-mammon-s01e03e04-720p",
|
||||||
|
{ forceEpisodeForSeasonFolder: true }
|
||||||
|
);
|
||||||
|
expect(result).toBe("Mammon.S01E03E04.German.1080P.Bluray.x264-SMAHD");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renames third double episode file correctly", () => {
|
||||||
|
const result = buildAutoRenameBaseNameFromFoldersWithOptions(
|
||||||
|
[
|
||||||
|
"Mammon.S01.German.1080P.Bluray.x264-SMAHD"
|
||||||
|
],
|
||||||
|
"tvr-mammon-s01e05e06-720p",
|
||||||
|
{ forceEpisodeForSeasonFolder: true }
|
||||||
|
);
|
||||||
|
expect(result).toBe("Mammon.S01E05E06.German.1080P.Bluray.x264-SMAHD");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -880,7 +880,14 @@ describe("download manager", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
manager.addPackages([{ name: "drain-stall", links: ["https://dummy/drain-stall"] }]);
|
manager.addPackages([{ name: "drain-stall", links: ["https://dummy/drain-stall"] }]);
|
||||||
|
const queuedSnapshot = manager.getSnapshot();
|
||||||
|
const packageId = queuedSnapshot.session.packageOrder[0] || "";
|
||||||
|
const itemId = queuedSnapshot.session.packages[packageId]?.itemIds[0] || "";
|
||||||
manager.start();
|
manager.start();
|
||||||
|
await waitFor(() => {
|
||||||
|
const status = manager.getSnapshot().session.items[itemId]?.fullStatus || "";
|
||||||
|
return status.includes("Warte auf Festplatte");
|
||||||
|
}, 12000);
|
||||||
await waitFor(() => !manager.getSnapshot().session.running, 40000);
|
await waitFor(() => !manager.getSnapshot().session.running, 40000);
|
||||||
|
|
||||||
const item = Object.values(manager.getSnapshot().session.items)[0];
|
const item = Object.values(manager.getSnapshot().session.items)[0];
|
||||||
@ -4329,6 +4336,63 @@ describe("download manager", () => {
|
|||||||
expect(internal.speedBytesLastWindow).toBe(0);
|
expect(internal.speedBytesLastWindow).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not trigger global stall abort while write-buffer is disk-blocked", () => {
|
||||||
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
|
tempDirs.push(root);
|
||||||
|
const previousGlobalWatchdog = process.env.RD_GLOBAL_STALL_TIMEOUT_MS;
|
||||||
|
process.env.RD_GLOBAL_STALL_TIMEOUT_MS = "2500";
|
||||||
|
|
||||||
|
try {
|
||||||
|
const manager = new DownloadManager(
|
||||||
|
{
|
||||||
|
...defaultSettings(),
|
||||||
|
token: "rd-token",
|
||||||
|
outputDir: path.join(root, "downloads"),
|
||||||
|
extractDir: path.join(root, "extract")
|
||||||
|
},
|
||||||
|
emptySession(),
|
||||||
|
createStoragePaths(path.join(root, "state"))
|
||||||
|
);
|
||||||
|
|
||||||
|
manager.addPackages([{ name: "disk-block-guard", links: ["https://dummy/disk-block-guard"] }]);
|
||||||
|
const snapshot = manager.getSnapshot();
|
||||||
|
const packageId = snapshot.session.packageOrder[0] || "";
|
||||||
|
const itemId = snapshot.session.packages[packageId]?.itemIds[0] || "";
|
||||||
|
|
||||||
|
const internal = manager as unknown as any;
|
||||||
|
internal.session.running = true;
|
||||||
|
internal.session.paused = false;
|
||||||
|
internal.session.reconnectUntil = 0;
|
||||||
|
internal.session.totalDownloadedBytes = 0;
|
||||||
|
internal.session.items[itemId].status = "downloading";
|
||||||
|
internal.lastGlobalProgressBytes = 0;
|
||||||
|
internal.lastGlobalProgressAt = Date.now() - 10000;
|
||||||
|
|
||||||
|
const abortController = new AbortController();
|
||||||
|
internal.activeTasks.set(itemId, {
|
||||||
|
itemId,
|
||||||
|
packageId,
|
||||||
|
abortController,
|
||||||
|
abortReason: "none",
|
||||||
|
resumable: true,
|
||||||
|
nonResumableCounted: false,
|
||||||
|
blockedOnDiskWrite: true,
|
||||||
|
blockedOnDiskSince: Date.now() - 5000
|
||||||
|
});
|
||||||
|
|
||||||
|
internal.runGlobalStallWatchdog(Date.now());
|
||||||
|
|
||||||
|
expect(abortController.signal.aborted).toBe(false);
|
||||||
|
expect(internal.lastGlobalProgressAt).toBeGreaterThan(Date.now() - 2000);
|
||||||
|
} finally {
|
||||||
|
if (previousGlobalWatchdog === undefined) {
|
||||||
|
delete process.env.RD_GLOBAL_STALL_TIMEOUT_MS;
|
||||||
|
} else {
|
||||||
|
process.env.RD_GLOBAL_STALL_TIMEOUT_MS = previousGlobalWatchdog;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("cleans run tracking when start conflict is skipped", async () => {
|
it("cleans run tracking when start conflict is skipped", async () => {
|
||||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), "rd-dm-"));
|
||||||
tempDirs.push(root);
|
tempDirs.push(root);
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user