Fix 16 bugs found by code review across all download modules
Critical fixes: - Post-processor: remove double attempts increment (onProgress + onArchiveFailure both counted) - Post-processor: fix slot leak when signal aborted after acquireSlot - Scheduler: reset global watchdog high-water mark after stall event (prevents permanent misfires) - Pipeline/DM: fix isPathInsideDir path traversal (add trailing separator check) - Retry-manager: check per-kind exhaustion before shelve threshold (prevents bypass) - Retry-manager: add MAX_SHELVE_COUNT=5 cap to prevent infinite shelve cycling Important fixes: - Scheduler: clear retryDelays and providerCooldowns on start() - Scheduler: skip already-aborting slots in stall detection - Download-manager: fix cleanupAfterExtraction using extractDir instead of outputDir for link removal - Download-manager: add "extracting" to package normalizeSessionStatuses - Download-manager: clear activeTasks map on stop() - Download-manager: remove useless cachedDirectUrls re-insertion after success - Stream-writer: remove duplicate truncation code in error path - Stream-writer: skip alignedFlush in finally when bodyError already set (avoids 5min drain wait) - Stream-writer: re-read elapsed after speed limiter sleep for accurate window reset - Error-classifier: add HTTP 401 (Forbidden) and 410 (NotFound) classification Tests updated to match new shelve/kind-exhaustion priority and 401 classification. All 216 tests pass, build verified. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
36dc6040ba
commit
d0885ba552
@@ -284,9 +284,9 @@ describe("classifyHttpStatus", () => {
|
||||
expect(err.httpStatus).toBe(503);
|
||||
});
|
||||
|
||||
it("classifies 401 as Unknown (no special branch)", () => {
|
||||
it("classifies 401 as Forbidden", () => {
|
||||
const err = classifyHttpStatus({ status: 401 });
|
||||
expect(err.kind).toBe(DownloadErrorKind.Unknown);
|
||||
expect(err.kind).toBe(DownloadErrorKind.Forbidden);
|
||||
expect(err.httpStatus).toBe(401);
|
||||
});
|
||||
|
||||
|
||||
+48
-15
@@ -441,18 +441,26 @@ describe("shelving", () => {
|
||||
|
||||
it("shelving increments shelveCount", () => {
|
||||
const mgr = new RetryManager();
|
||||
// Trigger shelve twice
|
||||
// First round: 15 failures -> shelve (halves to ~7)
|
||||
// Use mixed kinds to avoid per-kind exhaustion before shelve threshold
|
||||
// Timeout(10), RateLimited(8), ProviderBusy(8), ServerError(5), Unknown(5)
|
||||
const kinds = [
|
||||
DownloadErrorKind.Timeout,
|
||||
DownloadErrorKind.RateLimited,
|
||||
DownloadErrorKind.ProviderBusy,
|
||||
DownloadErrorKind.ServerError,
|
||||
DownloadErrorKind.Unknown,
|
||||
];
|
||||
// First round: 15 failures -> shelve (3 per kind, all within limits)
|
||||
for (let i = 0; i < 15; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
mgr.evaluate("a", mkError(kinds[i % kinds.length]));
|
||||
}
|
||||
const state1 = mgr.getState("a")!;
|
||||
expect(state1.shelveCount).toBe(1);
|
||||
|
||||
// After halving, totalFailures is ~7. Need 8 more to reach 15 again.
|
||||
// After halving, totalFailures is ~7. Need more to reach 15 again.
|
||||
const remaining = SHELVE_THRESHOLD - state1.totalFailures;
|
||||
for (let i = 0; i < remaining; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
mgr.evaluate("a", mkError(kinds[i % kinds.length]));
|
||||
}
|
||||
const state2 = mgr.getState("a")!;
|
||||
expect(state2.shelveCount).toBe(2);
|
||||
@@ -460,27 +468,45 @@ describe("shelving", () => {
|
||||
|
||||
it("shelve decision always has shouldRetry=true", () => {
|
||||
const mgr = new RetryManager();
|
||||
// Use mixed kinds to reach shelve threshold
|
||||
const kinds = [
|
||||
DownloadErrorKind.Timeout,
|
||||
DownloadErrorKind.RateLimited,
|
||||
DownloadErrorKind.ProviderBusy,
|
||||
DownloadErrorKind.ServerError,
|
||||
DownloadErrorKind.Unknown,
|
||||
];
|
||||
for (let i = 0; i < 15; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
mgr.evaluate("a", mkError(kinds[i % kinds.length]));
|
||||
}
|
||||
// The 15th call itself triggers shelve
|
||||
// Let's re-check: the state now has halved counters.
|
||||
// One more batch to trigger shelve again
|
||||
const state = mgr.getState("a")!;
|
||||
const needed = SHELVE_THRESHOLD - state.totalFailures;
|
||||
for (let i = 0; i < needed - 1; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
mgr.evaluate("a", mkError(kinds[i % kinds.length]));
|
||||
}
|
||||
const d = mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
const d = mgr.evaluate("a", mkError(kinds[0]));
|
||||
expect(d.shouldRetry).toBe(true);
|
||||
expect(d.delayMs).toBe(SHELVE_DELAY_MS);
|
||||
});
|
||||
|
||||
it("shelve is checked before per-kind exhaustion", () => {
|
||||
it("per-kind exhaustion is checked before shelve", () => {
|
||||
const mgr = new RetryManager();
|
||||
// NetworkReset has maxRetries=3. If we mix kinds to reach 15 total
|
||||
// without exhausting any single kind, shelve takes priority.
|
||||
// Use 5 kinds, 3 each = 15
|
||||
// NetworkReset has maxRetries=3. If we use only NetworkReset errors,
|
||||
// kind exhaustion hits at 4 (before shelve at 15).
|
||||
for (let i = 0; i < 3; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.NetworkReset));
|
||||
}
|
||||
// 4th NetworkReset -> kind exhausted (maxRetries=3)
|
||||
const d = mgr.evaluate("a", mkError(DownloadErrorKind.NetworkReset));
|
||||
expect(d.shouldRetry).toBe(false);
|
||||
expect(d.reason).toContain("erschöpft");
|
||||
});
|
||||
|
||||
it("shelve triggers when mixed kinds reach threshold without any kind exhausted", () => {
|
||||
const mgr = new RetryManager();
|
||||
// Use 5 kinds with high maxRetries so no single kind is exhausted
|
||||
const kinds = [
|
||||
DownloadErrorKind.Timeout,
|
||||
DownloadErrorKind.ServerError,
|
||||
@@ -728,9 +754,16 @@ describe("exportStates() and importStates()", () => {
|
||||
|
||||
it("shelveCount survives export/import roundtrip", () => {
|
||||
const mgr = new RetryManager();
|
||||
// Trigger shelve
|
||||
// Trigger shelve using mixed kinds to avoid per-kind exhaustion
|
||||
const kinds = [
|
||||
DownloadErrorKind.Timeout,
|
||||
DownloadErrorKind.RateLimited,
|
||||
DownloadErrorKind.ProviderBusy,
|
||||
DownloadErrorKind.ServerError,
|
||||
DownloadErrorKind.Unknown,
|
||||
];
|
||||
for (let i = 0; i < 15; i++) {
|
||||
mgr.evaluate("a", mkError(DownloadErrorKind.Unknown));
|
||||
mgr.evaluate("a", mkError(kinds[i % kinds.length]));
|
||||
}
|
||||
const originalShelve = mgr.getState("a")!.shelveCount;
|
||||
expect(originalShelve).toBeGreaterThan(0);
|
||||
|
||||
Reference in New Issue
Block a user