From 194f7b5bdbb11cf2f0f3ca1404c00fafd9414247 Mon Sep 17 00:00:00 2001 From: Sucukdeluxe <259325684+Sucukdeluxe@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:21:04 +0100 Subject: [PATCH] Update backup API filesystem dependency and preserve tests --- .gitignore | 1 - services/backup-api/package-lock.json | 8 +- services/backup-api/package.json | 3 +- services/backup-api/test/server.test.mjs | 499 +++++++++++++++++++++++ 4 files changed, 505 insertions(+), 6 deletions(-) create mode 100644 services/backup-api/test/server.test.mjs diff --git a/.gitignore b/.gitignore index 9c0a844..75da4b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ node_modules/ /tests/ -/services/backup-api/test/ /cloud-log-sync/ PROJECT_MEMORY.md .dev-runner.lock diff --git a/services/backup-api/package-lock.json b/services/backup-api/package-lock.json index 33abe63..d5e9d48 100644 --- a/services/backup-api/package-lock.json +++ b/services/backup-api/package-lock.json @@ -15,10 +15,10 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "license": "ISC" + "version": "4.2.12", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.12.tgz", + "integrity": "sha512-iuyGF2E4oafQfPsRrG2wtY2ILupMhvtdrhzrxB+0VcjsPpSPttpfYdjnGFGsTi5MaJFjI1jOKHPQ6UTbTj9TAw==", + "license": "BlueOak-1.0.0" }, "node_modules/proper-lockfile": { "version": "4.1.2", diff --git a/services/backup-api/package.json b/services/backup-api/package.json index 8aeec87..ff77449 100644 --- a/services/backup-api/package.json +++ b/services/backup-api/package.json @@ -4,7 +4,8 @@ "private": true, "type": "module", "scripts": { - "start": "node src/cli.mjs" + "start": "node src/cli.mjs", + "test": "node --test" }, "engines": { "node": ">=20" diff --git a/services/backup-api/test/server.test.mjs b/services/backup-api/test/server.test.mjs new file mode 100644 index 0000000..7e0571b --- /dev/null +++ b/services/backup-api/test/server.test.mjs @@ -0,0 +1,499 @@ +import assert from 'node:assert/strict' +import { createHash, randomBytes } from 'node:crypto' +import { mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createConnection } from 'node:net' +import test from 'node:test' +import lockfile from 'proper-lockfile' +import { createBackupServer } from '../src/server.mjs' + +const allowedOrigin = 'https://uploader.24-music.de' + +function fixture() { + const deleteSecret = randomBytes(32).toString('base64url') + return { + deleteSecret, + payload: { + id: randomBytes(16).toString('base64url'), + blob: randomBytes(96).toString('base64url'), + deleteVerifier: createHash('sha256').update(Buffer.from(deleteSecret, 'base64url')).digest('base64url') + } + } +} + +async function startApi(options = {}) { + const rootDir = await mkdtemp(join(tmpdir(), 'mhu-backup-api-')) + const server = createBackupServer({ + rootDir, + allowedOrigins: [allowedOrigin], + rateLimit: { max: 100, windowMs: 60_000 }, + ...options + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + return { + rootDir, + server, + baseUrl: `http://127.0.0.1:${server.address().port}`, + async close() { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())) + await rm(rootDir, { recursive: true, force: true }) + } + } +} + +function request(api, path, options = {}) { + return fetch(`${api.baseUrl}${path}`, options) +} + +test('health reports readiness without storage details and sends security headers', async (t) => { + const api = await startApi() + t.after(() => api.close()) + + const response = await request(api, '/health') + + assert.equal(response.status, 200) + assert.deepEqual(await response.json(), { status: 'ok' }) + assert.equal(response.headers.get('cache-control'), 'no-store') + assert.equal(response.headers.get('x-content-type-options'), 'nosniff') + assert.equal(response.headers.get('content-security-policy'), "default-src 'none'") +}) + +test('creates immutable ciphertext records and restores them after a restart', async (t) => { + const api = await startApi() + const backup = fixture() + t.after(async () => { + if (api.server.listening) await new Promise((resolve) => api.server.close(resolve)) + await rm(api.rootDir, { recursive: true, force: true }) + }) + + const created = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + assert.equal(created.status, 201) + assert.equal((await created.json()).sourceIp, '127.0.0.1') + await new Promise((resolve) => api.server.close(resolve)) + + api.server = createBackupServer({ rootDir: api.rootDir, allowedOrigins: [allowedOrigin] }) + await new Promise((resolve, reject) => { + api.server.once('error', reject) + api.server.listen(0, '127.0.0.1', resolve) + }) + api.baseUrl = `http://127.0.0.1:${api.server.address().port}` + + const restored = await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: backup.payload.id }) + }) + assert.equal(restored.status, 200) + assert.deepEqual(await restored.json(), { blob: backup.payload.blob }) + + const duplicate = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ ...backup.payload, blob: randomBytes(96).toString('base64url') }) + }) + assert.equal(duplicate.status, 409) +}) + +test('validates payload shape, content type and decoded blob size', async (t) => { + const api = await startApi() + t.after(() => api.close()) + const valid = fixture() + const invalid = [ + { ...valid.payload, id: 'short' }, + { ...valid.payload, blob: 'not+base64url' }, + { ...valid.payload, deleteVerifier: 'short' }, + { id: valid.payload.id, blob: valid.payload.blob }, + { ...valid.payload, extra: true }, + { ...valid.payload, expiresInSeconds: 0 }, + { ...valid.payload, expiresInSeconds: 30 * 24 * 60 * 60 }, + { ...valid.payload, expiresInSeconds: '604800' } + ] + + for (const body of invalid) { + const response = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body) + }) + assert.equal(response.status, 400) + } + + const wrongType = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'text/plain' }, + body: JSON.stringify(valid.payload) + }) + assert.equal(wrongType.status, 415) + + const oversized = fixture() + oversized.payload.blob = randomBytes(262_145).toString('base64url') + const tooLarge = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(oversized.payload) + }) + assert.equal(tooLarge.status, 413) +}) + +test('expires finite backups at the exact deadline and removes their ciphertext', async (t) => { + let nowMs = Date.parse('2026-09-01T10:00:00.000Z') + const api = await startApi({ now: () => nowMs }) + t.after(() => api.close()) + const backup = fixture() + backup.payload.expiresInSeconds = 86_400 + + const created = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + assert.equal(created.status, 201) + const storedPath = join(api.rootDir, `${backup.payload.id}.json`) + const stored = JSON.parse(await readFile(storedPath, 'utf8')) + assert.equal(stored.version, 4) + assert.equal(stored.expiresAt, '2026-09-02T10:00:00.000Z') + + const beforeDeadline = await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: backup.payload.id }) + }) + assert.equal(beforeDeadline.status, 200) + + nowMs += 86_400_000 + const expired = await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: backup.payload.id }) + }) + assert.equal(expired.status, 404) + await assert.rejects(stat(storedPath), error => error.code === 'ENOENT') +}) + +test('keeps legacy records unlimited and reclaims expired records on the next upload', async (t) => { + let nowMs = Date.parse('2026-09-01T10:00:00.000Z') + const api = await startApi({ now: () => nowMs }) + t.after(() => api.close()) + const legacy = fixture() + await writeFile(join(api.rootDir, `${legacy.payload.id}.json`), JSON.stringify({ + version: 1, + blob: legacy.payload.blob, + deleteVerifier: legacy.payload.deleteVerifier, + createdAt: '2026-01-01T00:00:00.000Z' + })) + const finite = fixture() + finite.payload.expiresInSeconds = 86_400 + await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(finite.payload) + }) + + nowMs += 2 * 86_400_000 + const replacement = fixture() + replacement.payload.expiresInSeconds = null + assert.equal((await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(replacement.payload) + })).status, 201) + + await assert.rejects(stat(join(api.rootDir, `${finite.payload.id}.json`)), error => error.code === 'ENOENT') + assert.equal((await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: legacy.payload.id }) + })).status, 200) + assert.equal((await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: replacement.payload.id }) + })).status, 200) +}) + +test('deletes only with the matching client secret and returns constant not-found responses', async (t) => { + const api = await startApi() + t.after(() => api.close()) + const backup = fixture() + await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + + const missing = await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: randomBytes(16).toString('base64url') }) + }) + const wrong = await request(api, '/v1/backups/delete', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: backup.payload.id, deleteSecret: randomBytes(32).toString('base64url') }) + }) + assert.equal(missing.status, 404) + assert.equal(wrong.status, 404) + assert.equal(await missing.text(), await wrong.text()) + + const deleted = await request(api, '/v1/backups/delete', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: backup.payload.id, deleteSecret: backup.deleteSecret }) + }) + assert.equal(deleted.status, 204) + assert.equal((await readdir(api.rootDir)).length, 0) +}) + +test('allows only configured origins and supports preflight', async (t) => { + const api = await startApi() + t.after(() => api.close()) + + const allowed = await request(api, '/health', { headers: { origin: allowedOrigin } }) + assert.equal(allowed.headers.get('access-control-allow-origin'), allowedOrigin) + const denied = await request(api, '/health', { headers: { origin: 'https://attacker.example' } }) + assert.equal(denied.status, 403) + const preflight = await request(api, '/v1/backups', { + method: 'OPTIONS', + headers: { + origin: allowedOrigin, + 'access-control-request-method': 'POST', + 'access-control-request-headers': 'content-type' + } + }) + assert.equal(preflight.status, 204) +}) + +test('separately rate limits uploads while restores and health remain available', async (t) => { + const api = await startApi({ uploadRateLimit: { max: 1, windowMs: 60_000 } }) + t.after(() => api.close()) + const first = fixture() + const second = fixture() + const create = (backup) => request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + + assert.equal((await create(first)).status, 201) + assert.equal((await create(second)).status, 429) + const restored = await request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ id: first.payload.id }) + }) + assert.equal(restored.status, 200) + assert.equal((await request(api, '/health')).status, 200) +}) + +test('rate limits invalid request bodies before validation', async (t) => { + const api = await startApi({ requestRateLimit: { max: 1, windowMs: 60_000 } }) + t.after(() => api.close()) + const sendInvalid = () => request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ invalid: 'x'.repeat(300_000) }) + }) + + assert.equal((await sendInvalid()).status, 400) + assert.equal((await sendInvalid()).status, 429) +}) + +test('ignores forwarded client addresses from untrusted socket peers', async (t) => { + const api = await startApi({ + trustedProxy: true, + trustedProxyAddresses: [], + rateLimit: { max: 1, windowMs: 60_000 } + }) + t.after(() => api.close()) + const body = JSON.stringify({ id: randomBytes(16).toString('base64url') }) + const restore = (forwarded) => request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': forwarded }, + body + }) + + assert.equal((await restore('198.51.100.1')).status, 404) + assert.equal((await restore('198.51.100.2')).status, 429) + const created = await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': '198.51.100.3' }, + body: JSON.stringify(fixture().payload) + }) + assert.equal(created.status, 201) +}) + +test('uses the last forwarded address from an explicitly trusted proxy', async (t) => { + const api = await startApi({ + trustedProxy: true, + trustedProxyAddresses: ['127.0.0.1'], + rateLimit: { max: 1, windowMs: 60_000 } + }) + t.after(() => api.close()) + const body = JSON.stringify({ id: randomBytes(16).toString('base64url') }) + const restore = (forwarded) => request(api, '/v1/backups/restore', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': forwarded }, + body + }) + + assert.equal((await restore('198.51.100.1, 203.0.113.9')).status, 404) + assert.equal((await restore('198.51.100.2, 203.0.113.9')).status, 429) +}) + +test('stores only server-derived source IP and keeps it out of public restore responses', async t => { + for (const [trustedProxy, forwarded, expected] of [ + [false, '198.51.100.2', '127.0.0.1'], + [true, '198.51.100.2, 203.0.113.9', '203.0.113.9'], + [true, '2001:db8::42', '2001:db8::42'], + [true, 'invalid', '127.0.0.1'] + ]) { + const api = await startApi({ trustedProxy, trustedProxyAddresses: ['127.0.0.1'] }) + t.after(() => api.close()) + const backup = fixture() + const create = payload => request(api, '/v1/backups', { method: 'POST', headers: { 'content-type': 'application/json', 'x-forwarded-for': forwarded }, body: JSON.stringify(payload) }) + assert.equal((await create({ ...backup.payload, sourceIp: '192.0.2.1' })).status, 400) + const created = await create(backup.payload) + assert.equal(created.status, 201) + const metadata = await created.json() + assert.equal(metadata.sourceIp, expected) + const file = join(api.rootDir, `${backup.payload.id}.json`) + const stored = JSON.parse(await readFile(file, 'utf8')) + assert.equal(stored.sourceIp, expected) + assert.equal(stored.createdAt, metadata.createdAt) + const restoreRecord = () => request(api, '/v1/backups/restore', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ id: backup.payload.id }) }) + assert.deepEqual(await (await restoreRecord()).json(), { blob: backup.payload.blob }) + delete stored.sourceIp + stored.version = 2 + await writeFile(file, JSON.stringify(stored)) + assert.equal((await restoreRecord()).status, 200) + } +}) + +test('keeps concurrency leases until storage mutations finish', async (t) => { + const api = await startApi({ maxConcurrentPerClient: 1, maxConcurrentTotal: 1 }) + t.after(() => api.close()) + const release = await lockfile.lock(api.rootDir, { + realpath: false, + lockfilePath: join(api.rootDir, '.storage.lock') + }) + const create = (backup) => request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + const first = create(fixture()) + await new Promise((resolve) => setTimeout(resolve, 40)) + const second = create(fixture()) + let secondStatus + try { + secondStatus = await Promise.race([ + second.then((response) => response.status), + new Promise((resolve) => setTimeout(() => resolve('pending'), 100)) + ]) + } finally { + await release() + } + assert.equal((await first).status, 201) + if (secondStatus === 'pending') await second + assert.equal(secondStatus, 429) +}) + +test('times out incomplete request bodies', async (t) => { + const api = await startApi({ bodyTimeoutMs: 30 }) + t.after(() => api.close()) + const response = await new Promise((resolve, reject) => { + const socket = createConnection(new URL(api.baseUrl).port, '127.0.0.1') + let data = '' + socket.setEncoding('utf8') + socket.once('error', reject) + socket.on('data', (chunk) => { data += chunk }) + socket.on('end', () => resolve(data)) + socket.once('connect', () => { + socket.write('POST /v1/backups HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: 10\r\nConnection: close\r\n\r\n{') + }) + }) + + assert.match(response, /^HTTP\/1\.1 408 /) + assert.match(response, /request_timeout/) +}) + +test('caches health readiness instead of writing on every request', async (t) => { + const api = await startApi({ healthCacheMs: 60_000 }) + t.after(() => api.close()) + + assert.equal((await request(api, '/health')).status, 200) + await rm(api.rootDir, { recursive: true, force: true }) + assert.equal((await request(api, '/health')).status, 200) + await assert.rejects(stat(api.rootDir), { code: 'ENOENT' }) +}) + +test('cleans orphaned temporary files and enforces a record limit', async (t) => { + const api = await startApi({ maxRecords: 1 }) + t.after(() => api.close()) + const orphan = join(api.rootDir, `.${randomBytes(16).toString('hex')}.tmp`) + await writeFile(orphan, 'orphan') + const create = (backup) => request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + + assert.equal((await create(fixture())).status, 201) + assert.equal((await create(fixture())).status, 507) + const files = await readdir(api.rootDir) + assert.equal(files.some((name) => name.endsWith('.tmp')), false) + assert.equal(files.filter((name) => name.endsWith('.json')).length, 1) +}) + +test('enforces atomic storage capacity without blocking existing restores', async (t) => { + const api = await startApi({ maxStorageBytes: 420 }) + const secondServer = createBackupServer({ rootDir: api.rootDir, allowedOrigins: [allowedOrigin], maxStorageBytes: 420 }) + await new Promise((resolve, reject) => { + secondServer.once('error', reject) + secondServer.listen(0, '127.0.0.1', resolve) + }) + t.after(() => api.close()) + t.after(() => new Promise((resolve) => secondServer.close(resolve))) + const first = fixture() + const second = fixture() + const create = (backup, baseUrl = api.baseUrl) => fetch(`${baseUrl}/v1/backups`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + + const results = await Promise.all([create(first), create(second, `http://127.0.0.1:${secondServer.address().port}`)]) + + assert.deepEqual(results.map((response) => response.status).sort(), [201, 507]) + assert.equal((await readdir(api.rootDir)).length, 1) +}) + +test('never accepts record ids in URLs and stores no client delete secret', async (t) => { + const api = await startApi() + t.after(() => api.close()) + const backup = fixture() + + assert.equal((await request(api, `/v1/backups/${backup.payload.id}`)).status, 404) + assert.equal((await request(api, `/v1/backups?backup=${backup.payload.id}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + })).status, 404) + + await request(api, '/v1/backups', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(backup.payload) + }) + const files = await readdir(api.rootDir) + const stored = await readFile(join(api.rootDir, files[0]), 'utf8') + assert.equal(stored.includes(backup.deleteSecret), false) +})