Administrator/Multi-Hoster-Upload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 47 Wiki Activity

release: Multi-Hoster-Upload v2.0.3

Browse Source
This commit is contained in:
Sucukdeluxe
2026-08-09 14:54:44 +02:00
parent 6cdccf71f2
commit 1c4d096cb9
31 changed files with 2304 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/backup-api/package-lock.json
Generated
+50
View File
@@ -0,0 +1,50 @@
{
"name": "multi-hoster-uploader-backup-api",
"version": "2.0.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "multi-hoster-uploader-backup-api",
"version": "2.0.3",
"dependencies": {
"proper-lockfile": "4.1.2"
},
"engines": {
"node": ">=20"
}
},
"node_modules/graceful-fs": {
"version": "4.2.11",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
"license": "ISC"
},
"node_modules/proper-lockfile": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
"integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.2.4",
"retry": "^0.12.0",
"signal-exit": "^3.0.2"
}
},
"node_modules/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/signal-exit": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
"license": "ISC"
}
}
}
services/backup-api/package.json
+16
View File
@@ -0,0 +1,16 @@
{
"name": "multi-hoster-uploader-backup-api",
"version": "2.0.3",
"private": true,
"type": "module",
"scripts": {
"start": "node src/cli.mjs",
"test": "node --test"
},
"engines": {
"node": ">=20"
},
"dependencies": {
"proper-lockfile": "4.1.2"
}
}
services/backup-api/src/cli.mjs
+67
View File
@@ -0,0 +1,67 @@
import { resolve } from 'node:path'
import { createBackupServer } from './server.mjs'
const port = Number.parseInt(process.env.PORT ?? '8788', 10)
const host = process.env.HOST ?? '127.0.0.1'
const rootDir = resolve(process.env.BACKUP_DATA_DIR ?? './data')
const allowedOrigins = (process.env.ALLOWED_ORIGINS ?? '')
.split(',')
.map((origin) => origin.trim())
.filter(Boolean)
const rateLimit = {
max: Number.parseInt(process.env.RATE_LIMIT_MAX ?? '60', 10),
windowMs: Number.parseInt(process.env.RATE_LIMIT_WINDOW_MS ?? '60000', 10)
}
const uploadRateLimit = {
max: Number.parseInt(process.env.UPLOAD_RATE_LIMIT_MAX ?? '10', 10),
windowMs: Number.parseInt(process.env.UPLOAD_RATE_LIMIT_WINDOW_MS ?? '3600000', 10)
}
const requestRateLimit = {
max: Number.parseInt(process.env.REQUEST_RATE_LIMIT_MAX ?? '120', 10),
windowMs: Number.parseInt(process.env.REQUEST_RATE_LIMIT_WINDOW_MS ?? '60000', 10)
}
const maxStorageBytes = Number.parseInt(process.env.MAX_STORAGE_BYTES ?? String(10 * 1024 * 1024 * 1024), 10)
const maxRecords = Number.parseInt(process.env.MAX_RECORDS ?? '10000', 10)
const bodyTimeoutMs = Number.parseInt(process.env.BODY_TIMEOUT_MS ?? '10000', 10)
const healthCacheMs = Number.parseInt(process.env.HEALTH_CACHE_MS ?? '5000', 10)
const maxConcurrentPerClient = Number.parseInt(process.env.MAX_CONCURRENT_PER_CLIENT ?? '8', 10)
const maxConcurrentTotal = Number.parseInt(process.env.MAX_CONCURRENT_TOTAL ?? '64', 10)
const trustedProxy = process.env.TRUST_PROXY === 'true'
const trustedProxyAddresses = (process.env.TRUSTED_PROXY_ADDRESSES ?? '127.0.0.1,::1,::ffff:127.0.0.1')
.split(',')
.map((address) => address.trim())
.filter(Boolean)
if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) throw new Error('Invalid PORT')
const server = createBackupServer({
rootDir,
allowedOrigins,
rateLimit,
uploadRateLimit,
requestRateLimit,
maxStorageBytes,
maxRecords,
bodyTimeoutMs,
healthCacheMs,
maxConcurrentPerClient,
maxConcurrentTotal,
trustedProxy,
trustedProxyAddresses
})
server.listen(port, host, () => {
process.stdout.write(`Backup API listening on ${host}:${port}\n`)
})
function shutdown() {
server.close((error) => {
if (error) {
process.stderr.write('Backup API shutdown failed\n')
process.exitCode = 1
}
})
}
process.on('SIGINT', shutdown)
process.on('SIGTERM', shutdown)
services/backup-api/src/server.mjs
+552
View File
@@ -0,0 +1,552 @@
import { createHash, timingSafeEqual, randomBytes } from 'node:crypto'
import { createServer } from 'node:http'
import { link, mkdir, open, readFile, readdir, stat, unlink } from 'node:fs/promises'
import { isIP } from 'node:net'
import { join } from 'node:path'
import lockfile from 'proper-lockfile'
const maxBlobBytes = 256 * 1024
const maxBodyBytes = 384 * 1024
const idPattern = /^[A-Za-z0-9_-]{22}$/
const verifierPattern = /^[A-Za-z0-9_-]{43}$/
const blobPattern = /^[A-Za-z0-9_-]+$/
const notFoundBody = '{"error":"not_found"}'
function isCanonicalBase64Url(value, byteLength, pattern) {
if (typeof value !== 'string' || !pattern.test(value)) return false
const decoded = Buffer.from(value, 'base64url')
return decoded.length === byteLength && decoded.toString('base64url') === value
}
function isValidBackup(payload) {
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return false
const keys = Object.keys(payload).sort()
if (keys.join(',') !== 'blob,deleteVerifier,id') return false
if (!isCanonicalBase64Url(payload.id, 16, idPattern)) return false
if (!isCanonicalBase64Url(payload.deleteVerifier, 32, verifierPattern)) return false
if (typeof payload.blob !== 'string' || !blobPattern.test(payload.blob)) return false
const decoded = Buffer.from(payload.blob, 'base64url')
return decoded.length <= maxBlobBytes && decoded.toString('base64url') === payload.blob
}
function createRateLimiter({ max, windowMs }) {
const clients = new Map()
let requestCount = 0
return (address) => {
const now = Date.now()
requestCount += 1
if (requestCount % 1024 === 0) {
for (const [key, value] of clients) {
if (now - value.startedAt >= windowMs) clients.delete(key)
}
}
const current = clients.get(address)
if (!current || now - current.startedAt >= windowMs) {
clients.set(address, { startedAt: now, count: 1 })
return null
}
if (current.count >= max) return Math.max(1, Math.ceil((windowMs - (now - current.startedAt)) / 1000))
current.count += 1
return null
}
}
function createConcurrencyLimiter({ perClient, total }) {
const clients = new Map()
let active = 0
return {
enter(address) {
const clientActive = clients.get(address) ?? 0
if (active >= total || clientActive >= perClient) return false
active += 1
clients.set(address, clientActive + 1)
return true
},
leave(address) {
const clientActive = clients.get(address) ?? 0
active = Math.max(0, active - 1)
if (clientActive <= 1) clients.delete(address)
else clients.set(address, clientActive - 1)
}
}
}
function readJsonBody(request, timeoutMs) {
const declaredLength = Number.parseInt(request.headers['content-length'] ?? '', 10)
if (Number.isFinite(declaredLength) && declaredLength > maxBodyBytes) {
request.resume()
return Promise.resolve({ error: 413 })
}
return new Promise((resolve, reject) => {
let size = 0
let settled = false
const chunks = []
const cleanup = () => {
clearTimeout(timer)
request.off('data', onData)
request.off('end', onEnd)
request.off('aborted', onAborted)
request.off('error', onError)
}
const finish = (result) => {
if (settled) return
settled = true
cleanup()
resolve(result)
}
const onData = (chunk) => {
size += chunk.length
if (size > maxBodyBytes) {
finish({ error: 413 })
request.resume()
return
}
chunks.push(chunk)
}
const onEnd = () => {
try {
finish({ value: JSON.parse(Buffer.concat(chunks).toString('utf8')) })
} catch {
finish({ error: 400 })
}
}
const onAborted = () => reject(new Error('Request aborted'))
const onError = (error) => reject(error)
const timer = setTimeout(() => {
finish({ error: 408 })
request.resume()
}, timeoutMs)
request.on('data', onData)
request.on('end', onEnd)
request.on('aborted', onAborted)
request.on('error', onError)
})
}
function recordPath(rootDir, id) {
return join(rootDir, `${id}.json`)
}
function createMutationQueue() {
let pending = Promise.resolve()
return (operation) => {
const result = pending.then(operation, operation)
pending = result.catch(() => {})
return result
}
}
async function cleanupTemporaryFiles(rootDir) {
const entries = await readdir(rootDir, { withFileTypes: true })
for (const entry of entries) {
if (!entry.isFile() || !/^\.[a-f0-9]{32}\.tmp$/.test(entry.name)) continue
try {
await unlink(join(rootDir, entry.name))
} catch (error) {
if (error.code !== 'ENOENT') throw error
}
}
}
async function directoryUsage(rootDir) {
let bytes = 0
let records = 0
const entries = await readdir(rootDir, { withFileTypes: true })
for (const entry of entries) {
if (!entry.isFile() || !entry.name.endsWith('.json')) continue
try {
bytes += (await stat(join(rootDir, entry.name))).size
records += 1
} catch (error) {
if (error.code !== 'ENOENT') throw error
}
}
return { bytes, records }
}
async function syncDirectory(rootDir) {
let handle
try {
handle = await open(rootDir, 'r')
await handle.sync()
} catch (error) {
if (!['EISDIR', 'EINVAL', 'ENOTSUP', 'EPERM', 'EBADF'].includes(error.code)) throw error
} finally {
await handle?.close().catch(() => {})
}
}
async function withStorageLock(rootDir, operation) {
await mkdir(rootDir, { recursive: true })
const release = await lockfile.lock(rootDir, {
realpath: false,
lockfilePath: join(rootDir, '.storage.lock'),
stale: 30_000,
update: 10_000,
retries: {
retries: 100,
factor: 1.1,
minTimeout: 10,
maxTimeout: 100,
randomize: true
}
})
try {
return await operation()
} finally {
let releaseError
try {
await release()
} catch (error) {
releaseError = error
}
try {
await syncDirectory(rootDir)
} catch (error) {
releaseError ??= error
}
if (releaseError) throw releaseError
}
}
async function recordExists(rootDir, id) {
try {
await stat(recordPath(rootDir, id))
return true
} catch (error) {
if (error.code === 'ENOENT') return false
throw error
}
}
async function createRecord(rootDir, payload, maxStorageBytes, maxRecords) {
await mkdir(rootDir, { recursive: true })
await cleanupTemporaryFiles(rootDir)
if (await recordExists(rootDir, payload.id)) return 'duplicate'
const contents = Buffer.from(JSON.stringify({
version: 1,
blob: payload.blob,
deleteVerifier: payload.deleteVerifier,
createdAt: new Date().toISOString()
}), 'utf8')
const usage = await directoryUsage(rootDir)
if (usage.bytes + contents.length > maxStorageBytes || usage.records >= maxRecords) return 'full'
const temporaryPath = join(rootDir, `.${randomBytes(16).toString('hex')}.tmp`)
let handle
let temporaryCreated = false
let published = false
try {
handle = await open(temporaryPath, 'wx', 0o600)
temporaryCreated = true
try {
await handle.writeFile(contents)
await handle.sync()
} finally {
await handle.close()
handle = undefined
}
try {
await link(temporaryPath, recordPath(rootDir, payload.id))
} catch (error) {
if (error.code === 'EEXIST') return 'duplicate'
throw error
}
published = true
return 'created'
} finally {
let cleanupError
try {
await handle?.close()
} catch (error) {
cleanupError = error
}
if (temporaryCreated) {
try {
await unlink(temporaryPath)
} catch (error) {
cleanupError ??= error
}
}
if (published) {
try {
await syncDirectory(rootDir)
} catch (error) {
cleanupError ??= error
}
}
if (cleanupError) throw cleanupError
}
}
async function readRecord(rootDir, id) {
try {
const raw = await readFile(recordPath(rootDir, id), 'utf8')
const record = JSON.parse(raw)
if (record?.version !== 1 || typeof record.blob !== 'string' || !isCanonicalBase64Url(record.deleteVerifier, 32, verifierPattern)) {
throw new Error('Invalid stored record')
}
return record
} catch (error) {
if (error.code === 'ENOENT') return null
throw error
}
}
function securityHeaders(response) {
response.setHeader('cache-control', 'no-store')
response.setHeader('x-content-type-options', 'nosniff')
response.setHeader('content-security-policy', "default-src 'none'")
response.setHeader('referrer-policy', 'no-referrer')
}
function sendJson(response, status, body) {
response.statusCode = status
response.setHeader('content-type', 'application/json; charset=utf-8')
response.end(JSON.stringify(body))
}
function sendNotFound(response) {
response.statusCode = 404
response.setHeader('content-type', 'application/json; charset=utf-8')
response.end(notFoundBody)
}
function authorizeOrigin(request, response, allowedOrigins) {
const origin = request.headers.origin
if (!origin) return true
if (!allowedOrigins.has(origin)) {
sendJson(response, 403, { error: 'origin_denied' })
return false
}
response.setHeader('access-control-allow-origin', origin)
response.setHeader('vary', 'Origin')
return true
}
function verifierMatches(secret, expectedVerifier) {
const actual = createHash('sha256').update(Buffer.from(secret, 'base64url')).digest()
const expected = Buffer.from(expectedVerifier, 'base64url')
return actual.length === expected.length && timingSafeEqual(actual, expected)
}
async function storageIsReady(rootDir) {
const probePath = join(rootDir, `.${randomBytes(16).toString('hex')}.health`)
try {
await mkdir(rootDir, { recursive: true })
const handle = await open(probePath, 'wx', 0o600)
await handle.close()
await unlink(probePath)
return true
} catch {
await unlink(probePath).catch(() => {})
return false
}
}
function createReadinessProbe(rootDir, cacheMs) {
let cached = null
let cachedAt = 0
let pending = null
return async () => {
const now = Date.now()
if (cached !== null && now - cachedAt < cacheMs) return cached
if (pending) return pending
pending = storageIsReady(rootDir).then((value) => {
cached = value
cachedAt = Date.now()
return value
}).finally(() => { pending = null })
return pending
}
}
function clientAddress(request, trustedProxy, trustedProxyAddresses) {
if (trustedProxy && trustedProxyAddresses.has(request.socket.remoteAddress ?? '')) {
const forwarded = request.headers['x-forwarded-for']
const value = Array.isArray(forwarded) ? forwarded.at(-1) : forwarded
const candidate = value?.split(',').at(-1)?.trim()
if (candidate && isIP(candidate)) return candidate
}
return request.socket.remoteAddress ?? 'unknown'
}
export function createBackupServer(options) {
if (!options?.rootDir) throw new Error('rootDir is required')
const allowedOrigins = new Set(options.allowedOrigins ?? [])
const rateLimit = options.rateLimit ?? { max: 60, windowMs: 60_000 }
const uploadRateLimit = options.uploadRateLimit ?? { max: 10, windowMs: 3_600_000 }
const requestRateLimit = options.requestRateLimit ?? { max: 120, windowMs: 60_000 }
const maxStorageBytes = options.maxStorageBytes ?? 10 * 1024 * 1024 * 1024
const maxRecords = options.maxRecords ?? 10_000
const bodyTimeoutMs = options.bodyTimeoutMs ?? 10_000
const healthCacheMs = options.healthCacheMs ?? 5_000
const maxConcurrentPerClient = options.maxConcurrentPerClient ?? 8
const maxConcurrentTotal = options.maxConcurrentTotal ?? 64
const trustedProxyAddresses = new Set(options.trustedProxyAddresses ?? [])
if (!Number.isSafeInteger(rateLimit.max) || rateLimit.max < 1 || !Number.isSafeInteger(rateLimit.windowMs) || rateLimit.windowMs < 1) {
throw new Error('Invalid rate limit')
}
if (!Number.isSafeInteger(uploadRateLimit.max) || uploadRateLimit.max < 1 || !Number.isSafeInteger(uploadRateLimit.windowMs) || uploadRateLimit.windowMs < 1) {
throw new Error('Invalid upload rate limit')
}
if (!Number.isSafeInteger(requestRateLimit.max) || requestRateLimit.max < 1 || !Number.isSafeInteger(requestRateLimit.windowMs) || requestRateLimit.windowMs < 1) {
throw new Error('Invalid request rate limit')
}
if (!Number.isSafeInteger(maxStorageBytes) || maxStorageBytes < 1) throw new Error('Invalid max storage size')
if (!Number.isSafeInteger(maxRecords) || maxRecords < 1) throw new Error('Invalid max records')
if (!Number.isSafeInteger(bodyTimeoutMs) || bodyTimeoutMs < 1) throw new Error('Invalid body timeout')
if (!Number.isSafeInteger(healthCacheMs) || healthCacheMs < 1) throw new Error('Invalid health cache')
if (!Number.isSafeInteger(maxConcurrentPerClient) || maxConcurrentPerClient < 1) throw new Error('Invalid per-client concurrency')
if (!Number.isSafeInteger(maxConcurrentTotal) || maxConcurrentTotal < maxConcurrentPerClient) throw new Error('Invalid total concurrency')
const consumeRateLimit = createRateLimiter(rateLimit)
const consumeUploadRateLimit = createRateLimiter(uploadRateLimit)
const consumeRequestRateLimit = createRateLimiter(requestRateLimit)
const bodyConcurrency = createConcurrencyLimiter({ perClient: maxConcurrentPerClient, total: maxConcurrentTotal })
const runStorageMutation = createMutationQueue()
const checkReadiness = createReadinessProbe(options.rootDir, healthCacheMs)
const server = createServer(async (request, response) => {
securityHeaders(response)
try {
const url = new URL(request.url, 'http://localhost')
if (!authorizeOrigin(request, response, allowedOrigins)) return
if (url.search) {
sendNotFound(response)
return
}
if (request.method === 'OPTIONS') {
const requestedMethod = request.headers['access-control-request-method']
if (!request.headers.origin || requestedMethod !== 'POST') {
sendJson(response, 400, { error: 'invalid_preflight' })
return
}
response.statusCode = 204
response.setHeader('access-control-allow-methods', 'POST, OPTIONS')
response.setHeader('access-control-allow-headers', 'content-type')
response.setHeader('access-control-max-age', '600')
response.end()
return
}
if (request.method === 'GET' && url.pathname === '/health') {
const ready = await checkReadiness()
sendJson(response, ready ? 200 : 503, { status: ready ? 'ok' : 'unavailable' })
return
}
const address = clientAddress(request, options.trustedProxy === true, trustedProxyAddresses)
if (url.pathname === '/v1/backups/restore' || url.pathname === '/v1/backups/delete') {
const retryAfter = consumeRateLimit(address)
if (retryAfter !== null) {
response.setHeader('retry-after', String(retryAfter))
sendJson(response, 429, { error: 'rate_limited' })
return
}
}
if (request.method === 'POST' && ['/v1/backups', '/v1/backups/restore', '/v1/backups/delete'].includes(url.pathname)) {
const requestRetryAfter = consumeRequestRateLimit(address)
if (requestRetryAfter !== null) {
response.setHeader('retry-after', String(requestRetryAfter))
sendJson(response, 429, { error: 'rate_limited' })
return
}
if (request.headers['content-type']?.split(';', 1)[0].trim().toLowerCase() !== 'application/json') {
sendJson(response, 415, { error: 'unsupported_media_type' })
return
}
if (!bodyConcurrency.enter(address)) {
sendJson(response, 429, { error: 'too_many_requests' })
return
}
try {
const parsed = await readJsonBody(request, bodyTimeoutMs)
if (parsed.error) {
if (parsed.error === 413) response.setHeader('connection', 'close')
const error = parsed.error === 413 ? 'payload_too_large' : parsed.error === 408 ? 'request_timeout' : 'invalid_request'
sendJson(response, parsed.error, { error })
return
}
if (url.pathname === '/v1/backups/restore') {
const keys = parsed.value && typeof parsed.value === 'object' && !Array.isArray(parsed.value) ? Object.keys(parsed.value) : []
if (keys.length !== 1 || keys[0] !== 'id' || !isCanonicalBase64Url(parsed.value.id, 16, idPattern)) {
sendNotFound(response)
return
}
const record = await readRecord(options.rootDir, parsed.value.id)
if (!record) {
sendNotFound(response)
return
}
sendJson(response, 200, { blob: record.blob })
return
}
if (url.pathname === '/v1/backups/delete') {
const keys = parsed.value && typeof parsed.value === 'object' && !Array.isArray(parsed.value) ? Object.keys(parsed.value).sort() : []
const valid = keys.join(',') === 'deleteSecret,id'
&& isCanonicalBase64Url(parsed.value.id, 16, idPattern)
&& isCanonicalBase64Url(parsed.value.deleteSecret, 32, verifierPattern)
if (!valid) {
sendNotFound(response)
return
}
const deleted = await runStorageMutation(() => withStorageLock(options.rootDir, async () => {
const record = await readRecord(options.rootDir, parsed.value.id)
if (!record || !verifierMatches(parsed.value.deleteSecret, record.deleteVerifier)) return false
try {
await unlink(recordPath(options.rootDir, parsed.value.id))
return true
} catch (error) {
if (error.code === 'ENOENT') return false
throw error
}
}))
if (!deleted) {
sendNotFound(response)
return
}
response.statusCode = 204
response.end()
return
}
if (typeof parsed.value?.blob === 'string' && blobPattern.test(parsed.value.blob) && Buffer.from(parsed.value.blob, 'base64url').length > maxBlobBytes) {
sendJson(response, 413, { error: 'payload_too_large' })
return
}
if (!isValidBackup(parsed.value)) {
sendJson(response, 400, { error: 'invalid_request' })
return
}
const uploadRetryAfter = consumeUploadRateLimit(address)
if (uploadRetryAfter !== null) {
response.setHeader('retry-after', String(uploadRetryAfter))
sendJson(response, 429, { error: 'rate_limited' })
return
}
const result = await runStorageMutation(() => withStorageLock(
options.rootDir,
() => createRecord(options.rootDir, parsed.value, maxStorageBytes, maxRecords)
))
if (result === 'duplicate') {
sendJson(response, 409, { error: 'already_exists' })
return
}
if (result === 'full') {
sendJson(response, 507, { error: 'insufficient_storage' })
return
}
sendJson(response, 201, { created: true })
return
} finally {
bodyConcurrency.leave(address)
}
}
sendNotFound(response)
} catch {
if (!response.headersSent) sendJson(response, 500, { error: 'internal_error' })
else response.destroy()
}
})
server.requestTimeout = bodyTimeoutMs + 5_000
server.headersTimeout = Math.min(10_000, bodyTimeoutMs)
server.keepAliveTimeout = 5_000
server.maxRequestsPerSocket = 100
return server
}
services/backup-api/test/server.test.mjs
+390
View File
@@ -0,0 +1,390 @@
import assert from 'node:assert/strict'
import { createHash, randomBytes } from 'node:crypto'
import { mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { createConnection } from 'node:net'
import test from 'node:test'
import lockfile from 'proper-lockfile'
import { createBackupServer } from '../src/server.mjs'
const allowedOrigin = 'https://uploader.24-music.de'
function fixture() {
const deleteSecret = randomBytes(32).toString('base64url')
return {
deleteSecret,
payload: {
id: randomBytes(16).toString('base64url'),
blob: randomBytes(96).toString('base64url'),
deleteVerifier: createHash('sha256').update(Buffer.from(deleteSecret, 'base64url')).digest('base64url')
}
}
}
async function startApi(options = {}) {
const rootDir = await mkdtemp(join(tmpdir(), 'mhu-backup-api-'))
const server = createBackupServer({
rootDir,
allowedOrigins: [allowedOrigin],
rateLimit: { max: 100, windowMs: 60_000 },
...options
})
await new Promise((resolve, reject) => {
server.once('error', reject)
server.listen(0, '127.0.0.1', resolve)
})
return {
rootDir,
server,
baseUrl: `http://127.0.0.1:${server.address().port}`,
async close() {
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()))
await rm(rootDir, { recursive: true, force: true })
}
}
}
function request(api, path, options = {}) {
return fetch(`${api.baseUrl}${path}`, options)
}
test('health reports readiness without storage details and sends security headers', async (t) => {
const api = await startApi()
t.after(() => api.close())
const response = await request(api, '/health')
assert.equal(response.status, 200)
assert.deepEqual(await response.json(), { status: 'ok' })
assert.equal(response.headers.get('cache-control'), 'no-store')
assert.equal(response.headers.get('x-content-type-options'), 'nosniff')
assert.equal(response.headers.get('content-security-policy'), "default-src 'none'")
})
test('creates immutable ciphertext records and restores them after a restart', async (t) => {
const api = await startApi()
const backup = fixture()
t.after(async () => {
if (api.server.listening) await new Promise((resolve) => api.server.close(resolve))
await rm(api.rootDir, { recursive: true, force: true })
})
const created = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
assert.equal(created.status, 201)
await new Promise((resolve) => api.server.close(resolve))
api.server = createBackupServer({ rootDir: api.rootDir, allowedOrigins: [allowedOrigin] })
await new Promise((resolve, reject) => {
api.server.once('error', reject)
api.server.listen(0, '127.0.0.1', resolve)
})
api.baseUrl = `http://127.0.0.1:${api.server.address().port}`
const restored = await request(api, '/v1/backups/restore', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ id: backup.payload.id })
})
assert.equal(restored.status, 200)
assert.deepEqual(await restored.json(), { blob: backup.payload.blob })
const duplicate = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ ...backup.payload, blob: randomBytes(96).toString('base64url') })
})
assert.equal(duplicate.status, 409)
})
test('validates payload shape, content type and decoded blob size', async (t) => {
const api = await startApi()
t.after(() => api.close())
const valid = fixture()
const invalid = [
{ ...valid.payload, id: 'short' },
{ ...valid.payload, blob: 'not+base64url' },
{ ...valid.payload, deleteVerifier: 'short' },
{ id: valid.payload.id, blob: valid.payload.blob },
{ ...valid.payload, extra: true }
]
for (const body of invalid) {
const response = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body)
})
assert.equal(response.status, 400)
}
const wrongType = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'text/plain' },
body: JSON.stringify(valid.payload)
})
assert.equal(wrongType.status, 415)
const oversized = fixture()
oversized.payload.blob = randomBytes(262_145).toString('base64url')
const tooLarge = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(oversized.payload)
})
assert.equal(tooLarge.status, 413)
})
test('deletes only with the matching client secret and returns constant not-found responses', async (t) => {
const api = await startApi()
t.after(() => api.close())
const backup = fixture()
await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
const missing = await request(api, '/v1/backups/restore', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ id: randomBytes(16).toString('base64url') })
})
const wrong = await request(api, '/v1/backups/delete', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ id: backup.payload.id, deleteSecret: randomBytes(32).toString('base64url') })
})
assert.equal(missing.status, 404)
assert.equal(wrong.status, 404)
assert.equal(await missing.text(), await wrong.text())
const deleted = await request(api, '/v1/backups/delete', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ id: backup.payload.id, deleteSecret: backup.deleteSecret })
})
assert.equal(deleted.status, 204)
assert.equal((await readdir(api.rootDir)).length, 0)
})
test('allows only configured origins and supports preflight', async (t) => {
const api = await startApi()
t.after(() => api.close())
const allowed = await request(api, '/health', { headers: { origin: allowedOrigin } })
assert.equal(allowed.headers.get('access-control-allow-origin'), allowedOrigin)
const denied = await request(api, '/health', { headers: { origin: 'https://attacker.example' } })
assert.equal(denied.status, 403)
const preflight = await request(api, '/v1/backups', {
method: 'OPTIONS',
headers: {
origin: allowedOrigin,
'access-control-request-method': 'POST',
'access-control-request-headers': 'content-type'
}
})
assert.equal(preflight.status, 204)
})
test('separately rate limits uploads while restores and health remain available', async (t) => {
const api = await startApi({ uploadRateLimit: { max: 1, windowMs: 60_000 } })
t.after(() => api.close())
const first = fixture()
const second = fixture()
const create = (backup) => request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
assert.equal((await create(first)).status, 201)
assert.equal((await create(second)).status, 429)
const restored = await request(api, '/v1/backups/restore', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ id: first.payload.id })
})
assert.equal(restored.status, 200)
assert.equal((await request(api, '/health')).status, 200)
})
test('rate limits invalid request bodies before validation', async (t) => {
const api = await startApi({ requestRateLimit: { max: 1, windowMs: 60_000 } })
t.after(() => api.close())
const sendInvalid = () => request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ invalid: 'x'.repeat(300_000) })
})
assert.equal((await sendInvalid()).status, 400)
assert.equal((await sendInvalid()).status, 429)
})
test('ignores forwarded client addresses from untrusted socket peers', async (t) => {
const api = await startApi({
trustedProxy: true,
trustedProxyAddresses: [],
rateLimit: { max: 1, windowMs: 60_000 }
})
t.after(() => api.close())
const body = JSON.stringify({ id: randomBytes(16).toString('base64url') })
const restore = (forwarded) => request(api, '/v1/backups/restore', {
method: 'POST',
headers: { 'content-type': 'application/json', 'x-forwarded-for': forwarded },
body
})
assert.equal((await restore('198.51.100.1')).status, 404)
assert.equal((await restore('198.51.100.2')).status, 429)
const created = await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json', 'x-forwarded-for': '198.51.100.3' },
body: JSON.stringify(fixture().payload)
})
assert.equal(created.status, 201)
})
test('uses the last forwarded address from an explicitly trusted proxy', async (t) => {
const api = await startApi({
trustedProxy: true,
trustedProxyAddresses: ['127.0.0.1'],
rateLimit: { max: 1, windowMs: 60_000 }
})
t.after(() => api.close())
const body = JSON.stringify({ id: randomBytes(16).toString('base64url') })
const restore = (forwarded) => request(api, '/v1/backups/restore', {
method: 'POST',
headers: { 'content-type': 'application/json', 'x-forwarded-for': forwarded },
body
})
assert.equal((await restore('198.51.100.1, 203.0.113.9')).status, 404)
assert.equal((await restore('198.51.100.2, 203.0.113.9')).status, 429)
})
test('keeps concurrency leases until storage mutations finish', async (t) => {
const api = await startApi({ maxConcurrentPerClient: 1, maxConcurrentTotal: 1 })
t.after(() => api.close())
const release = await lockfile.lock(api.rootDir, {
realpath: false,
lockfilePath: join(api.rootDir, '.storage.lock')
})
const create = (backup) => request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
const first = create(fixture())
await new Promise((resolve) => setTimeout(resolve, 40))
const second = create(fixture())
let secondStatus
try {
secondStatus = await Promise.race([
second.then((response) => response.status),
new Promise((resolve) => setTimeout(() => resolve('pending'), 100))
])
} finally {
await release()
}
assert.equal((await first).status, 201)
if (secondStatus === 'pending') await second
assert.equal(secondStatus, 429)
})
test('times out incomplete request bodies', async (t) => {
const api = await startApi({ bodyTimeoutMs: 30 })
t.after(() => api.close())
const response = await new Promise((resolve, reject) => {
const socket = createConnection(new URL(api.baseUrl).port, '127.0.0.1')
let data = ''
socket.setEncoding('utf8')
socket.once('error', reject)
socket.on('data', (chunk) => { data += chunk })
socket.on('end', () => resolve(data))
socket.once('connect', () => {
socket.write('POST /v1/backups HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: 10\r\nConnection: close\r\n\r\n{')
})
})
assert.match(response, /^HTTP\/1\.1 408 /)
assert.match(response, /request_timeout/)
})
test('caches health readiness instead of writing on every request', async (t) => {
const api = await startApi({ healthCacheMs: 60_000 })
t.after(() => api.close())
assert.equal((await request(api, '/health')).status, 200)
await rm(api.rootDir, { recursive: true, force: true })
assert.equal((await request(api, '/health')).status, 200)
await assert.rejects(stat(api.rootDir), { code: 'ENOENT' })
})
test('cleans orphaned temporary files and enforces a record limit', async (t) => {
const api = await startApi({ maxRecords: 1 })
t.after(() => api.close())
const orphan = join(api.rootDir, `.${randomBytes(16).toString('hex')}.tmp`)
await writeFile(orphan, 'orphan')
const create = (backup) => request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
assert.equal((await create(fixture())).status, 201)
assert.equal((await create(fixture())).status, 507)
const files = await readdir(api.rootDir)
assert.equal(files.some((name) => name.endsWith('.tmp')), false)
assert.equal(files.filter((name) => name.endsWith('.json')).length, 1)
})
test('enforces atomic storage capacity without blocking existing restores', async (t) => {
const api = await startApi({ maxStorageBytes: 420 })
const secondServer = createBackupServer({ rootDir: api.rootDir, allowedOrigins: [allowedOrigin], maxStorageBytes: 420 })
await new Promise((resolve, reject) => {
secondServer.once('error', reject)
secondServer.listen(0, '127.0.0.1', resolve)
})
t.after(() => api.close())
t.after(() => new Promise((resolve) => secondServer.close(resolve)))
const first = fixture()
const second = fixture()
const create = (backup, baseUrl = api.baseUrl) => fetch(`${baseUrl}/v1/backups`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
const results = await Promise.all([create(first), create(second, `http://127.0.0.1:${secondServer.address().port}`)])
assert.deepEqual(results.map((response) => response.status).sort(), [201, 507])
assert.equal((await readdir(api.rootDir)).length, 1)
})
test('never accepts record ids in URLs and stores no client delete secret', async (t) => {
const api = await startApi()
t.after(() => api.close())
const backup = fixture()
assert.equal((await request(api, `/v1/backups/${backup.payload.id}`)).status, 404)
assert.equal((await request(api, `/v1/backups?backup=${backup.payload.id}`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})).status, 404)
await request(api, '/v1/backups', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(backup.payload)
})
const files = await readdir(api.rootDir)
const stored = await readFile(join(api.rootDir, files[0]), 'utf8')
assert.equal(stored.includes(backup.deleteSecret), false)
})