Files
Multi-Hoster-Upload/lib/cloud-log-sync.js
T
2026-10-15 04:11:20 +02:00

306 lines
12 KiB
JavaScript

const fs = require('node:fs/promises');
const path = require('node:path');
const { randomUUID, createHash } = require('node:crypto');
const { EventEmitter } = require('node:events');
const { ClouddropLogApi, CloudLogError } = require('./clouddrop-log-api');
const MAX_DOCUMENT_BYTES = 700000;
const hash = value => createHash('sha256').update(value).digest('hex');
const validId = value => typeof value === 'string' && /^[a-zA-Z0-9_-]{1,100}$/.test(value);
class CloudLogSync extends EventEmitter {
constructor({ directory, loadSettings, saveSettings, apiFactory, now = Date.now, delayMs = 750 }) {
super();
this.directory = directory;
this.loadSettings = loadSettings;
this.saveSettings = saveSettings;
this.apiFactory = apiFactory || (key => new ClouddropLogApi(key));
this.now = now;
this.delayMs = delayMs;
this.documents = new Map();
this.dirty = new Set();
this.localWrites = Promise.resolve();
this.diskWrites = Promise.resolve();
this.control = Promise.resolve();
this.ready = null;
this.connection = null;
this.timer = null;
this.worker = null;
this.controller = null;
this.phase = 'disabled';
this.errorCode = '';
this.failures = 0;
this.closed = false;
this.retryBlocked = false;
this.documentTime = 0;
}
async initialize() {
if (!this.ready) this.ready = this._load();
await this.ready;
}
async _load() {
await fs.mkdir(this.directory, { recursive: true });
for (const name of await fs.readdir(this.directory)) {
if (!/^[a-f0-9-]{36}\.json$/.test(name)) continue;
let doc;
try { doc = JSON.parse(await fs.readFile(path.join(this.directory, name), 'utf8')); }
catch { throw new CloudLogError('LOCAL_STATE_INVALID'); }
if (doc.id + '.json' !== name || !validId(doc.accountId) || typeof doc.content !== 'string'
|| Buffer.byteLength(doc.content) > MAX_DOCUMENT_BYTES || !validId(doc.folderId)
|| !/^[a-f0-9]{64}$/.test(doc.source) || typeof doc.filename !== 'string') {
throw new CloudLogError('LOCAL_STATE_INVALID');
}
this.documents.set(doc.id, doc);
doc.contentHash = hash(doc.content);
this.documentTime = Math.max(this.documentTime, Number(doc.createdAt) || 0);
}
}
status() {
const settings = this.loadSettings() || {};
const accountId = this.connection?.accountId || settings.accountId;
const docs = [...this.documents.values()].filter(doc => doc.accountId === accountId && doc.source !== hash('connection-check'));
const pending = docs.filter(doc => doc.syncedHash !== doc.contentHash);
return {
enabled: settings.enabled === true, hasApiKey: Boolean(settings.apiKey),
phase: this.phase, errorCode: this.errorCode,
ip: this.connection?.ip || settings.ip || '',
pendingFiles: pending.length,
lastSyncedAt: Math.max(0, ...docs.map(doc => Number(doc.lastSyncedAt) || 0)),
fileCount: docs.length
};
}
notify() { this.emit('status', this.status()); }
configure({ apiKey, enabled }) {
const operation = this.control.then(() => this._configure(apiKey, enabled));
this.control = operation.catch(() => {});
return operation;
}
async _configure(apiKey, enabled) {
await this.initialize();
clearTimeout(this.timer);
this.timer = null;
this.controller?.abort();
if (this.worker) await this.worker;
await this.flushLocal();
const previous = this.loadSettings() || {};
this.retryBlocked = false;
this.connection = null;
if (!enabled) {
await this.saveSettings({ ...previous, enabled: false });
this.phase = 'disabled';
this.errorCode = '';
this.notify();
return this.status();
}
this.phase = 'checking';
this.errorCode = '';
this.notify();
const key = String(apiKey || previous.apiKey || '').trim();
const api = this.apiFactory(key);
const controller = new AbortController();
this.controller = controller;
try {
const identity = await api.identify(controller.signal);
if (!validId(identity.accountId)) throw new CloudLogError('INVALID_RESPONSE');
const folderId = await api.ensureFolder(identity.folderName, controller.signal);
if (!validId(folderId)) throw new CloudLogError('INVALID_RESPONSE');
const connection = { ...identity, folderId, api };
const source = hash('connection-check');
let probe = [...this.documents.values()].find(doc => doc.accountId === identity.accountId
&& doc.folderId === folderId && doc.source === source);
if (!probe) {
probe = this._newDocument(connection, source, 'fileuploader-connection', '');
await this._saveDocument(probe);
this.documents.set(probe.id, probe);
}
if (!probe.remoteId) {
probe.remoteId = await api.createFile(folderId, probe.filename, probe.id, controller.signal);
await this._saveDocument(probe);
}
await api.writeFile(probe.remoteId, '', controller.signal);
const verification = await api.readFile(probe.remoteId, controller.signal);
if (verification.content !== '') throw new CloudLogError('VERIFICATION_FAILED');
probe.syncedHash = hash('');
probe.lastSyncedAt = this.now();
await this._saveDocument(probe);
await this.saveSettings({ enabled: true, apiKey: key, accountId: identity.accountId, ip: identity.ip, folderId });
this.connection = connection;
this.phase = 'ready';
this.failures = 0;
this.notify();
this.schedule();
return this.status();
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
this.notify();
throw new CloudLogError(this.errorCode);
} finally {
if (this.controller === controller) this.controller = null;
}
}
async start() {
try {
await this.initialize();
if (this.loadSettings()?.enabled) await this.configure({ enabled: true });
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
this.notify();
if (!['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode)) {
this.timer = setTimeout(() => { this.timer = null; void this.start(); }, 30000);
this.timer.unref?.();
}
}
}
_newDocument(connection, source, stem, content) {
const id = randomUUID();
this.documentTime = Math.max(this.now(), this.documentTime + 1);
return {
id, accountId: connection.accountId, folderId: connection.folderId, source,
filename: `${stem.replace(/[^\p{L}\p{N}._-]/gu, '_').slice(0, 110)}-${id}.txt`,
content, contentHash: hash(content), remoteId: null, syncedHash: null, lastSyncedAt: 0, createdAt: this.documentTime
};
}
async append(sourcePath, line) {
await this.initialize();
const settings = this.loadSettings() || {};
if (!settings.enabled || !settings.accountId || typeof line !== 'string' || !line) return;
if (Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
this.errorCode = 'LINE_TOO_LARGE';
this.phase = 'error';
this.notify();
return;
}
const connection = this.connection || {
accountId: settings.accountId, folderId: settings.folderId, ip: settings.ip
};
const known = [...this.documents.values()].filter(doc => doc.accountId === settings.accountId && doc.source !== hash('connection-check'));
if (!connection.folderId) connection.folderId = [...this.documents.values()]
.find(doc => doc.accountId === settings.accountId && doc.source === hash('connection-check'))?.folderId;
if (!connection.folderId) return;
const source = hash(`${sourcePath}|${connection.ip}|${connection.folderId}`);
let doc = known.filter(item => item.source === source).sort((a, b) => b.createdAt - a.createdAt)[0];
if (!doc || Buffer.byteLength(doc.content) + Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
doc = this._newDocument(connection, source, path.basename(sourcePath, path.extname(sourcePath)), '');
this.documents.set(doc.id, doc);
}
doc.content += line;
doc.contentHash = hash(doc.content);
this.dirty.add(doc.id);
await this.flushLocal().then(() => this.schedule()).catch(error => {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.notify();
this.schedule();
});
}
async _saveDocument(doc) {
const destination = path.join(this.directory, doc.id + '.json');
const temporary = destination + '.tmp';
const content = JSON.stringify(doc);
const operation = this.diskWrites.then(async () => {
await fs.writeFile(temporary, content, { encoding: 'utf8', mode: 0o600 });
await fs.rename(temporary, destination);
});
this.diskWrites = operation.catch(() => {});
return operation;
}
flushLocal() {
const operation = this.localWrites.then(async () => {
for (const id of [...this.dirty]) {
const doc = this.documents.get(id);
const content = doc.content;
await this._saveDocument({ ...doc, content });
if (doc.content === content) this.dirty.delete(id);
}
});
this.localWrites = operation.catch(() => {});
return operation;
}
schedule(delay = this.delayMs) {
if (this.closed || this.retryBlocked || this.timer || this.worker || !this.loadSettings()?.enabled) return;
this.timer = setTimeout(() => {
this.timer = null;
if (!this.connection) { void this.start(); return; }
this.worker = this._sync().finally(() => { this.worker = null; });
}, delay);
this.timer.unref?.();
}
async _sync() {
const connection = this.connection;
const controller = new AbortController();
this.controller = controller;
try {
await this.flushLocal();
this.phase = 'syncing';
this.notify();
for (const doc of this.documents.values()) {
if (doc.accountId !== connection.accountId || doc.syncedHash === doc.contentHash) continue;
const content = doc.content;
const contentHash = doc.contentHash;
if (!doc.remoteId) {
doc.remoteId = await connection.api.createFile(doc.folderId, doc.filename, doc.id, controller.signal);
await this._saveDocument(doc);
}
await connection.api.writeFile(doc.remoteId, content, controller.signal);
doc.syncedHash = contentHash;
doc.lastSyncedAt = this.now();
await this._saveDocument(doc);
this.notify();
}
this.phase = 'ready';
this.errorCode = '';
this.failures = 0;
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.failures++;
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED'].includes(this.errorCode);
} finally {
if (this.controller === controller) this.controller = null;
this.notify();
const pending = [...this.documents.values()].some(doc => doc.accountId === connection.accountId && doc.syncedHash !== doc.contentHash);
if (pending && !controller.signal.aborted && !this.retryBlocked && !this.closed) {
const delay = this.failures ? Math.min(300000, 5000 * 2 ** Math.min(this.failures - 1, 6)) : this.delayMs;
this.timer = setTimeout(() => { this.timer = null; this.schedule(0); }, delay);
this.timer.unref?.();
}
}
}
_errorCode(error) {
if (error instanceof CloudLogError) return error.code;
if (String(error?.code || '').startsWith('SECRET_STORE_')) return 'KEY_STORAGE_UNAVAILABLE';
if (['EACCES', 'EPERM', 'ENOSPC', 'EIO'].includes(error?.code)) return 'LOCAL_WRITE_FAILED';
return 'CONNECTION_FAILED';
}
async close() {
this.closed = true;
clearTimeout(this.timer);
this.timer = null;
this.controller?.abort();
if (this.worker) await this.worker;
await this.flushLocal();
await this.diskWrites;
}
}
module.exports = { CloudLogSync, MAX_DOCUMENT_BYTES };