306 lines
12 KiB
JavaScript
306 lines
12 KiB
JavaScript
const fs = require('node:fs/promises');
|
|
const path = require('node:path');
|
|
const { randomUUID, createHash } = require('node:crypto');
|
|
const { EventEmitter } = require('node:events');
|
|
const { ClouddropLogApi, CloudLogError } = require('./clouddrop-log-api');
|
|
|
|
const MAX_DOCUMENT_BYTES = 700000;
|
|
const hash = value => createHash('sha256').update(value).digest('hex');
|
|
const validId = value => typeof value === 'string' && /^[a-zA-Z0-9_-]{1,100}$/.test(value);
|
|
|
|
class CloudLogSync extends EventEmitter {
|
|
constructor({ directory, loadSettings, saveSettings, apiFactory, now = Date.now, delayMs = 750 }) {
|
|
super();
|
|
this.directory = directory;
|
|
this.loadSettings = loadSettings;
|
|
this.saveSettings = saveSettings;
|
|
this.apiFactory = apiFactory || (key => new ClouddropLogApi(key));
|
|
this.now = now;
|
|
this.delayMs = delayMs;
|
|
this.documents = new Map();
|
|
this.dirty = new Set();
|
|
this.localWrites = Promise.resolve();
|
|
this.diskWrites = Promise.resolve();
|
|
this.control = Promise.resolve();
|
|
this.ready = null;
|
|
this.connection = null;
|
|
this.timer = null;
|
|
this.worker = null;
|
|
this.controller = null;
|
|
this.phase = 'disabled';
|
|
this.errorCode = '';
|
|
this.failures = 0;
|
|
this.closed = false;
|
|
this.retryBlocked = false;
|
|
this.documentTime = 0;
|
|
}
|
|
|
|
async initialize() {
|
|
if (!this.ready) this.ready = this._load();
|
|
await this.ready;
|
|
}
|
|
|
|
async _load() {
|
|
await fs.mkdir(this.directory, { recursive: true });
|
|
for (const name of await fs.readdir(this.directory)) {
|
|
if (!/^[a-f0-9-]{36}\.json$/.test(name)) continue;
|
|
let doc;
|
|
try { doc = JSON.parse(await fs.readFile(path.join(this.directory, name), 'utf8')); }
|
|
catch { throw new CloudLogError('LOCAL_STATE_INVALID'); }
|
|
if (doc.id + '.json' !== name || !validId(doc.accountId) || typeof doc.content !== 'string'
|
|
|| Buffer.byteLength(doc.content) > MAX_DOCUMENT_BYTES || !validId(doc.folderId)
|
|
|| !/^[a-f0-9]{64}$/.test(doc.source) || typeof doc.filename !== 'string') {
|
|
throw new CloudLogError('LOCAL_STATE_INVALID');
|
|
}
|
|
this.documents.set(doc.id, doc);
|
|
doc.contentHash = hash(doc.content);
|
|
this.documentTime = Math.max(this.documentTime, Number(doc.createdAt) || 0);
|
|
}
|
|
}
|
|
|
|
status() {
|
|
const settings = this.loadSettings() || {};
|
|
const accountId = this.connection?.accountId || settings.accountId;
|
|
const docs = [...this.documents.values()].filter(doc => doc.accountId === accountId);
|
|
const pending = docs.filter(doc => doc.syncedHash !== doc.contentHash);
|
|
return {
|
|
enabled: settings.enabled === true, hasApiKey: Boolean(settings.apiKey),
|
|
phase: this.phase, errorCode: this.errorCode,
|
|
ip: this.connection?.ip || settings.ip || '',
|
|
pendingFiles: pending.length,
|
|
lastSyncedAt: Math.max(0, ...docs.map(doc => Number(doc.lastSyncedAt) || 0)),
|
|
fileCount: docs.length
|
|
};
|
|
}
|
|
|
|
notify() { this.emit('status', this.status()); }
|
|
|
|
configure({ apiKey, enabled }) {
|
|
const operation = this.control.then(() => this._configure(apiKey, enabled));
|
|
this.control = operation.catch(() => {});
|
|
return operation;
|
|
}
|
|
|
|
async _configure(apiKey, enabled) {
|
|
await this.initialize();
|
|
clearTimeout(this.timer);
|
|
this.timer = null;
|
|
this.controller?.abort();
|
|
if (this.worker) await this.worker;
|
|
await this.flushLocal();
|
|
const previous = this.loadSettings() || {};
|
|
this.retryBlocked = false;
|
|
this.connection = null;
|
|
if (!enabled) {
|
|
await this.saveSettings({ ...previous, enabled: false });
|
|
this.phase = 'disabled';
|
|
this.errorCode = '';
|
|
this.notify();
|
|
return this.status();
|
|
}
|
|
this.phase = 'checking';
|
|
this.errorCode = '';
|
|
this.notify();
|
|
const key = String(apiKey || previous.apiKey || '').trim();
|
|
const api = this.apiFactory(key);
|
|
const controller = new AbortController();
|
|
this.controller = controller;
|
|
try {
|
|
const identity = await api.identify(controller.signal);
|
|
if (!validId(identity.accountId)) throw new CloudLogError('INVALID_RESPONSE');
|
|
const folderId = await api.ensureFolder(identity.folderName, controller.signal);
|
|
if (!validId(folderId)) throw new CloudLogError('INVALID_RESPONSE');
|
|
const connection = { ...identity, folderId, api };
|
|
const source = hash('connection-check');
|
|
let probe = [...this.documents.values()].find(doc => doc.accountId === identity.accountId
|
|
&& doc.folderId === folderId && doc.source === source);
|
|
if (!probe) {
|
|
probe = this._newDocument(connection, source, 'fileuploader-connection', '');
|
|
await this._saveDocument(probe);
|
|
this.documents.set(probe.id, probe);
|
|
}
|
|
if (!probe.remoteId) {
|
|
probe.remoteId = await api.createFile(folderId, probe.filename, probe.id, controller.signal);
|
|
await this._saveDocument(probe);
|
|
}
|
|
await api.writeFile(probe.remoteId, '', controller.signal);
|
|
const verification = await api.readFile(probe.remoteId, controller.signal);
|
|
if (verification.content !== '') throw new CloudLogError('VERIFICATION_FAILED');
|
|
probe.syncedHash = hash('');
|
|
probe.lastSyncedAt = this.now();
|
|
await this._saveDocument(probe);
|
|
await this.saveSettings({ enabled: true, apiKey: key, accountId: identity.accountId, ip: identity.ip, folderId });
|
|
this.connection = connection;
|
|
this.phase = 'ready';
|
|
this.failures = 0;
|
|
this.notify();
|
|
this.schedule();
|
|
return this.status();
|
|
} catch (error) {
|
|
this.phase = 'error';
|
|
this.errorCode = this._errorCode(error);
|
|
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
|
|
this.notify();
|
|
throw new CloudLogError(this.errorCode);
|
|
} finally {
|
|
if (this.controller === controller) this.controller = null;
|
|
}
|
|
}
|
|
|
|
async start() {
|
|
try {
|
|
await this.initialize();
|
|
if (this.loadSettings()?.enabled) await this.configure({ enabled: true });
|
|
} catch (error) {
|
|
this.phase = 'error';
|
|
this.errorCode = this._errorCode(error);
|
|
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
|
|
this.notify();
|
|
if (!['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode)) {
|
|
this.timer = setTimeout(() => { this.timer = null; void this.start(); }, 30000);
|
|
this.timer.unref?.();
|
|
}
|
|
}
|
|
}
|
|
|
|
_newDocument(connection, source, stem, content) {
|
|
const id = randomUUID();
|
|
this.documentTime = Math.max(this.now(), this.documentTime + 1);
|
|
return {
|
|
id, accountId: connection.accountId, folderId: connection.folderId, source,
|
|
filename: `${stem.replace(/[^\p{L}\p{N}._-]/gu, '_').slice(0, 110)}-${id}.txt`,
|
|
content, contentHash: hash(content), remoteId: null, syncedHash: null, lastSyncedAt: 0, createdAt: this.documentTime
|
|
};
|
|
}
|
|
|
|
async append(sourcePath, line) {
|
|
await this.initialize();
|
|
const settings = this.loadSettings() || {};
|
|
if (!settings.enabled || !settings.accountId || typeof line !== 'string' || !line) return;
|
|
if (Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
|
|
this.errorCode = 'LINE_TOO_LARGE';
|
|
this.phase = 'error';
|
|
this.notify();
|
|
return;
|
|
}
|
|
const connection = this.connection || {
|
|
accountId: settings.accountId, folderId: settings.folderId, ip: settings.ip
|
|
};
|
|
const known = [...this.documents.values()].filter(doc => doc.accountId === settings.accountId && doc.source !== hash('connection-check'));
|
|
if (!connection.folderId) connection.folderId = [...this.documents.values()]
|
|
.find(doc => doc.accountId === settings.accountId && doc.source === hash('connection-check'))?.folderId;
|
|
if (!connection.folderId) return;
|
|
const source = hash(`${sourcePath}|${connection.ip}|${connection.folderId}`);
|
|
let doc = known.filter(item => item.source === source).sort((a, b) => b.createdAt - a.createdAt)[0];
|
|
if (!doc || Buffer.byteLength(doc.content) + Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
|
|
doc = this._newDocument(connection, source, path.basename(sourcePath, path.extname(sourcePath)), '');
|
|
this.documents.set(doc.id, doc);
|
|
}
|
|
doc.content += line;
|
|
doc.contentHash = hash(doc.content);
|
|
this.dirty.add(doc.id);
|
|
await this.flushLocal().then(() => this.schedule()).catch(error => {
|
|
this.phase = 'error';
|
|
this.errorCode = this._errorCode(error);
|
|
this.notify();
|
|
this.schedule();
|
|
});
|
|
}
|
|
|
|
async _saveDocument(doc) {
|
|
const destination = path.join(this.directory, doc.id + '.json');
|
|
const temporary = destination + '.tmp';
|
|
const content = JSON.stringify(doc);
|
|
const operation = this.diskWrites.then(async () => {
|
|
await fs.writeFile(temporary, content, { encoding: 'utf8', mode: 0o600 });
|
|
await fs.rename(temporary, destination);
|
|
});
|
|
this.diskWrites = operation.catch(() => {});
|
|
return operation;
|
|
}
|
|
|
|
flushLocal() {
|
|
const operation = this.localWrites.then(async () => {
|
|
for (const id of [...this.dirty]) {
|
|
const doc = this.documents.get(id);
|
|
const content = doc.content;
|
|
await this._saveDocument({ ...doc, content });
|
|
if (doc.content === content) this.dirty.delete(id);
|
|
}
|
|
});
|
|
this.localWrites = operation.catch(() => {});
|
|
return operation;
|
|
}
|
|
|
|
schedule(delay = this.delayMs) {
|
|
if (this.closed || this.retryBlocked || this.timer || this.worker || !this.loadSettings()?.enabled) return;
|
|
this.timer = setTimeout(() => {
|
|
this.timer = null;
|
|
if (!this.connection) { void this.start(); return; }
|
|
this.worker = this._sync().finally(() => { this.worker = null; });
|
|
}, delay);
|
|
this.timer.unref?.();
|
|
}
|
|
|
|
async _sync() {
|
|
const connection = this.connection;
|
|
const controller = new AbortController();
|
|
this.controller = controller;
|
|
try {
|
|
await this.flushLocal();
|
|
this.phase = 'syncing';
|
|
this.notify();
|
|
for (const doc of this.documents.values()) {
|
|
if (doc.accountId !== connection.accountId || doc.syncedHash === doc.contentHash) continue;
|
|
const content = doc.content;
|
|
const contentHash = doc.contentHash;
|
|
if (!doc.remoteId) {
|
|
doc.remoteId = await connection.api.createFile(doc.folderId, doc.filename, doc.id, controller.signal);
|
|
await this._saveDocument(doc);
|
|
}
|
|
await connection.api.writeFile(doc.remoteId, content, controller.signal);
|
|
doc.syncedHash = contentHash;
|
|
doc.lastSyncedAt = this.now();
|
|
await this._saveDocument(doc);
|
|
this.notify();
|
|
}
|
|
this.phase = 'ready';
|
|
this.errorCode = '';
|
|
this.failures = 0;
|
|
} catch (error) {
|
|
this.phase = 'error';
|
|
this.errorCode = this._errorCode(error);
|
|
this.failures++;
|
|
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED'].includes(this.errorCode);
|
|
} finally {
|
|
if (this.controller === controller) this.controller = null;
|
|
this.notify();
|
|
const pending = [...this.documents.values()].some(doc => doc.accountId === connection.accountId && doc.syncedHash !== doc.contentHash);
|
|
if (pending && !controller.signal.aborted && !this.retryBlocked && !this.closed) {
|
|
const delay = this.failures ? Math.min(300000, 5000 * 2 ** Math.min(this.failures - 1, 6)) : this.delayMs;
|
|
this.timer = setTimeout(() => { this.timer = null; this.schedule(0); }, delay);
|
|
this.timer.unref?.();
|
|
}
|
|
}
|
|
}
|
|
|
|
_errorCode(error) {
|
|
if (error instanceof CloudLogError) return error.code;
|
|
if (String(error?.code || '').startsWith('SECRET_STORE_')) return 'KEY_STORAGE_UNAVAILABLE';
|
|
if (['EACCES', 'EPERM', 'ENOSPC', 'EIO'].includes(error?.code)) return 'LOCAL_WRITE_FAILED';
|
|
return 'CONNECTION_FAILED';
|
|
}
|
|
|
|
async close() {
|
|
this.closed = true;
|
|
clearTimeout(this.timer);
|
|
this.timer = null;
|
|
this.controller?.abort();
|
|
if (this.worker) await this.worker;
|
|
await this.flushLocal();
|
|
await this.diskWrites;
|
|
}
|
|
}
|
|
|
|
module.exports = { CloudLogSync, MAX_DOCUMENT_BYTES };
|