Add persistent Clouddrop synchronization for successful upload logs
CI / verify (push) Canceled after 0s

This commit is contained in:
Sucukdeluxe committed 2026-10-15 04:09:27 +02:00
1 parent be57ce13e2
commit 6182aa2262
13 files changed
+650 -2

No files matched your search

+305
View File
@@ -0,0 +1,305 @@
const fs = require('node:fs/promises');
const path = require('node:path');
const { randomUUID, createHash } = require('node:crypto');
const { EventEmitter } = require('node:events');
const { ClouddropLogApi, CloudLogError } = require('./clouddrop-log-api');
const MAX_DOCUMENT_BYTES = 700000;
const hash = value => createHash('sha256').update(value).digest('hex');
const validId = value => typeof value === 'string' && /^[a-zA-Z0-9_-]{1,100}$/.test(value);
class CloudLogSync extends EventEmitter {
constructor({ directory, loadSettings, saveSettings, apiFactory, now = Date.now, delayMs = 750 }) {
super();
this.directory = directory;
this.loadSettings = loadSettings;
this.saveSettings = saveSettings;
this.apiFactory = apiFactory || (key => new ClouddropLogApi(key));
this.now = now;
this.delayMs = delayMs;
this.documents = new Map();
this.dirty = new Set();
this.localWrites = Promise.resolve();
this.diskWrites = Promise.resolve();
this.control = Promise.resolve();
this.ready = null;
this.connection = null;
this.timer = null;
this.worker = null;
this.controller = null;
this.phase = 'disabled';
this.errorCode = '';
this.failures = 0;
this.closed = false;
this.retryBlocked = false;
this.documentTime = 0;
}
async initialize() {
if (!this.ready) this.ready = this._load();
await this.ready;
}
async _load() {
await fs.mkdir(this.directory, { recursive: true });
for (const name of await fs.readdir(this.directory)) {
if (!/^[a-f0-9-]{36}\.json$/.test(name)) continue;
let doc;
try { doc = JSON.parse(await fs.readFile(path.join(this.directory, name), 'utf8')); }
catch { throw new CloudLogError('LOCAL_STATE_INVALID'); }
if (doc.id + '.json' !== name || !validId(doc.accountId) || typeof doc.content !== 'string'
|| Buffer.byteLength(doc.content) > MAX_DOCUMENT_BYTES || !validId(doc.folderId)
|| !/^[a-f0-9]{64}$/.test(doc.source) || typeof doc.filename !== 'string') {
throw new CloudLogError('LOCAL_STATE_INVALID');
}
this.documents.set(doc.id, doc);
doc.contentHash = hash(doc.content);
this.documentTime = Math.max(this.documentTime, Number(doc.createdAt) || 0);
}
}
status() {
const settings = this.loadSettings() || {};
const accountId = this.connection?.accountId || settings.accountId;
const docs = [...this.documents.values()].filter(doc => doc.accountId === accountId);
const pending = docs.filter(doc => doc.syncedHash !== doc.contentHash);
return {
enabled: settings.enabled === true, hasApiKey: Boolean(settings.apiKey),
phase: this.phase, errorCode: this.errorCode,
ip: this.connection?.ip || settings.ip || '',
pendingFiles: pending.length,
lastSyncedAt: Math.max(0, ...docs.map(doc => Number(doc.lastSyncedAt) || 0)),
fileCount: docs.length
};
}
notify() { this.emit('status', this.status()); }
configure({ apiKey, enabled }) {
const operation = this.control.then(() => this._configure(apiKey, enabled));
this.control = operation.catch(() => {});
return operation;
}
async _configure(apiKey, enabled) {
await this.initialize();
clearTimeout(this.timer);
this.timer = null;
this.controller?.abort();
if (this.worker) await this.worker;
await this.flushLocal();
const previous = this.loadSettings() || {};
this.retryBlocked = false;
this.connection = null;
if (!enabled) {
await this.saveSettings({ ...previous, enabled: false });
this.phase = 'disabled';
this.errorCode = '';
this.notify();
return this.status();
}
this.phase = 'checking';
this.errorCode = '';
this.notify();
const key = String(apiKey || previous.apiKey || '').trim();
const api = this.apiFactory(key);
const controller = new AbortController();
this.controller = controller;
try {
const identity = await api.identify(controller.signal);
if (!validId(identity.accountId)) throw new CloudLogError('INVALID_RESPONSE');
const folderId = await api.ensureFolder(identity.folderName, controller.signal);
if (!validId(folderId)) throw new CloudLogError('INVALID_RESPONSE');
const connection = { ...identity, folderId, api };
const source = hash('connection-check');
let probe = [...this.documents.values()].find(doc => doc.accountId === identity.accountId
&& doc.folderId === folderId && doc.source === source);
if (!probe) {
probe = this._newDocument(connection, source, 'fileuploader-connection', '');
await this._saveDocument(probe);
this.documents.set(probe.id, probe);
}
if (!probe.remoteId) {
probe.remoteId = await api.createFile(folderId, probe.filename, probe.id, controller.signal);
await this._saveDocument(probe);
}
await api.writeFile(probe.remoteId, '', controller.signal);
const verification = await api.readFile(probe.remoteId, controller.signal);
if (verification.content !== '') throw new CloudLogError('VERIFICATION_FAILED');
probe.syncedHash = hash('');
probe.lastSyncedAt = this.now();
await this._saveDocument(probe);
await this.saveSettings({ enabled: true, apiKey: key, accountId: identity.accountId, ip: identity.ip, folderId });
this.connection = connection;
this.phase = 'ready';
this.failures = 0;
this.notify();
this.schedule();
return this.status();
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
this.notify();
throw new CloudLogError(this.errorCode);
} finally {
if (this.controller === controller) this.controller = null;
}
}
async start() {
try {
await this.initialize();
if (this.loadSettings()?.enabled) await this.configure({ enabled: true });
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode);
this.notify();
if (!['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED', 'LOCAL_STATE_INVALID'].includes(this.errorCode)) {
this.timer = setTimeout(() => { this.timer = null; void this.start(); }, 30000);
this.timer.unref?.();
}
}
}
_newDocument(connection, source, stem, content) {
const id = randomUUID();
this.documentTime = Math.max(this.now(), this.documentTime + 1);
return {
id, accountId: connection.accountId, folderId: connection.folderId, source,
filename: `${stem.replace(/[^\p{L}\p{N}._-]/gu, '_').slice(0, 110)}-${id}.txt`,
content, contentHash: hash(content), remoteId: null, syncedHash: null, lastSyncedAt: 0, createdAt: this.documentTime
};
}
async append(sourcePath, line) {
await this.initialize();
const settings = this.loadSettings() || {};
if (!settings.enabled || !settings.accountId || typeof line !== 'string' || !line) return;
if (Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
this.errorCode = 'LINE_TOO_LARGE';
this.phase = 'error';
this.notify();
return;
}
const connection = this.connection || {
accountId: settings.accountId, folderId: settings.folderId, ip: settings.ip
};
const known = [...this.documents.values()].filter(doc => doc.accountId === settings.accountId && doc.source !== hash('connection-check'));
if (!connection.folderId) connection.folderId = [...this.documents.values()]
.find(doc => doc.accountId === settings.accountId && doc.source === hash('connection-check'))?.folderId;
if (!connection.folderId) return;
const source = hash(`${sourcePath}|${connection.ip}|${connection.folderId}`);
let doc = known.filter(item => item.source === source).sort((a, b) => b.createdAt - a.createdAt)[0];
if (!doc || Buffer.byteLength(doc.content) + Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
doc = this._newDocument(connection, source, path.basename(sourcePath, path.extname(sourcePath)), '');
this.documents.set(doc.id, doc);
}
doc.content += line;
doc.contentHash = hash(doc.content);
this.dirty.add(doc.id);
await this.flushLocal().then(() => this.schedule()).catch(error => {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.notify();
this.schedule();
});
}
async _saveDocument(doc) {
const destination = path.join(this.directory, doc.id + '.json');
const temporary = destination + '.tmp';
const content = JSON.stringify(doc);
const operation = this.diskWrites.then(async () => {
await fs.writeFile(temporary, content, { encoding: 'utf8', mode: 0o600 });
await fs.rename(temporary, destination);
});
this.diskWrites = operation.catch(() => {});
return operation;
}
flushLocal() {
const operation = this.localWrites.then(async () => {
for (const id of [...this.dirty]) {
const doc = this.documents.get(id);
const content = doc.content;
await this._saveDocument({ ...doc, content });
if (doc.content === content) this.dirty.delete(id);
}
});
this.localWrites = operation.catch(() => {});
return operation;
}
schedule(delay = this.delayMs) {
if (this.closed || this.retryBlocked || this.timer || this.worker || !this.loadSettings()?.enabled) return;
this.timer = setTimeout(() => {
this.timer = null;
if (!this.connection) { void this.start(); return; }
this.worker = this._sync().finally(() => { this.worker = null; });
}, delay);
this.timer.unref?.();
}
async _sync() {
const connection = this.connection;
const controller = new AbortController();
this.controller = controller;
try {
await this.flushLocal();
this.phase = 'syncing';
this.notify();
for (const doc of this.documents.values()) {
if (doc.accountId !== connection.accountId || doc.syncedHash === doc.contentHash) continue;
const content = doc.content;
const contentHash = doc.contentHash;
if (!doc.remoteId) {
doc.remoteId = await connection.api.createFile(doc.folderId, doc.filename, doc.id, controller.signal);
await this._saveDocument(doc);
}
await connection.api.writeFile(doc.remoteId, content, controller.signal);
doc.syncedHash = contentHash;
doc.lastSyncedAt = this.now();
await this._saveDocument(doc);
this.notify();
}
this.phase = 'ready';
this.errorCode = '';
this.failures = 0;
} catch (error) {
this.phase = 'error';
this.errorCode = this._errorCode(error);
this.failures++;
this.retryBlocked = ['INVALID_KEY', 'ACCESS_DENIED', 'UNRESTRICTED_KEY_REQUIRED'].includes(this.errorCode);
} finally {
if (this.controller === controller) this.controller = null;
this.notify();
const pending = [...this.documents.values()].some(doc => doc.accountId === connection.accountId && doc.syncedHash !== doc.contentHash);
if (pending && !controller.signal.aborted && !this.retryBlocked && !this.closed) {
const delay = this.failures ? Math.min(300000, 5000 * 2 ** Math.min(this.failures - 1, 6)) : this.delayMs;
this.timer = setTimeout(() => { this.timer = null; this.schedule(0); }, delay);
this.timer.unref?.();
}
}
}
_errorCode(error) {
if (error instanceof CloudLogError) return error.code;
if (String(error?.code || '').startsWith('SECRET_STORE_')) return 'KEY_STORAGE_UNAVAILABLE';
if (['EACCES', 'EPERM', 'ENOSPC', 'EIO'].includes(error?.code)) return 'LOCAL_WRITE_FAILED';
return 'CONNECTION_FAILED';
}
async close() {
this.closed = true;
clearTimeout(this.timer);
this.timer = null;
this.controller?.abort();
if (this.worker) await this.worker;
await this.flushLocal();
await this.diskWrites;
}
}
module.exports = { CloudLogSync, MAX_DOCUMENT_BYTES };
+119
View File
@@ -0,0 +1,119 @@
const { request } = require('undici');
const { isIP } = require('node:net');
class CloudLogError extends Error {
constructor(code, status = 0) {
super(code);
this.code = code;
this.status = status;
}
}
class ClouddropLogApi {
constructor(apiKey, options = {}) {
this.apiKey = String(apiKey || '').trim();
this.request = options.request || request;
}
async call(route, { method = 'GET', body, signal, idempotencyKey } = {}) {
const response = await this.request(`https://clouddrop.cc/api/cloud${route}`, {
method, signal, maxRedirections: 0, headersTimeout: 20000, bodyTimeout: 20000,
headers: {
Authorization: `Bearer ${this.apiKey}`,
Accept: 'application/json',
'Content-Type': 'application/json',
'User-Agent': 'Multi-Hoster-Upload Log Sync',
...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {})
},
...(body === undefined ? {} : { body: JSON.stringify(body) })
});
const chunks = [];
let size = 0;
for await (const chunk of response.body) {
size += chunk.length;
if (size > 2 * 1024 * 1024) {
response.body.destroy();
throw new CloudLogError('RESPONSE_TOO_LARGE');
}
chunks.push(chunk);
}
if (response.statusCode < 200 || response.statusCode >= 300) {
const code = response.statusCode === 401 ? 'INVALID_KEY'
: response.statusCode === 403 ? 'ACCESS_DENIED'
: response.statusCode === 429 ? 'RATE_LIMIT' : 'REMOTE_ERROR';
throw new CloudLogError(code, response.statusCode);
}
let data;
try { data = JSON.parse(Buffer.concat(chunks).toString('utf8')); }
catch { throw new CloudLogError('INVALID_RESPONSE', response.statusCode); }
return data;
}
async identify(signal) {
if (!/^cd_[a-f0-9]{32}$/.test(this.apiKey)) throw new CloudLogError('INVALID_KEY');
const user = await this.call('/me', { signal });
if (typeof user.id !== 'string' || !user.id) throw new CloudLogError('INVALID_RESPONSE');
const response = await this.request('https://api64.ipify.org?format=json', {
signal, maxRedirections: 0, headersTimeout: 15000, bodyTimeout: 15000,
headers: { Accept: 'application/json' }
});
const text = await response.body.text();
let ip;
try { ip = JSON.parse(text).ip; } catch {}
if (response.statusCode !== 200 || !isIP(ip || '')) throw new CloudLogError('IP_UNAVAILABLE');
return { accountId: user.id, ip, folderName: ip.replaceAll(':', '_') };
}
async ensureFolder(name, signal) {
const folders = await this.call('/folders?limit=5000', { signal });
if (!Array.isArray(folders)) throw new CloudLogError('INVALID_RESPONSE');
const existing = folders.find(folder => folder.name === name && !folder.parentId);
if (existing) return String(existing.id);
try {
const created = await this.call('/folder', { method: 'POST', signal, body: { name, parentId: null } });
if (!created.id || created.file?.parentId) throw new CloudLogError('UNRESTRICTED_KEY_REQUIRED');
return String(created.id);
} catch (error) {
if (error.status !== 409) throw error;
const refreshed = await this.call('/folders?limit=5000', { signal });
const folder = refreshed.find(item => item.name === name && !item.parentId);
if (!folder) throw new CloudLogError('UNRESTRICTED_KEY_REQUIRED');
return String(folder.id);
}
}
async createFile(folderId, name, id, signal) {
let created;
try {
created = await this.call('/create-file', {
method: 'POST', signal, idempotencyKey: `mhu-log:${id}`,
body: { parentId: folderId, name }
});
} catch (error) {
if (error.status !== 409) throw error;
const existing = await this.call(`/files?parent=${encodeURIComponent(folderId)}&limit=2500&sync=${encodeURIComponent(id)}`, { signal });
const file = existing.files?.find(item => item.filename === name && item.parentId === folderId && !item.isFolder);
if (!file) throw error;
created = { id: file.id, filename: file.filename };
}
if (!created.id || created.filename !== name) throw new CloudLogError('INVALID_RESPONSE');
const listing = await this.call(`/files?parent=${encodeURIComponent(folderId)}&limit=2500&sync=${encodeURIComponent(id)}`, { signal });
if (!Array.isArray(listing.files) || !listing.files.some(file => file.id === created.id && file.parentId === folderId)) {
throw new CloudLogError('UNRESTRICTED_KEY_REQUIRED');
}
return String(created.id);
}
async writeFile(id, content, signal) {
const data = await this.call(`/files/${encodeURIComponent(id)}/content`, {
method: 'PUT', signal, body: { content }
});
if (data.ok !== true || data.size !== Buffer.byteLength(content)) throw new CloudLogError('INVALID_RESPONSE');
}
async readFile(id, signal) {
return this.call(`/files/${encodeURIComponent(id)}/content`, { signal });
}
}
module.exports = { ClouddropLogApi, CloudLogError };
+12 -1
View File
@@ -62,6 +62,7 @@ const DEFAULTS = {
alwaysOnTop: false,
shutdownAfterFinish: 'nothing', // nothing | sleep | shutdown | restart
logFilePath: '',
cloudLogSync: { enabled: false, apiKey: '', accountId: '', ip: '', folderId: '' },
sessionLog: false, // legacy boolean (kept for back-compat reads); normalized into logMode on load
logVerbose: false, // when true, [DEBUG] level entries are written to debug.log
webhookUrl: '', // POST target on batch-done (Discord or generic JSON)
@@ -499,7 +500,7 @@ class ConfigStore {
const hosters = this._clone(config.hosters || {});
const globalSettings = this._clone(config.globalSettings || {});
delete globalSettings.allowPlaintextCredentialStorage;
secretStore.encryptCredentials({ hosters });
secretStore.encryptCredentials({ hosters, globalSettings });
return JSON.stringify({ ...config, globalSettings, hosters }, null, 2);
}
@@ -721,6 +722,7 @@ class ConfigStore {
uploadRecovery: currentGlobalSettings.uploadRecovery ?? null,
lastBrowseDirectory: currentGlobalSettings.lastBrowseDirectory || '',
diagnostics: this._clone(currentGlobalSettings.diagnostics || {}),
cloudLogSync: this._clone(currentGlobalSettings.cloudLogSync || {}),
historyRetention: currentGlobalSettings.historyRetention || 'all',
folderMonitor: {
...currentFolderMonitor,
@@ -738,6 +740,15 @@ class ConfigStore {
});
}
saveCloudLogSettings(settings) {
const snapshot = this._clone(settings);
return this._enqueueWrite(() => {
const current = this.load();
current.globalSettings.cloudLogSync = snapshot;
return this._commit(current);
});
}
saveRemoteSettings(remoteSettings, createToken) {
const incoming = this._clone(remoteSettings || {});
return this._enqueueWrite(async () => {
+3
View File
@@ -77,6 +77,9 @@ function decryptField(value) {
}
function mapHosterAccounts(config, fn) {
if (config?.globalSettings?.cloudLogSync?.apiKey) {
config.globalSettings.cloudLogSync.apiKey = fn(config.globalSettings.cloudLogSync.apiKey);
}
if (!config || !config.hosters || typeof config.hosters !== 'object') return config;
for (const accounts of Object.values(config.hosters)) {
if (!Array.isArray(accounts)) continue;
+5
View File
@@ -35,6 +35,11 @@ function createPortableSettingsSnapshot(config) {
snapshot.globalSettings.pendingQueue = null;
snapshot.globalSettings.uploadRecovery = null;
snapshot.globalSettings.lastBrowseDirectory = '';
if (snapshot.globalSettings.cloudLogSync) {
delete snapshot.globalSettings.cloudLogSync.accountId;
delete snapshot.globalSettings.cloudLogSync.ip;
delete snapshot.globalSettings.cloudLogSync.folderId;
}
if (snapshot.globalSettings.folderMonitor && typeof snapshot.globalSettings.folderMonitor === 'object') {
snapshot.globalSettings.folderMonitor.paused = false;
snapshot.globalSettings.folderMonitor.pausedAt = null;