Add CloudDrop log backfill and clear connection status
CI / verify (push) Canceled after 0s

This commit is contained in:
Sucukdeluxe committed 2026-10-15 12:57:26 +02:00
1 parent 63abaa819c
commit 63296e7ad3
7 files changed
+226 -19

No files matched your search

+101 -3
View File
@@ -3,6 +3,7 @@ const path = require('node:path');
const { randomUUID, createHash } = require('node:crypto');
const { EventEmitter } = require('node:events');
const { ClouddropLogApi, CloudLogError } = require('./clouddrop-log-api');
const { iterateUploadLogEntries } = require('./upload-log');
const MAX_DOCUMENT_BYTES = 700000;
const hash = value => createHash('sha256').update(value).digest('hex');
@@ -35,6 +36,9 @@ class CloudLogSync extends EventEmitter {
this.closed = false;
this.retryBlocked = false;
this.documentTime = 0;
this.backfillJob = null;
this.backfillContext = null;
this.backfillResult = null;
}
async initialize() {
@@ -71,7 +75,9 @@ class CloudLogSync extends EventEmitter {
ip: this.connection?.ip || settings.ip || '',
pendingFiles: pending.length,
lastSyncedAt: Math.max(0, ...docs.map(doc => Number(doc.lastSyncedAt) || 0)),
fileCount: docs.length
fileCount: docs.length,
backfillRunning: Boolean(this.backfillJob),
backfillResult: this.backfillResult ? { ...this.backfillResult } : null
};
}
@@ -158,7 +164,7 @@ class CloudLogSync extends EventEmitter {
this.errorCode = this._errorCode(error);
this.retryBlocked = blocksRetry(this.errorCode);
this.notify();
if (!blocksRetry(this.errorCode)) {
if (!this.closed && !blocksRetry(this.errorCode)) {
this.timer = setTimeout(() => { this.timer = null; void this.start(); }, 30000);
this.timer.unref?.();
}
@@ -175,7 +181,7 @@ class CloudLogSync extends EventEmitter {
};
}
async append(sourcePath, line) {
async append(sourcePath, line, trackBackfill = true) {
await this.initialize();
const settings = this.loadSettings() || {};
if (!settings.enabled || !settings.accountId || typeof line !== 'string' || !line) return;
@@ -193,6 +199,12 @@ class CloudLogSync extends EventEmitter {
.find(doc => doc.accountId === settings.accountId && doc.source === hash('connection-check'))?.folderId;
if (!connection.folderId) return;
const source = hash(`${sourcePath}|${connection.ip}|${connection.folderId}`);
const backfill = this.backfillContext;
if (trackBackfill && backfill?.folderId === connection.folderId && backfill.accountId === settings.accountId) {
for (const entry of line.split(/\r?\n/).filter(Boolean)) {
backfill.counts.set(entry, (backfill.counts.get(entry) || 0) + 1);
}
}
let doc = known.filter(item => item.source === source).sort((a, b) => b.createdAt - a.createdAt)[0];
if (!doc || Buffer.byteLength(doc.content) + Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
doc = this._newDocument(connection, source, path.basename(sourcePath, path.extname(sourcePath)), '');
@@ -206,6 +218,7 @@ class CloudLogSync extends EventEmitter {
this.errorCode = this._errorCode(error);
this.notify();
this.schedule();
if (!trackBackfill) throw new CloudLogError('LOCAL_WRITE_FAILED');
});
}
@@ -221,6 +234,90 @@ class CloudLogSync extends EventEmitter {
return operation;
}
backfill(sourcePath) {
if (this.backfillJob) return this.backfillJob;
const operation = this.control.then(() => this._backfill(sourcePath));
this.backfillJob = operation.finally(() => {
this.backfillJob = null;
this.backfillContext = null;
this.notify();
});
this.control = this.backfillJob.catch(() => {});
this.notify();
return this.backfillJob;
}
async _backfill(sourcePath) {
await this.initialize();
if (this.closed || !this.loadSettings()?.enabled || !this.connection || this.retryBlocked) {
throw new CloudLogError('CONNECTION_REQUIRED');
}
const connection = this.connection;
let file;
const result = { added: 0, skipped: 0 };
this.backfillResult = null;
try {
file = await fs.open(sourcePath, 'r');
const stat = await file.stat();
if (!stat.isFile()) throw new CloudLogError('LOG_READ_FAILED');
if (stat.size > 256 * 1024 * 1024) throw new CloudLogError('LOG_TOO_LARGE');
if (stat.size === 0) {
this.backfillResult = result;
return result;
}
const counts = new Map();
for (const doc of this.documents.values()) {
if (doc.accountId !== connection.accountId || doc.folderId !== connection.folderId) continue;
for (const line of doc.content.split(/\r?\n/).filter(Boolean)) {
counts.set(line, (counts.get(line) || 0) + 1);
if (counts.size > 250000) throw new CloudLogError('LOG_TOO_LARGE');
}
}
this.backfillContext = { folderId: connection.folderId, accountId: connection.accountId, counts };
const seen = new Map();
let pending = '';
let pendingLines = 0;
const savePending = async () => {
if (!pending) return;
const content = pending;
pending = '';
pendingLines = 0;
await this.append(sourcePath, content, false);
};
try {
for await (const entry of iterateUploadLogEntries(sourcePath, {
fs: { createReadStream: () => file.createReadStream({ encoding: 'utf8', highWaterMark: 32768, end: stat.size - 1, autoClose: false }) },
includeLine: true, requireCompleteLines: true
})) {
if (this.closed) throw new CloudLogError('BACKFILL_CANCELED');
if (!entry.confirmed) continue;
const count = (seen.get(entry.line) || 0) + 1;
seen.set(entry.line, count);
if (seen.size > 250000) throw new CloudLogError('LOG_TOO_LARGE');
if (count <= (counts.get(entry.line) || 0)) {
result.skipped++;
continue;
}
counts.set(entry.line, (counts.get(entry.line) || 0) + 1);
pending += entry.line + '\n';
pendingLines++;
result.added++;
if (pendingLines >= 100 || Buffer.byteLength(pending) >= 32768) await savePending();
}
} finally {
await savePending();
}
this.backfillResult = result;
this.schedule(0);
return { ...result };
} catch (error) {
if (error instanceof CloudLogError) throw error;
throw new CloudLogError(error?.code === 'ENOENT' ? 'LOG_NOT_FOUND' : 'LOG_READ_FAILED');
} finally {
if (file) await file.close();
}
}
flushLocal() {
const operation = this.localWrites.then(async () => {
for (const id of [...this.dirty]) {
@@ -298,6 +395,7 @@ class CloudLogSync extends EventEmitter {
clearTimeout(this.timer);
this.timer = null;
this.controller?.abort();
if (this.backfillJob) await this.backfillJob.catch(() => {});
if (this.worker) await this.worker;
await this.flushLocal();
await this.diskWrites;