This commit is contained in:
1 parent
63abaa819c
commit
63296e7ad3
7 files changed
+226
-19
No files matched your search
+101
-3
@@ -3,6 +3,7 @@ const path = require('node:path');
|
||||
const { randomUUID, createHash } = require('node:crypto');
|
||||
const { EventEmitter } = require('node:events');
|
||||
const { ClouddropLogApi, CloudLogError } = require('./clouddrop-log-api');
|
||||
const { iterateUploadLogEntries } = require('./upload-log');
|
||||
|
||||
const MAX_DOCUMENT_BYTES = 700000;
|
||||
const hash = value => createHash('sha256').update(value).digest('hex');
|
||||
@@ -35,6 +36,9 @@ class CloudLogSync extends EventEmitter {
|
||||
this.closed = false;
|
||||
this.retryBlocked = false;
|
||||
this.documentTime = 0;
|
||||
this.backfillJob = null;
|
||||
this.backfillContext = null;
|
||||
this.backfillResult = null;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
@@ -71,7 +75,9 @@ class CloudLogSync extends EventEmitter {
|
||||
ip: this.connection?.ip || settings.ip || '',
|
||||
pendingFiles: pending.length,
|
||||
lastSyncedAt: Math.max(0, ...docs.map(doc => Number(doc.lastSyncedAt) || 0)),
|
||||
fileCount: docs.length
|
||||
fileCount: docs.length,
|
||||
backfillRunning: Boolean(this.backfillJob),
|
||||
backfillResult: this.backfillResult ? { ...this.backfillResult } : null
|
||||
};
|
||||
}
|
||||
|
||||
@@ -158,7 +164,7 @@ class CloudLogSync extends EventEmitter {
|
||||
this.errorCode = this._errorCode(error);
|
||||
this.retryBlocked = blocksRetry(this.errorCode);
|
||||
this.notify();
|
||||
if (!blocksRetry(this.errorCode)) {
|
||||
if (!this.closed && !blocksRetry(this.errorCode)) {
|
||||
this.timer = setTimeout(() => { this.timer = null; void this.start(); }, 30000);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
@@ -175,7 +181,7 @@ class CloudLogSync extends EventEmitter {
|
||||
};
|
||||
}
|
||||
|
||||
async append(sourcePath, line) {
|
||||
async append(sourcePath, line, trackBackfill = true) {
|
||||
await this.initialize();
|
||||
const settings = this.loadSettings() || {};
|
||||
if (!settings.enabled || !settings.accountId || typeof line !== 'string' || !line) return;
|
||||
@@ -193,6 +199,12 @@ class CloudLogSync extends EventEmitter {
|
||||
.find(doc => doc.accountId === settings.accountId && doc.source === hash('connection-check'))?.folderId;
|
||||
if (!connection.folderId) return;
|
||||
const source = hash(`${sourcePath}|${connection.ip}|${connection.folderId}`);
|
||||
const backfill = this.backfillContext;
|
||||
if (trackBackfill && backfill?.folderId === connection.folderId && backfill.accountId === settings.accountId) {
|
||||
for (const entry of line.split(/\r?\n/).filter(Boolean)) {
|
||||
backfill.counts.set(entry, (backfill.counts.get(entry) || 0) + 1);
|
||||
}
|
||||
}
|
||||
let doc = known.filter(item => item.source === source).sort((a, b) => b.createdAt - a.createdAt)[0];
|
||||
if (!doc || Buffer.byteLength(doc.content) + Buffer.byteLength(line) > MAX_DOCUMENT_BYTES) {
|
||||
doc = this._newDocument(connection, source, path.basename(sourcePath, path.extname(sourcePath)), '');
|
||||
@@ -206,6 +218,7 @@ class CloudLogSync extends EventEmitter {
|
||||
this.errorCode = this._errorCode(error);
|
||||
this.notify();
|
||||
this.schedule();
|
||||
if (!trackBackfill) throw new CloudLogError('LOCAL_WRITE_FAILED');
|
||||
});
|
||||
}
|
||||
|
||||
@@ -221,6 +234,90 @@ class CloudLogSync extends EventEmitter {
|
||||
return operation;
|
||||
}
|
||||
|
||||
backfill(sourcePath) {
|
||||
if (this.backfillJob) return this.backfillJob;
|
||||
const operation = this.control.then(() => this._backfill(sourcePath));
|
||||
this.backfillJob = operation.finally(() => {
|
||||
this.backfillJob = null;
|
||||
this.backfillContext = null;
|
||||
this.notify();
|
||||
});
|
||||
this.control = this.backfillJob.catch(() => {});
|
||||
this.notify();
|
||||
return this.backfillJob;
|
||||
}
|
||||
|
||||
async _backfill(sourcePath) {
|
||||
await this.initialize();
|
||||
if (this.closed || !this.loadSettings()?.enabled || !this.connection || this.retryBlocked) {
|
||||
throw new CloudLogError('CONNECTION_REQUIRED');
|
||||
}
|
||||
const connection = this.connection;
|
||||
let file;
|
||||
const result = { added: 0, skipped: 0 };
|
||||
this.backfillResult = null;
|
||||
try {
|
||||
file = await fs.open(sourcePath, 'r');
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile()) throw new CloudLogError('LOG_READ_FAILED');
|
||||
if (stat.size > 256 * 1024 * 1024) throw new CloudLogError('LOG_TOO_LARGE');
|
||||
if (stat.size === 0) {
|
||||
this.backfillResult = result;
|
||||
return result;
|
||||
}
|
||||
const counts = new Map();
|
||||
for (const doc of this.documents.values()) {
|
||||
if (doc.accountId !== connection.accountId || doc.folderId !== connection.folderId) continue;
|
||||
for (const line of doc.content.split(/\r?\n/).filter(Boolean)) {
|
||||
counts.set(line, (counts.get(line) || 0) + 1);
|
||||
if (counts.size > 250000) throw new CloudLogError('LOG_TOO_LARGE');
|
||||
}
|
||||
}
|
||||
this.backfillContext = { folderId: connection.folderId, accountId: connection.accountId, counts };
|
||||
const seen = new Map();
|
||||
let pending = '';
|
||||
let pendingLines = 0;
|
||||
const savePending = async () => {
|
||||
if (!pending) return;
|
||||
const content = pending;
|
||||
pending = '';
|
||||
pendingLines = 0;
|
||||
await this.append(sourcePath, content, false);
|
||||
};
|
||||
try {
|
||||
for await (const entry of iterateUploadLogEntries(sourcePath, {
|
||||
fs: { createReadStream: () => file.createReadStream({ encoding: 'utf8', highWaterMark: 32768, end: stat.size - 1, autoClose: false }) },
|
||||
includeLine: true, requireCompleteLines: true
|
||||
})) {
|
||||
if (this.closed) throw new CloudLogError('BACKFILL_CANCELED');
|
||||
if (!entry.confirmed) continue;
|
||||
const count = (seen.get(entry.line) || 0) + 1;
|
||||
seen.set(entry.line, count);
|
||||
if (seen.size > 250000) throw new CloudLogError('LOG_TOO_LARGE');
|
||||
if (count <= (counts.get(entry.line) || 0)) {
|
||||
result.skipped++;
|
||||
continue;
|
||||
}
|
||||
counts.set(entry.line, (counts.get(entry.line) || 0) + 1);
|
||||
pending += entry.line + '\n';
|
||||
pendingLines++;
|
||||
result.added++;
|
||||
if (pendingLines >= 100 || Buffer.byteLength(pending) >= 32768) await savePending();
|
||||
}
|
||||
} finally {
|
||||
await savePending();
|
||||
}
|
||||
this.backfillResult = result;
|
||||
this.schedule(0);
|
||||
return { ...result };
|
||||
} catch (error) {
|
||||
if (error instanceof CloudLogError) throw error;
|
||||
throw new CloudLogError(error?.code === 'ENOENT' ? 'LOG_NOT_FOUND' : 'LOG_READ_FAILED');
|
||||
} finally {
|
||||
if (file) await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
flushLocal() {
|
||||
const operation = this.localWrites.then(async () => {
|
||||
for (const id of [...this.dirty]) {
|
||||
@@ -298,6 +395,7 @@ class CloudLogSync extends EventEmitter {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
this.controller?.abort();
|
||||
if (this.backfillJob) await this.backfillJob.catch(() => {});
|
||||
if (this.worker) await this.worker;
|
||||
await this.flushLocal();
|
||||
await this.diskWrites;
|
||||
|
||||
Reference in new issue
Block a user