import {randomUUID} from 'node:crypto'; import {constants} from 'node:fs'; import {lstat, open, opendir, realpath} from 'node:fs/promises'; import type {BigIntStats} from 'node:fs'; import path from 'node:path'; import {performance} from 'node:perf_hooks'; import {get_project_root_path} from '../../utils/path_utils.ts'; import {exact, ForwardFailure, integer, uuid} from '../../utils/forward_protocol.ts'; export type DataFileEntry = {type: 'directory'; name: string; children: DataFileEntry[]} | {type: 'file'; name: string; id: string; size: number; modified_at: number}; export interface DataFileList { listing_id: string; expires_at: number; partial: boolean; children: DataFileEntry[]; } export interface FileData { id: string; name: string; encoding: 'base64'; offset: number; next_offset: number; total_bytes: number; eof: boolean; data: string; } export interface BotFileServiceOptions { /** Trusted server-side override of the bot data root, primarily for tests. */ root?: string; now?: () => number; } type FileRecord = {parts: string[]; stat: BigIntStats; expires_at: number; append: boolean}; const TTL = 300_000; const CHUNK = 262_144; const MAX_IDS = 8000; const MAX_ENTRIES = 2000; const MAX_BYTES = 512 * 1024; const identity = (a: BigIntStats, b: BigIntStats) => a.dev === b.dev && a.ino === b.ino && a.birthtimeNs === b.birthtimeNs; const missing = (error: unknown) => ['ENOENT', 'ENOTDIR', 'ELOOP'].includes((error as NodeJS.ErrnoException)?.code ?? ''); export class BotFileService { private readonly root: string; private readonly now: () => number; private readonly files = new Map(); private disposed = false; private listing = false; private reads = 0; constructor(bot_id: number, options: BotFileServiceOptions = {}) { if (!integer(bot_id, 1)) throw new ForwardFailure('INVALID_REQUEST'); this.root = path.resolve(options.root ?? path.join(get_project_root_path(), 'data', 'bot', String(bot_id))); this.now = options.now ?? Date.now; } dispose(): void { this.disposed = true; this.files.clear(); } private check(deadline: number): void { if (this.disposed) throw new ForwardFailure('CANCELLED'); if (!integer(deadline, 1)) throw new ForwardFailure('INVALID_REQUEST'); if (this.now() >= deadline) throw new ForwardFailure('TIMEOUT'); } private prune(): void { for (const [id, record] of this.files) { if (record.expires_at <= this.now()) this.files.delete(id); } } // Check every ancestor, including ancestors above the configured root. These // checks narrow parent replacement races; they do not make path lookup atomic. private async safe(parts: string[], deadline: number): Promise { const target = path.join(this.root, ...parts); const relative = path.relative(this.root, target); if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { throw new ForwardFailure('RESYNC_REQUIRED'); } const parsed = path.parse(target); let current = parsed.root; let stat = await lstat(current, {bigint: true}); for (const component of target.slice(parsed.root.length).split(path.sep).filter(Boolean)) { this.check(deadline); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new ForwardFailure('RESYNC_REQUIRED'); current = path.join(current, component); stat = await lstat(current, {bigint: true}); this.check(deadline); if (stat.isSymbolicLink()) throw new ForwardFailure('RESYNC_REQUIRED'); } const resolved = await realpath(target); this.check(deadline); if (resolved !== target) throw new ForwardFailure('RESYNC_REQUIRED'); return stat; } async list(deadline_at: number, expires_at?: number): Promise { this.check(deadline_at); if (expires_at !== undefined && (!integer(expires_at, 1) || expires_at <= this.now())) throw new ForwardFailure('TOKEN_EXPIRED'); if (this.listing) throw new ForwardFailure('RATE_LIMITED'); this.listing = true; const pending = new Map(); const result: DataFileList = {listing_id: randomUUID(), expires_at: Math.min(this.now() + TTL, expires_at ?? Infinity), partial: false, children: []}; let entries = 0; let bytes = Buffer.byteLength(JSON.stringify(result)); const started = performance.now(); const budget = () => { this.check(deadline_at); if (performance.now() - started >= 2000) result.partial = true; return !result.partial; }; try { this.prune(); const rootStat = await this.safe([], deadline_at); if (!rootStat.isDirectory()) throw new ForwardFailure('RESYNC_REQUIRED'); const visit = async (parts: string[], children: DataFileEntry[], depth: number): Promise => { if (!budget()) return; const before = await this.safe(parts, deadline_at); if (!before.isDirectory()) throw new ForwardFailure('RESYNC_REQUIRED'); const names: string[] = []; const directory = await opendir(path.join(this.root, ...parts)); try { while (budget()) { const entry = await directory.read(); this.check(deadline_at); if (!entry) break; if (names.length >= MAX_ENTRIES) { result.partial = true; break; } names.push(entry.name); } } finally { await directory.close(); } const after = await this.safe(parts, deadline_at); if (!identity(before, after)) throw new ForwardFailure('RESYNC_REQUIRED'); // A bounded directory read may have hit its limit; still emit its // collected entries, then stop further traversal. const stopped = result.partial; result.partial = false; for (const name of names.sort()) { if (!budget()) break; if (entries >= MAX_ENTRIES) { result.partial = true; break; } if (depth >= 32) { result.partial = true; break; } const childParts = [...parts, name]; let stat: BigIntStats; try { stat = await lstat(path.join(this.root, ...childParts), {bigint: true}); } catch { this.check(deadline_at); result.partial = true; break; } this.check(deadline_at); if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) continue; if (stat.isFile() && (stat.size > BigInt(Number.MAX_SAFE_INTEGER) || this.files.size + pending.size >= MAX_IDS)) { result.partial = true; break; } const entry: DataFileEntry = stat.isDirectory() ? {type: 'directory', name, children: []} : {type: 'file', name, id: randomUUID(), size: Number(stat.size), modified_at: Number(stat.mtimeMs)}; const cost = Buffer.byteLength(JSON.stringify(entry)) + 1; if (bytes + cost > MAX_BYTES) { result.partial = true; break; } bytes += cost; entries++; children.push(entry); if (entry.type === 'file') pending.set(entry.id, {parts: childParts, stat, expires_at: result.expires_at, append: childParts.length === 2 && childParts[0] === 'log' && name.endsWith('.log')}); else { try { await visit(childParts, entry.children, depth + 1); } catch (error) { this.check(deadline_at); if (error instanceof ForwardFailure) throw error; result.partial = true; } } if (result.partial) break; } result.partial ||= stopped; }; await visit([], result.children, 0); const finalRoot = await this.safe([], deadline_at); if (!identity(rootStat, finalRoot)) throw new ForwardFailure('RESYNC_REQUIRED'); this.check(deadline_at); for (const [id, record] of pending) this.files.set(id, record); return result; } catch (error) { this.check(deadline_at); if (missing(error) && result.children.length === 0 && pending.size === 0) { // Only a genuinely absent root is an empty listing. try { await lstat(this.root); } catch (rootError) { this.check(deadline_at); if ((rootError as NodeJS.ErrnoException).code === 'ENOENT') return result; } } throw error instanceof ForwardFailure ? error : new ForwardFailure('INTERNAL_ERROR'); } finally { this.listing = false; } } async read(payload: {id: string; offset?: number; length?: number}, deadline_at: number): Promise { this.check(deadline_at); if (!exact(payload, ['id'], ['offset', 'length']) || !uuid(payload.id) || (payload.offset !== undefined && !integer(payload.offset)) || (payload.length !== undefined && (!integer(payload.length, 1) || payload.length > CHUNK))) { throw new ForwardFailure('INVALID_REQUEST'); } this.prune(); const record = this.files.get(payload.id); if (!record) throw new ForwardFailure('INVALID_REQUEST'); const offset = payload.offset ?? 0; const size = Number(record.stat.size); if (offset > size) throw new ForwardFailure('INVALID_REQUEST'); if (this.reads >= 4) throw new ForwardFailure('RATE_LIMITED'); this.reads++; const validate = (stat: BigIntStats) => { if (!stat.isFile() || !identity(record.stat, stat) || (record.append ? stat.size < record.stat.size : stat.size !== record.stat.size || stat.mtimeNs !== record.stat.mtimeNs || stat.ctimeNs !== record.stat.ctimeNs)) throw new ForwardFailure('RESYNC_REQUIRED'); }; try { validate(await this.safe(record.parts, deadline_at)); const handle = await open(path.join(this.root, ...record.parts), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); try { this.check(deadline_at); validate(await handle.stat({bigint: true})); validate(await this.safe(record.parts, deadline_at)); const buffer = Buffer.alloc(Math.min(payload.length ?? CHUNK, size - offset)); let received = 0; while (received < buffer.length) { const {bytesRead} = await handle.read(buffer, received, buffer.length - received, offset + received); this.check(deadline_at); if (bytesRead === 0) throw new ForwardFailure('RESYNC_REQUIRED'); received += bytesRead; } validate(await handle.stat({bigint: true})); validate(await this.safe(record.parts, deadline_at)); this.check(deadline_at); if (record.expires_at <= this.now()) throw new ForwardFailure('INVALID_REQUEST'); return {id: payload.id, name: record.parts.at(-1)!, encoding: 'base64', offset, next_offset: offset + received, total_bytes: size, eof: offset + received === size, data: buffer.toString('base64')}; } finally { await handle.close(); } } catch (error) { this.check(deadline_at); throw error instanceof ForwardFailure ? error : new ForwardFailure(missing(error) ? 'RESYNC_REQUIRED' : 'INTERNAL_ERROR'); } finally { this.reads--; } } }