251 lines
12 KiB
TypeScript
251 lines
12 KiB
TypeScript
import {randomUUID} from 'node:crypto';
|
|
import {constants} from 'node:fs';
|
|
import {lstat, open, opendir, realpath} from 'node:fs/promises';
|
|
import type {BigIntStats} from 'node:fs';
|
|
import path from 'node:path';
|
|
import {performance} from 'node:perf_hooks';
|
|
import {get_project_root_path} from '../../utils/path_utils.ts';
|
|
import {exact, ForwardFailure, integer, uuid} from '../../utils/forward_protocol.ts';
|
|
|
|
export type DataFileEntry = {type: 'directory'; name: string; children: DataFileEntry[]}
|
|
| {type: 'file'; name: string; id: string; size: number; modified_at: number};
|
|
export interface DataFileList {
|
|
listing_id: string;
|
|
expires_at: number;
|
|
partial: boolean;
|
|
children: DataFileEntry[];
|
|
}
|
|
export interface FileData {
|
|
id: string;
|
|
name: string;
|
|
encoding: 'base64';
|
|
offset: number;
|
|
next_offset: number;
|
|
total_bytes: number;
|
|
eof: boolean;
|
|
data: string;
|
|
}
|
|
export interface BotFileServiceOptions {
|
|
/** Trusted server-side override of the bot data root, primarily for tests. */
|
|
root?: string;
|
|
now?: () => number;
|
|
}
|
|
type FileRecord = {parts: string[]; stat: BigIntStats; expires_at: number; append: boolean};
|
|
const TTL = 300_000;
|
|
const CHUNK = 262_144;
|
|
const MAX_IDS = 8000;
|
|
const MAX_ENTRIES = 2000;
|
|
const MAX_BYTES = 512 * 1024;
|
|
const identity = (a: BigIntStats, b: BigIntStats) => a.dev === b.dev && a.ino === b.ino
|
|
&& a.birthtimeNs === b.birthtimeNs;
|
|
const missing = (error: unknown) => ['ENOENT', 'ENOTDIR', 'ELOOP'].includes((error as NodeJS.ErrnoException)?.code ?? '');
|
|
|
|
export class BotFileService {
|
|
private readonly root: string;
|
|
private readonly now: () => number;
|
|
private readonly files = new Map<string, FileRecord>();
|
|
private disposed = false;
|
|
private listing = false;
|
|
private reads = 0;
|
|
|
|
constructor(bot_id: number, options: BotFileServiceOptions = {}) {
|
|
if (!integer(bot_id, 1)) throw new ForwardFailure('INVALID_REQUEST');
|
|
this.root = path.resolve(options.root ?? path.join(get_project_root_path(), 'data', 'bot', String(bot_id)));
|
|
this.now = options.now ?? Date.now;
|
|
}
|
|
|
|
dispose(): void {
|
|
this.disposed = true;
|
|
this.files.clear();
|
|
}
|
|
|
|
private check(deadline: number): void {
|
|
if (this.disposed) throw new ForwardFailure('CANCELLED');
|
|
if (!integer(deadline, 1)) throw new ForwardFailure('INVALID_REQUEST');
|
|
if (this.now() >= deadline) throw new ForwardFailure('TIMEOUT');
|
|
}
|
|
|
|
private prune(): void {
|
|
for (const [id, record] of this.files) {
|
|
if (record.expires_at <= this.now()) this.files.delete(id);
|
|
}
|
|
}
|
|
|
|
// Check every ancestor, including ancestors above the configured root. These
|
|
// checks narrow parent replacement races; they do not make path lookup atomic.
|
|
private async safe(parts: string[], deadline: number): Promise<BigIntStats> {
|
|
const target = path.join(this.root, ...parts);
|
|
const relative = path.relative(this.root, target);
|
|
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
throw new ForwardFailure('RESYNC_REQUIRED');
|
|
}
|
|
const parsed = path.parse(target);
|
|
let current = parsed.root;
|
|
let stat = await lstat(current, {bigint: true});
|
|
for (const component of target.slice(parsed.root.length).split(path.sep).filter(Boolean)) {
|
|
this.check(deadline);
|
|
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
current = path.join(current, component);
|
|
stat = await lstat(current, {bigint: true});
|
|
this.check(deadline);
|
|
if (stat.isSymbolicLink()) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
}
|
|
const resolved = await realpath(target);
|
|
this.check(deadline);
|
|
if (resolved !== target) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
return stat;
|
|
}
|
|
|
|
async list(deadline_at: number, expires_at?: number): Promise<DataFileList> {
|
|
this.check(deadline_at);
|
|
if (expires_at !== undefined && (!integer(expires_at, 1) || expires_at <= this.now()))
|
|
throw new ForwardFailure('TOKEN_EXPIRED');
|
|
if (this.listing) throw new ForwardFailure('RATE_LIMITED');
|
|
this.listing = true;
|
|
const pending = new Map<string, FileRecord>();
|
|
const result: DataFileList = {listing_id: randomUUID(), expires_at: Math.min(this.now() + TTL, expires_at ?? Infinity),
|
|
partial: false, children: []};
|
|
let entries = 0;
|
|
let bytes = Buffer.byteLength(JSON.stringify(result));
|
|
const started = performance.now();
|
|
const budget = () => {
|
|
this.check(deadline_at);
|
|
if (performance.now() - started >= 2000) result.partial = true;
|
|
return !result.partial;
|
|
};
|
|
try {
|
|
this.prune();
|
|
const rootStat = await this.safe([], deadline_at);
|
|
if (!rootStat.isDirectory()) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
const visit = async (parts: string[], children: DataFileEntry[], depth: number): Promise<void> => {
|
|
if (!budget()) return;
|
|
const before = await this.safe(parts, deadline_at);
|
|
if (!before.isDirectory()) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
const names: string[] = [];
|
|
const directory = await opendir(path.join(this.root, ...parts));
|
|
try {
|
|
while (budget()) {
|
|
const entry = await directory.read();
|
|
this.check(deadline_at);
|
|
if (!entry) break;
|
|
if (names.length >= MAX_ENTRIES) { result.partial = true; break; }
|
|
names.push(entry.name);
|
|
}
|
|
} finally { await directory.close(); }
|
|
const after = await this.safe(parts, deadline_at);
|
|
if (!identity(before, after)) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
// A bounded directory read may have hit its limit; still emit its
|
|
// collected entries, then stop further traversal.
|
|
const stopped = result.partial;
|
|
result.partial = false;
|
|
for (const name of names.sort()) {
|
|
if (!budget()) break;
|
|
if (entries >= MAX_ENTRIES) { result.partial = true; break; }
|
|
if (depth >= 32) { result.partial = true; break; }
|
|
const childParts = [...parts, name];
|
|
let stat: BigIntStats;
|
|
try { stat = await lstat(path.join(this.root, ...childParts), {bigint: true}); }
|
|
catch {
|
|
this.check(deadline_at);
|
|
result.partial = true;
|
|
break;
|
|
}
|
|
this.check(deadline_at);
|
|
if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) continue;
|
|
if (stat.isFile() && (stat.size > BigInt(Number.MAX_SAFE_INTEGER)
|
|
|| this.files.size + pending.size >= MAX_IDS)) { result.partial = true; break; }
|
|
const entry: DataFileEntry = stat.isDirectory()
|
|
? {type: 'directory', name, children: []}
|
|
: {type: 'file', name, id: randomUUID(), size: Number(stat.size), modified_at: Number(stat.mtimeMs)};
|
|
const cost = Buffer.byteLength(JSON.stringify(entry)) + 1;
|
|
if (bytes + cost > MAX_BYTES) { result.partial = true; break; }
|
|
bytes += cost;
|
|
entries++;
|
|
children.push(entry);
|
|
if (entry.type === 'file') pending.set(entry.id, {parts: childParts, stat,
|
|
expires_at: result.expires_at, append: childParts.length === 2
|
|
&& childParts[0] === 'log' && name.endsWith('.log')});
|
|
else {
|
|
try { await visit(childParts, entry.children, depth + 1); }
|
|
catch (error) {
|
|
this.check(deadline_at);
|
|
if (error instanceof ForwardFailure) throw error;
|
|
result.partial = true;
|
|
}
|
|
}
|
|
if (result.partial) break;
|
|
}
|
|
result.partial ||= stopped;
|
|
};
|
|
await visit([], result.children, 0);
|
|
const finalRoot = await this.safe([], deadline_at);
|
|
if (!identity(rootStat, finalRoot)) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
this.check(deadline_at);
|
|
for (const [id, record] of pending) this.files.set(id, record);
|
|
return result;
|
|
} catch (error) {
|
|
this.check(deadline_at);
|
|
if (missing(error) && result.children.length === 0 && pending.size === 0) {
|
|
// Only a genuinely absent root is an empty listing.
|
|
try { await lstat(this.root); }
|
|
catch (rootError) {
|
|
this.check(deadline_at);
|
|
if ((rootError as NodeJS.ErrnoException).code === 'ENOENT') return result;
|
|
}
|
|
}
|
|
throw error instanceof ForwardFailure ? error : new ForwardFailure('INTERNAL_ERROR');
|
|
} finally { this.listing = false; }
|
|
}
|
|
|
|
async read(payload: {id: string; offset?: number; length?: number}, deadline_at: number): Promise<FileData> {
|
|
this.check(deadline_at);
|
|
if (!exact(payload, ['id'], ['offset', 'length']) || !uuid(payload.id)
|
|
|| (payload.offset !== undefined && !integer(payload.offset))
|
|
|| (payload.length !== undefined && (!integer(payload.length, 1) || payload.length > CHUNK))) {
|
|
throw new ForwardFailure('INVALID_REQUEST');
|
|
}
|
|
this.prune();
|
|
const record = this.files.get(payload.id);
|
|
if (!record) throw new ForwardFailure('INVALID_REQUEST');
|
|
const offset = payload.offset ?? 0;
|
|
const size = Number(record.stat.size);
|
|
if (offset > size) throw new ForwardFailure('INVALID_REQUEST');
|
|
if (this.reads >= 4) throw new ForwardFailure('RATE_LIMITED');
|
|
this.reads++;
|
|
const validate = (stat: BigIntStats) => {
|
|
if (!stat.isFile() || !identity(record.stat, stat) || (record.append
|
|
? stat.size < record.stat.size
|
|
: stat.size !== record.stat.size || stat.mtimeNs !== record.stat.mtimeNs
|
|
|| stat.ctimeNs !== record.stat.ctimeNs)) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
};
|
|
try {
|
|
validate(await this.safe(record.parts, deadline_at));
|
|
const handle = await open(path.join(this.root, ...record.parts), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
try {
|
|
this.check(deadline_at);
|
|
validate(await handle.stat({bigint: true}));
|
|
validate(await this.safe(record.parts, deadline_at));
|
|
const buffer = Buffer.alloc(Math.min(payload.length ?? CHUNK, size - offset));
|
|
let received = 0;
|
|
while (received < buffer.length) {
|
|
const {bytesRead} = await handle.read(buffer, received, buffer.length - received, offset + received);
|
|
this.check(deadline_at);
|
|
if (bytesRead === 0) throw new ForwardFailure('RESYNC_REQUIRED');
|
|
received += bytesRead;
|
|
}
|
|
validate(await handle.stat({bigint: true}));
|
|
validate(await this.safe(record.parts, deadline_at));
|
|
this.check(deadline_at);
|
|
if (record.expires_at <= this.now()) throw new ForwardFailure('INVALID_REQUEST');
|
|
return {id: payload.id, name: record.parts.at(-1)!, encoding: 'base64', offset,
|
|
next_offset: offset + received, total_bytes: size, eof: offset + received === size,
|
|
data: buffer.toString('base64')};
|
|
} finally { await handle.close(); }
|
|
} catch (error) {
|
|
this.check(deadline_at);
|
|
throw error instanceof ForwardFailure ? error
|
|
: new ForwardFailure(missing(error) ? 'RESYNC_REQUIRED' : 'INTERNAL_ERROR');
|
|
} finally { this.reads--; }
|
|
}
|
|
}
|