forward_server/tests/relay.integration.test.ts

import assert from 'node:assert/strict';
import {randomUUID} from 'node:crypto';
import {createServer} from 'node:http';
import {once} from 'node:events';
import {setTimeout as delay} from 'node:timers/promises';
import test from 'node:test';
import {Server} from 'socket.io';
import {io, type Socket as Client} from 'socket.io-client';
import {create_session_store} from '../runtime/session_store.ts';
import {install_relay_routes} from '../runtime/relay_routes.ts';
import {DEFAULT_LIMITS} from '../utils/relay_config.ts';
import {encode_token} from '../utils/token_utils.ts';
import type {BotLogRecord, ForwardAck, RelayClaims, RelayLimits, RelayMethod, RelayRequest, SessionScope} from '../types.ts';

const key = 'relay-integration-test-secret-only';
const permissions: RelayMethod[] = ['bot.snapshot', 'bot.logs', 'bot.subscribe', 'bot.unsubscribe', 'bot.chat', 'execute_function', 'bot.files'];
const scope = (): SessionScope => ({forward_id: 1, forward_session_id: randomUUID(), node_session_id: randomUUID(), node_id: 2, bot_id: 3});
function token(s: SessionScope, role: 'node' | 'web', grant_id?: string, expires_at = Date.now() + 10_000, user_id = 4,
    granted = permissions): string {
    const now = Date.now();
    const claims: RelayClaims = {...s, role, v: 1, permissions: granted, issued_at: now, join_before: Math.min(now + 1000, expires_at),
        expires_at, jti: randomUUID(), ...(role === 'web' ? {grant_id, user_id} : {})};
    return encode_token(key, claims);
}
const request = (method: RelayRequest['method'], payload: unknown = {}, request_id = randomUUID()) => ({method, payload, request_id});
function received(socket: Client, event: string): Promise<any[]> {return new Promise(resolve => socket.once(event, (...args) => resolve(args)));}
async function ack(socket: Client, name: string, value: unknown): Promise<ForwardAck<any>> {
    return socket.timeout(1500).emitWithAck(name, value);
}
function code(result: ForwardAck<unknown>, expected: string): void {
    assert.equal(result.status, 'error');
    if (result.status === 'error') assert.equal(result.data.code, expected);
}
async function fixture(overrides: Partial<RelayLimits> = {}) {
    const http = createServer();
    const server = new Server(http);
    const namespace = server.of('/relay');
    const statuses: unknown[] = [];
    const store = create_session_store({forward_id: 1, key, limits: {...DEFAULT_LIMITS, rpc_timeout_ms: 120, ...overrides},
        on_status: value => statuses.push(value)});
    const dispose = install_relay_routes(namespace, store);
    http.listen(0, '127.0.0.1');
    await once(http, 'listening');
    const port = (http.address() as {port: number}).port;
    const clients: Client[] = [];
    const s = scope();
    store.open({...s, lease_expires_at: Date.now() + 20_000});
    async function connect() {
        const socket = io(`http://127.0.0.1:${port}/relay`, {transports: ['websocket'], reconnection: false, forceNew: true});
        clients.push(socket);
        await received(socket, 'connect');
        return socket;
    }
    async function node(granted = permissions, ready = true) {
        const socket = await connect();
        assert.equal((await ack(socket, 'join_room', {token: token(s, 'node', undefined, Date.now() + 10_000, 4, granted)})).status, 'success');
        if (ready) assert.equal((await ack(socket, 'forward_node_ready', {forward_session_id: s.forward_session_id})).status, 'success');
        return socket;
    }
    async function web(grant_id = randomUUID(), granted = permissions, expires_at = Date.now() + 10_000) {
        store.authorize({forward_session_id: s.forward_session_id, grant_id, user_id: 4, permissions: granted, expires_at});
        const socket = await connect();
        assert.equal((await ack(socket, 'join_room', {token: token(s, 'web', grant_id, expires_at, 4, granted)})).status, 'success');
        return {socket, grant_id};
    }
    async function close() {
        store.close(); dispose();
        for (const client of clients) client.close();
        await server.close();
    }
    return {store, namespace, connect, node, web, s, close, statuses, dispose, http};
}
function snapshot(bot_id: number, seq = 0) {
    return {status: 'success', data: {bot_id, bot_status: 'offline', context_exists: false,
        context_generation: null as string | null, stream_id: randomUUID(), seq, captured_at: Date.now()}};
}

test('file RPCs require bot.files, match requested ranges and redispatch without caching ACK bodies', async () => {
    const f = await fixture({requests_per_second: 100});
    try {
        const node = await f.node(); const web = await f.web(randomUUID(), ['bot.files']);
        const limited = await f.web(randomUUID(), ['bot.snapshot']);
        const file_id = randomUUID();
        const listing = request('get_data_file_list');
        const read = request('get_file_data', {id: file_id});
        let calls = 0;
        let response: unknown = {status: 'success', data: {listing_id: randomUUID(), expires_at: Date.now() + 1000,
            partial: false, children: []}};
        let held: ((value: unknown) => void) | undefined;
        let hold = false;
        node.on('relay_request', (_value, reply) => {calls++; if (hold) held = reply; else reply(response);});
        for (const command of [listing, read]) code(await ack(limited.socket, 'relay_request', command), 'FORBIDDEN');
        code(await ack(node, 'relay_request', listing), 'FORBIDDEN');
        code(await ack(web.socket, 'relay_request', request('bot.files')), 'INVALID_REQUEST');
        assert.deepEqual(await ack(web.socket, 'relay_request', listing), response);
        assert.deepEqual(await ack(web.socket, 'relay_request', listing), response);
        assert.equal(calls, 2);
        const sessions = (f.store as any).sessions as Map<string, any>;
        const entries = sessions.get(f.s.forward_session_id).grants.get(web.grant_id).dedupe as Map<string, any>;
        assert.equal(entries.get(listing.request_id).result, undefined);
        assert.equal(entries.get(listing.request_id).completed, true);
        response = {status: 'success', data: {id: file_id, name: 'data.bin', encoding: 'base64',
            offset: 0, next_offset: 1, total_bytes: 1, eof: true, data: 'YQ=='}};
        assert.deepEqual(await ack(web.socket, 'relay_request', read), response);
        assert.deepEqual(await ack(web.socket, 'relay_request', read), response);
        assert.equal(entries.get(read.request_id).result, undefined);
        code(await ack(web.socket, 'relay_request', {...read, payload: {id: randomUUID()}}), 'INVALID_REQUEST');
        code(await ack(web.socket, 'relay_request', {...listing, method: 'get_file_data', payload: {id: file_id}}), 'INVALID_REQUEST');
        for (const changes of [{id: randomUUID()}, {offset: 1, next_offset: 2, total_bytes: 2},
            {data: 'YWI=', next_offset: 2, total_bytes: 2}]) {
            const original = response as {status: string; data: Record<string, unknown>};
            response = {...original, data: {...original.data, ...changes}};
            const command = request('get_file_data', {id: file_id, length: 1});
            code(await ack(web.socket, 'relay_request', command), 'INVALID_REQUEST');
            assert.equal(entries.get(command.request_id).result, undefined);
            response = original;
        }
        response = {status: 'success', data: {id: file_id, name: 'data.bin', encoding: 'base64',
            offset: 7, next_offset: 7, total_bytes: 7, eof: true, data: ''}};
        assert.deepEqual(await ack(web.socket, 'relay_request', request('get_file_data', {id: file_id, offset: 7, length: 1})), response);
        hold = true;
        const pending = ack(web.socket, 'relay_request', read);
        while (!held) await delay(1);
        code(await ack(web.socket, 'relay_request', read), 'RATE_LIMITED');
        held({status: 'error', data: {code: 'NO_CONTEXT', message: 'No context', retryable: false}});
        code(await pending, 'NO_CONTEXT');
        assert.equal(entries.get(read.request_id).result, undefined);
        hold = false;
        response = {status: 'error', data: {code: 'BOT_NOT_ONLINE', message: 'Offline', retryable: false}};
        const before = calls;
        code(await ack(web.socket, 'relay_request', read), 'BOT_NOT_ONLINE');
        assert.equal(calls, before + 1);
        f.store.set_control_available(false);
        code(await ack(web.socket, 'relay_request', listing), 'BOT_NOT_ONLINE');
        assert.equal(calls, before + 2);
    } finally {await f.close();}
});

test('file RPC redispatch rechecks Node permission and readiness', async () => {
    const f = await fixture();
    try {
        const node = await f.node(['bot.snapshot']);
        const web = await f.web(randomUUID(), ['bot.files']);
        const command = request('get_data_file_list');
        code(await ack(web.socket, 'relay_request', command), 'FORBIDDEN');
        const replacement = await f.node(['bot.files'], false);
        code(await ack(web.socket, 'relay_request', command), 'NODE_OFFLINE');
        await ack(replacement, 'forward_node_ready', {forward_session_id: f.s.forward_session_id});
        replacement.on('relay_request', (_value, reply) => reply({status: 'success', data: {
            listing_id: randomUUID(), expires_at: 1, partial: true, children: []}}));
        assert.equal((await ack(web.socket, 'relay_request', command)).status, 'success');
        await f.node(['bot.snapshot']);
        code(await ack(web.socket, 'relay_request', command), 'FORBIDDEN');
        node.close();
    } finally {await f.close();}
});

test('RPC responses stay private, request IDs are grant-scoped, dedupe conflicts and chat replay are rejected', async () => {
    const f = await fixture();
    try {
        const node = await f.node();
        let calls = 0;
        node.on('relay_request', (value, reply) => {calls++; reply(value.method === 'bot.chat' ? {status: 'success', data: {accepted: true}} : snapshot(f.s.bot_id, calls));});
        const a = await f.web(); const b = await f.web();
        let leaked = 0;
        b.socket.on('relay_request', () => leaked++);
        const id = randomUUID();
        const [ra, rb] = await Promise.all([ack(a.socket, 'relay_request', request('bot.snapshot', {}, id)), ack(b.socket, 'relay_request', request('bot.snapshot', {}, id))]);
        assert.equal(ra.status, 'success'); assert.equal(rb.status, 'success');
        assert.equal(calls, 2); assert.equal(leaked, 0);
        assert.deepEqual(await ack(a.socket, 'relay_request', request('bot.snapshot', {}, id)), ra);
        assert.equal(calls, 2);
        code(await ack(a.socket, 'relay_request', request('bot.logs', {limit: 1}, id)), 'INVALID_REQUEST');
        const chat = request('bot.chat', {message: 'hello'});
        assert.equal((await ack(a.socket, 'relay_request', chat)).status, 'success');
        code(await ack(a.socket, 'relay_request', chat), 'CANCELLED');
        assert.equal(calls, 3);
    } finally {await f.close();}
});

test('execute_function forwards exact payloads, validates ACKs and rejects pending and completed replay', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const web = await f.web(randomUUID(), ['execute_function']);
        const payload = {function: 'movement/navigation', name: 'go_to', args: [{x: 1, y: 2}, true, null]};
        const command = request('execute_function', payload);
        let calls = 0;
        let reply!: (value: unknown) => void;
        const dispatched = new Promise<void>(resolve => node.on('relay_request', (value, callback) => {
            calls++;
            assert.equal(value.method, command.method);
            assert.deepEqual(value.payload, payload);
            assert.equal(value.forward_session_id, f.s.forward_session_id);
            reply = callback; resolve();
        }));
        const pending = ack(web.socket, 'relay_request', command);
        await dispatched;
        code(await ack(web.socket, 'relay_request', command), 'RATE_LIMITED');
        reply({status: 'success', data: true});
        assert.deepEqual(await pending, {status: 'success', data: true});
        code(await ack(web.socket, 'relay_request', command), 'CANCELLED');
        code(await ack(web.socket, 'relay_request', {...command, payload: {...payload, args: []}}), 'INVALID_REQUEST');
        assert.equal(calls, 1);
        node.removeAllListeners('relay_request');
        let response: unknown = {status: 'success', data: false};
        node.on('relay_request', (_value, callback) => {calls++; callback(response);});
        assert.deepEqual(await ack(web.socket, 'relay_request', request('execute_function', payload)), response);
        response = {status: 'success', data: {accepted: true}};
        const invalid_ack = request('execute_function', payload);
        code(await ack(web.socket, 'relay_request', invalid_ack), 'INVALID_REQUEST');
        code(await ack(web.socket, 'relay_request', invalid_ack), 'CANCELLED');
        f.store.set_control_available(false);
        code(await ack(web.socket, 'relay_request', request('execute_function', payload)), 'RELAY_UNAVAILABLE');
        assert.equal(calls, 3);
    } finally {await f.close();}
});

test('execute_function requires permission in Web token, grant and Node token', async () => {
    const f = await fixture();
    try {
        const node = await f.node(['bot.chat']);
        let calls = 0; node.on('relay_request', () => calls++);
        const grant_id = randomUUID();
        f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id, user_id: 4,
            permissions, expires_at: Date.now() + 10_000});
        const limited = await f.connect();
        assert.equal((await ack(limited, 'join_room', {token: token(f.s, 'web', grant_id,
            Date.now() + 10_000, 4, ['bot.chat'])})).status, 'success');
        const command = request('execute_function', {function: 'movement', name: 'stop', args: []});
        code(await ack(limited, 'relay_request', command), 'FORBIDDEN');
        const web = await f.web(randomUUID(), ['execute_function']);
        code(await ack(web.socket, 'relay_request', command), 'FORBIDDEN');
        const restricted_grant = randomUUID();
        f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: restricted_grant, user_id: 4,
            permissions: ['bot.chat'], expires_at: Date.now() + 10_000});
        const excessive = await f.connect();
        code(await ack(excessive, 'join_room', {token: token(f.s, 'web', restricted_grant,
            Date.now() + 10_000, 4, ['execute_function'])}), 'FORBIDDEN');
        assert.equal(calls, 0);
    } finally {await f.close();}
});

test('Node replacement cancels pending, ignores late ACK, resets readiness, and old disconnect cannot clear new Node', async () => {
    const f = await fixture();
    try {
        const old = await f.node(); const web = await f.web();
        let late: (value: unknown) => void;
        const received = new Promise<void>(resolve => old.on('relay_request', (_value, reply) => {late = reply; resolve();}));
        const response = ack(web.socket, 'relay_request', request('bot.snapshot'));
        await received;
        const replacement = await f.connect();
        assert.equal((await ack(replacement, 'join_room', {token: token(f.s, 'node')})).status, 'success');
        code(await response, 'NODE_OFFLINE');
        late!(snapshot(f.s.bot_id, 999));
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'NODE_OFFLINE');
        replacement.on('relay_request', (_value, reply) => reply(snapshot(f.s.bot_id, 2)));
        await ack(replacement, 'forward_node_ready', {forward_session_id: f.s.forward_session_id});
        old.disconnect();
        assert.equal((await ack(web.socket, 'relay_request', request('bot.snapshot'))).data.seq, 2);
    } finally {await f.close();}
});

test('tab_complete dispatch rejects ranges beyond request text and accepts exact boundaries', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const web = await f.web();
        let range = {start: 0, length: 0};
        node.on('relay_request', (_value, reply) => reply({status: 'success', data: {matches: ['/help'], ...range}}));
        for (const [text, start, length] of [['', 0, 0], ['/h', 0, 2], ['/h', 2, 0], ['界'.repeat(256), 128, 128]] as const) {
            range = {start, length};
            assert.deepEqual(await ack(web.socket, 'relay_request', {method: 'tab_complete', payload: {text}, request_id: randomUUID()}),
                {status: 'success', data: {matches: ['/help'], start, length}});
        }
        for (const [start, length] of [[2, 0], [1, 1], [0, 2]]) {
            range = {start, length};
            code(await ack(web.socket, 'relay_request', {method: 'tab_complete', payload: {text: '/'}, request_id: randomUUID()}), 'INVALID_REQUEST');
        }
    } finally {await f.close();}
});

test('status lifecycle events fan out to snapshot readers with scope, permission and sequence checks', async () => {
    const f = await fixture();
    try {
        const node = await f.node();
        const web = await f.web(randomUUID(), ['bot.snapshot']);
        const log_reader = await f.web(randomUUID(), ['bot.logs']);
        let log_reader_events = 0;
        log_reader.socket.on('bot_event', () => log_reader_events++);
        const stream = randomUUID();
        const context_generation = randomUUID();
        const event = (seq: number, status: string, context_exists: boolean) => ({
            forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: context_exists ? context_generation : null,
            stream_id: stream, seq, emitted_at: Date.now(), name: 'status_update',
            payload: {status, context_exists},
        });
        for (const [seq, status, context_exists] of [[1, 'online', true], [2, 'disconnecting', true],
            [3, 'offline', true], [4, 'offline', false]] as const) {
            const expected = event(seq, status, context_exists);
            const delivered = received(web.socket, 'bot_event');
            assert.deepEqual(await ack(node, 'bot_event', expected), {status: 'success', data: {accepted: true}});
            assert.deepEqual((await delivered)[0], expected);
        }
        assert.equal(log_reader_events, 0);

        code(await ack(web.socket, 'bot_event', event(5, 'offline', false)), 'FORBIDDEN');
        for (const [name, payload] of [['bot_error', {message: 'unexpected'}], ['message', {text: 'hello', position: 'chat'}],
            ['log', {records: []}], ['arbitrary', {}], ['status_update', {status: 'invalid', context_exists: false}],
            ['status_update', {status: 'offline', context_exists: false, extra: true}]] as const) {
            code(await ack(node, 'bot_event', {...event(5, 'offline', false), name, payload}), 'INVALID_REQUEST');
        }
        code(await ack(node, 'bot_event', event(5, 'online', false)), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', {...event(5, 'offline', true), context_generation: null}), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', {...event(5, 'offline', false), seq: -1}), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', {...event(5, 'offline', false), extra: true}), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', event(4, 'offline', false)), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', {...event(5, 'offline', false), bot_id: f.s.bot_id + 1}), 'FORBIDDEN');
        code(await ack(node, 'bot_event', {...event(5, 'offline', false), forward_session_id: randomUUID()}), 'FORBIDDEN');
        const next = event(5, 'offline', false);
        const delivered = received(web.socket, 'bot_event');
        node.emit('bot_event', next);
        assert.deepEqual((await delivered)[0], next);

        const log = {...next, seq: 6, name: 'log', payload: {records: [] as never[]}};
        let limited_logs = 0; web.socket.on('bot_log', () => limited_logs++);
        const log_delivered = received(log_reader.socket, 'bot_log');
        assert.equal((await ack(node, 'bot_log', log)).status, 'success');
        assert.deepEqual((await log_delivered)[0], log);
        code(await ack(node, 'bot_log', {...log, seq: 5}), 'INVALID_REQUEST');
        code(await ack(web.socket, 'bot_log', {...log, seq: 6}), 'FORBIDDEN');
        code(await ack(node, 'bot_log', {...log, seq: 6, bot_id: f.s.bot_id + 1}), 'FORBIDDEN');
        code(await ack(node, 'bot_log', {...next, seq: 6}), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', event(6, 'offline', false)), 'INVALID_REQUEST');
        code(await ack(node, 'bot_event', event(5, 'offline', false)), 'INVALID_REQUEST');
        const update = {forward_session_id: next.forward_session_id, bot_id: next.bot_id,
            stream_id: stream, context_generation, emitted_at: next.emitted_at, seq: 7,
            data: {name: 'health', value: 20}};
        assert.equal((await ack(node, 'info_update', update)).status, 'success');
        code(await ack(node, 'bot_event', event(7, 'offline', false)), 'INVALID_REQUEST');
        const replacement = {...event(0, 'offline', false), stream_id: randomUUID()};
        assert.equal((await ack(node, 'bot_event', replacement)).status, 'success');
        code(await ack(node, 'bot_event', event(8, 'offline', false)), 'RESYNC_REQUIRED');
        assert.equal(limited_logs, 0);
        assert.equal(log_reader_events, 0);

        const restricted = await f.node(['bot.logs']);
        code(await ack(restricted, 'bot_event', event(8, 'offline', false)), 'FORBIDDEN');
        const not_ready = await f.node(['bot.snapshot'], false);
        code(await ack(not_ready, 'bot_event', event(8, 'offline', false)), 'NODE_OFFLINE');
    } finally {await f.close();}
});

test('get_context_info maps to snapshot permissions and validates Node ACKs', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const web = await f.web(randomUUID(), ['bot.snapshot']);
        const limited = await f.web(randomUUID(), ['bot.logs']);
        let calls = 0;
        let response: unknown = {status: 'success', data: [{name: 'health',
            i18n: {lang: 'en', name: 'Health', description: 'Current HP'}, value: {current: 20}}]};
        node.on('relay_request', (value, reply) => {
            calls++;
            assert.equal(value.method, 'get_context_info');
            assert.deepEqual(value.payload, {});
            assert.equal(value.forward_session_id, f.s.forward_session_id);
            reply(response);
        });
        assert.deepEqual(await ack(web.socket, 'relay_request', request('get_context_info')), response);
        code(await ack(limited.socket, 'relay_request', request('get_context_info')), 'FORBIDDEN');
        code(await ack(node, 'relay_request', request('get_context_info')), 'FORBIDDEN');
        code(await ack(web.socket, 'relay_request', request('get_context_info', {name: 'health'})), 'INVALID_REQUEST');
        assert.equal(calls, 1);
        response = {status: 'success', data: [{name: 'health', value: 20}]};
        code(await ack(web.socket, 'relay_request', request('get_context_info')), 'INVALID_REQUEST');
        await f.node(['bot.logs']);
        code(await ack(web.socket, 'relay_request', request('get_context_info')), 'FORBIDDEN');
    } finally {await f.close();}
});

test('completed get_context_info IDs recheck Node online state, readiness and permissions', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const web = await f.web();
        let calls = 0;
        node.on('relay_request', (_value, reply) => {
            calls++;
            reply(calls === 1 ? {status: 'success', data: []}
                : {status: 'error', data: {code: 'BOT_NOT_ONLINE', message: 'Offline', retryable: false}});
        });
        const read = request('get_context_info');
        assert.deepEqual(await ack(web.socket, 'relay_request', read), {status: 'success', data: []});
        code(await ack(web.socket, 'relay_request', read), 'BOT_NOT_ONLINE');
        assert.equal(calls, 2);
        const replacement = await f.node(permissions, false);
        code(await ack(web.socket, 'relay_request', read), 'NODE_OFFLINE');
        replacement.on('relay_request', (_value, reply) => reply({status: 'success', data: [{name: 'health',
            i18n: {lang: '', name: '', description: ''}, value: 10}]}));
        await ack(replacement, 'forward_node_ready', {forward_session_id: f.s.forward_session_id});
        assert.equal((await ack(web.socket, 'relay_request', read)).data[0].value, 10);
        code(await ack(web.socket, 'relay_request', request('bot.snapshot', {}, read.request_id)), 'INVALID_REQUEST');
        const restricted = await f.node(['bot.logs']);
        code(await ack(web.socket, 'relay_request', read), 'FORBIDDEN');
        const server_node = f.namespace.sockets.get(restricted.id!)!;
        const disconnected = once(server_node, 'disconnect'); restricted.disconnect(); await disconnected;
        code(await ack(web.socket, 'relay_request', read), 'NODE_OFFLINE');
    } finally {await f.close();}
});

test('get_context_info duplicate pending and reused payload checks survive fresh dispatch', async () => {
    const f = await fixture({max_dedupe_entries: 1});
    try {
        const node = await f.node(); const web = await f.web();
        let calls = 0;
        let reply!: (value: unknown) => void;
        const incoming = received(node, 'relay_request');
        node.on('relay_request', (_value, callback) => {calls++; reply = callback;});
        const read = request('get_context_info');
        const first = ack(web.socket, 'relay_request', read);
        await incoming;
        code(await ack(web.socket, 'relay_request', read), 'RATE_LIMITED');
        code(await ack(web.socket, 'relay_request', request('bot.snapshot', {}, read.request_id)), 'INVALID_REQUEST');
        reply({status: 'success', data: []}); await first;
        const repeated = received(node, 'relay_request');
        const second = ack(web.socket, 'relay_request', read); await repeated;
        code(await ack(web.socket, 'relay_request', read), 'RATE_LIMITED');
        reply({status: 'success', data: []}); await second;
        assert.equal(calls, 2);
        const cached = request('bot.snapshot');
        node.removeAllListeners('relay_request');
        node.on('relay_request', (_value, callback) => callback(snapshot(f.s.bot_id)));
        await ack(web.socket, 'relay_request', cached);
        code(await ack(web.socket, 'relay_request', request('get_context_info', {}, cached.request_id)), 'INVALID_REQUEST');
    } finally {await f.close();}
});

test('info_update auto-pushes exact envelopes and shares log stream and sequence state', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const web = await f.web(randomUUID(), ['bot.snapshot']);
        const logs = await f.web(randomUUID(), ['bot.logs']);
        let info_leaks = 0; let log_leaks = 0;
        logs.socket.on('info_update', () => info_leaks++);
        web.socket.on('bot_log', () => log_leaks++);
        const update = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: randomUUID(), stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
            data: {name: 'health', value: {current: 20}}};
        const log = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id, context_generation: null as string | null,
            stream_id: update.stream_id, seq: 2, emitted_at: Date.now(), name: 'log', payload: {records: [] as BotLogRecord[]}};
        const first = received(web.socket, 'info_update');
        assert.equal((await ack(node, 'info_update', update)).status, 'success');
        assert.deepEqual((await first)[0], update);
        const next = received(logs.socket, 'bot_log');
        assert.equal((await ack(node, 'bot_log', log)).status, 'success');
        assert.deepEqual((await next)[0], log);
        code(await ack(node, 'info_update', {...update, seq: 2}), 'INVALID_REQUEST');
        const third = received(web.socket, 'info_update');
        node.emit('info_update', {...update, seq: 3});
        assert.equal((await third)[0].seq, 3);
        code(await ack(node, 'bot_log', {...log, seq: 3}), 'INVALID_REQUEST');
        code(await ack(web.socket, 'info_update', {...update, seq: 4}), 'FORBIDDEN');
        for (const invalid of [{...update, seq: 4, context_generation: null as string | null}, {...update, seq: 4, name: 'info_update'},
            {...update, seq: 4, payload: update.data}]) {
            code(await ack(node, 'info_update', invalid), 'INVALID_REQUEST');
        }
        for (const invalid of [{...update, seq: 4, bot_id: f.s.bot_id + 1},
            {...update, seq: 4, forward_session_id: randomUUID()}]) {
            code(await ack(node, 'info_update', invalid), 'FORBIDDEN');
        }
        const replacement = {...update, stream_id: randomUUID(), seq: 0};
        assert.equal((await ack(node, 'info_update', replacement)).status, 'success');
        code(await ack(node, 'bot_log', {...log, seq: 4}), 'RESYNC_REQUIRED');
        assert.equal(info_leaks, 0); assert.equal(log_leaks, 0);
    } finally {await f.close();}
});

test('lifecycle packets use reliable bounded buffering and share the Node event rate', async () => {
    const f = await fixture({max_buffered_packets: 2, events_per_second: 3});
    try {
        const node = await f.node(['bot.snapshot', 'bot.logs']);
        const web = await f.web(randomUUID(), ['bot.snapshot']);
        const server_web = f.namespace.sockets.get(web.socket.id!)!;
        const transport = server_web.conn as unknown as {writeBuffer: unknown[]};
        server_web.conn.transport.writable = false;
        const event = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: null as string | null, stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
            name: 'status_update', payload: {status: 'offline', context_exists: false}};
        for (let seq = 1; seq <= 3; seq++) {
            assert.equal((await ack(node, 'bot_event', {...event, seq})).status, 'success');
            if (seq === 1) assert.equal(transport.writeBuffer.length, 1);
        }
        assert.equal(server_web.connected, false);
        assert.ok(transport.writeBuffer.length <= 4);
        const closed = received(node, 'forward_session_closed');
        node.emit('bot_log', {...event, seq: 4, name: 'log', payload: {records: []}});
        assert.equal((await closed)[0].code, 'RATE_LIMITED');
    } finally {await f.close();}
});

test('info_update enforces readiness and Node and Web token permissions independently of subscriptions', async () => {
    const f = await fixture();
    try {
        const node = await f.node(permissions, false);
        const web = await f.web();
        const update = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: randomUUID(), stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
            data: {name: 'health', value: 20}};
        code(await ack(node, 'info_update', update), 'NODE_OFFLINE');
        const restricted = await f.node(['bot.logs']);
        code(await ack(restricted, 'info_update', update), 'FORBIDDEN');
        const full = await f.node();
        const narrow = await f.connect();
        assert.equal((await ack(narrow, 'join_room', {token: token(f.s, 'web', web.grant_id,
            Date.now() + 10_000, 4, ['bot.logs'])})).status, 'success');
        let leaked = 0; narrow.on('info_update', () => leaked++);
        code(await ack(narrow, 'relay_request', request('get_context_info')), 'FORBIDDEN');
        assert.equal((await ack(full, 'info_update', update)).status, 'success');
        const allowed = await f.web(randomUUID(), ['bot.snapshot']);
        const pushed = received(allowed.socket, 'info_update');
        await ack(full, 'info_update', {...update, seq: 2}); await pushed;
        assert.equal(leaked, 0);
    } finally {await f.close();}
});

test('logs and info updates share the Node event rate and slow-Web buffer protection', async () => {
    for (const first_info of [false, true]) {
        const f = await fixture({events_per_second: 2, max_buffered_packets: 2});
        try {
            const node = await f.node(); const web = await f.web();
            const info = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
                context_generation: randomUUID(), stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
                data: {name: 'health', value: 20}};
            const log = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
                context_generation: null as string | null, stream_id: info.stream_id, seq: 2, emitted_at: Date.now(),
                name: 'log', payload: {records: [] as BotLogRecord[]}};
            const server_web = f.namespace.sockets.get(web.socket.id!)!;
            server_web.conn.transport.writable = false;
            assert.equal((await ack(node, first_info ? 'info_update' : 'bot_log', first_info ? info : log)).status, 'success');
            assert.equal((await ack(node, first_info ? 'bot_log' : 'info_update',
                first_info ? log : {...info, seq: 3})).status, 'success');
            const closed = received(node, 'forward_session_closed');
            node.emit('info_update', {...info, seq: 4});
            assert.equal((await closed)[0].code, 'RATE_LIMITED');
        } finally {await f.close();}
    }
    const f = await fixture({max_buffered_packets: 2});
    try {
        const node = await f.node(); const web = await f.web(randomUUID(), ['bot.snapshot']);
        const server_web = f.namespace.sockets.get(web.socket.id!)!;
        server_web.conn.transport.writable = false;
        const info = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: randomUUID(), stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
            data: {name: 'health', value: 20}};
        for (let seq = 1; seq <= 3; seq++) assert.equal((await ack(node, 'info_update', {...info, seq})).status, 'success');
        assert.equal(server_web.connected, false);
        assert.ok((server_web.conn as unknown as {writeBuffer: unknown[]}).writeBuffer.length <= 4);
    } finally {await f.close();}
});

test('async join awaits the adapter and rechecks revoked grant before binding', async () => {
    const f = await fixture();
    try {
        const grant_id = randomUUID();
        f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id, user_id: 4, permissions, expires_at: Date.now() + 10_000});
        const client = await f.connect();
        const socket = f.namespace.sockets.get(client.id!)!;
        const original = socket.join.bind(socket);
        let release!: () => void;
        let entered!: () => void;
        const joining = new Promise<void>(resolve => {entered = resolve;});
        socket.join = async room => {entered(); await new Promise<void>(resolve => {release = resolve;}); await original(room);};
        let completed = false;
        const joined = ack(client, 'join_room', {token: token(f.s, 'web', grant_id)}).then(value => {completed = true; return value;});
        await joining;
        assert.equal(completed, false);
        code(await ack(client, 'reauth', {token: token(f.s, 'web', grant_id)}), 'INVALID_REQUEST');
        f.store.revoke_grant({forward_session_id: f.s.forward_session_id, grant_id, reason: 'revoked_during_join'});
        release();
        await joined.catch((): void => {});
        await delay(0);
        assert.equal(socket.rooms.size, 0);
    } finally {await f.close();}
});

test('grant connection replacement and immutable identity, control disconnect allows reads but blocks joins, reauth, chat', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); node.on('relay_request', (_value, reply) => reply(snapshot(f.s.bot_id)));
        const web = await f.web();
        const old_closed = received(web.socket, 'forward_session_closed');
        const replacement = await f.connect();
        const credential = token(f.s, 'web', web.grant_id);
        const joined = await ack(replacement, 'join_room', {token: credential});
        assert.equal(joined.status, 'success'); assert.equal((await old_closed)[0].reason, 'grant_connection_replaced');
        assert.deepEqual(await ack(replacement, 'join_room', {token: credential}), joined);
        code(await ack(replacement, 'reauth', {token: token(f.s, 'web', web.grant_id, Date.now() + 10_000, 5)}), 'FORBIDDEN');
        f.store.set_control_available(false);
        assert.equal((await ack(replacement, 'relay_request', request('bot.snapshot'))).status, 'success');
        code(await ack(replacement, 'relay_request', request('bot.chat', {message: 'no'})), 'RELAY_UNAVAILABLE');
        code(await ack(replacement, 'reauth', {token: credential}), 'RELAY_UNAVAILABLE');
        const another = await f.connect(); code(await ack(another, 'join_room', {token: credential}), 'RELAY_UNAVAILABLE');
    } finally {await f.close();}
});

test('timeouts, pending caps and invalid Node responses return one strict ACK', async () => {
    const f = await fixture({max_pending_per_web: 1});
    try {
        const node = await f.node(); const web = await f.web();
        let started!: () => void;
        const received = new Promise<void>(resolve => {started = resolve;});
        node.on('relay_request', () => started());
        const pending = ack(web.socket, 'relay_request', request('bot.snapshot'));
        await received;
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'RATE_LIMITED');
        code(await pending, 'TIMEOUT');
        node.removeAllListeners('relay_request');
        node.on('relay_request', (_value, reply) => reply({status: 'success', data: {bot_id: 999}}));
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'INVALID_REQUEST');
        node.removeAllListeners('relay_request');
        node.on('relay_request', (_value, reply) => reply({status: 'success', data: {stream_id: randomUUID(), seq: 2,
            truncated: false, records: [1, 2].map((seq): BotLogRecord => ({id: randomUUID(), timestamp: Date.now(),
                context_generation: null, seq, segments: []}))}}));
        code(await ack(web.socket, 'relay_request', request('bot.logs', {limit: 1})), 'INVALID_REQUEST');
        assert.equal((f.namespace.sockets.get(node.id!)! as unknown as {acks: Map<number, unknown>}).acks.size, 0);
    } finally {await f.close();}
});

test('grant renewal does not silently extend socket lease and revoke releases pending transport timers', async () => {
    const f = await fixture();
    try {
        const node = await f.node(); const expires_at = Date.now() + 180;
        const web = await f.web(randomUUID(), permissions, expires_at);
        f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: web.grant_id, user_id: 4, permissions, expires_at: Date.now() + 10_000});
        const closed = received(web.socket, 'forward_session_closed');
        assert.equal((await closed)[0].code, 'TOKEN_EXPIRED');
        const renewed = await f.web();
        let started!: () => void; const request_received = new Promise<void>(resolve => {started = resolve;});
        node.on('relay_request', () => started());
        const response = ack(renewed.socket, 'relay_request', request('bot.snapshot'));
        await request_received;
        f.store.revoke_session({forward_session_id: f.s.forward_session_id, reason: 'ownership_changed'});
        // The close notification may terminate the client before its pending ACK arrives.
        await response.catch((): void => {});
        assert.equal(f.store.size, 0);
        assert.equal((f.namespace.sockets.get(node.id!) as unknown as {acks?: Map<number, unknown>})?.acks?.size ?? 0, 0);
    } finally {await f.close();}
});

test('initial-ready unused and final-Web idle reclamation release sessions with explicit reason', async () => {
    const unused = await fixture({unused_timeout_ms: 45});
    try {
        const node = await unused.node();
        const closed = await received(node, 'forward_session_closed');
        assert.equal(closed[0].reason, 'unused_session'); assert.equal(unused.store.size, 0);
    } finally {await unused.close();}
    const idle = await fixture({idle_timeout_ms: 45});
    try {
        const node = await idle.node(); const web = await idle.web();
        const closed = received(node, 'forward_session_closed'); web.socket.disconnect();
        assert.equal((await closed)[0].reason, 'last_web_left'); assert.equal(idle.store.size, 0);
    } finally {await idle.close();}
});

test('slow-client status buffer overflow and forbidden event floods require explicit recovery', async () => {
    const f = await fixture({requests_per_second: 6, max_buffered_packets: 2});
    try {
        const node = await f.node(); const web = await f.web();
        await ack(web.socket, 'relay_request', request('bot.subscribe', {events: ['status_update']}));
        const server_web = f.namespace.sockets.get(web.socket.id!)!;
        const transport = server_web.conn as unknown as {writeBuffer: unknown[]};
        const actual = transport.writeBuffer;
        let checks = 0;
        Object.defineProperty(transport, 'writeBuffer', {configurable: true, get: () => ++checks === 1 ? [null, null] : actual, set: () => {}});
        const closed = received(web.socket, 'forward_session_closed');
        const event = {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id, context_generation: null as string | null,
            stream_id: randomUUID(), seq: 1, emitted_at: Date.now(), name: 'status_update', payload: {status: 'offline', context_exists: false}};
        const replacement = await f.connect();
        const first = ack(replacement, 'join_room', {token: token(f.s, 'node')});
        await delay(10);
        Object.defineProperty(transport, 'writeBuffer', {configurable: true, writable: true, value: actual});
        await first;
        // Notification is emitted before disconnect; packet delivery depends on Engine.IO's buffer flush.
        assert.equal(server_web.connected, false);
        await Promise.race([closed, delay(30)]);
        code(await ack(replacement, 'bot_event', event), 'NODE_OFFLINE');
        const node_closed = received(replacement, 'forward_session_closed');
        for (let seq = 2; seq <= 10; seq++) replacement.emit('unknown', {...event, seq});
        assert.equal((await node_closed)[0].code, 'RATE_LIMITED');
    } finally {await f.close();}
});

test('scope conflicts and route disposer preserve HTTP ownership', async () => {
    const f = await fixture();
    try {
        assert.throws(() => f.store.open({...f.s, forward_session_id: randomUUID(), node_session_id: randomUUID(), lease_expires_at: Date.now() + 10_000}), {code: 'SESSION_STALE'});
        assert.throws(() => f.store.revoke_session({forward_session_id: f.s.forward_session_id, reason: 'x'.repeat(257)}), {code: 'INVALID_REQUEST'});
        const node = await f.node();
        const extra = install_relay_routes(f.namespace, f.store);
        assert.equal(f.namespace.listenerCount('connection'), 2);
        extra(); assert.equal(f.namespace.listenerCount('connection'), 1);
        assert.equal(node.connected, true);
        const closed = received(node, 'forward_session_closed');
        f.dispose();
        assert.equal((await closed)[0].reason, 'relay_routes_closed');
        assert.equal(f.namespace.listenerCount('connection'), 0);
        assert.equal(f.http.listening, true);
    } finally {await f.close();}
});

test('reauth extends only the same identity, reconnect restores subscriptions, revoked IDs cannot revive', async () => {
    const f = await fixture();
    try {
        const node = await f.node();
        const web = await f.web(randomUUID(), permissions, Date.now() + 180);
        const subscribe = request('bot.subscribe', {events: ['status_update']});
        await ack(web.socket, 'relay_request', subscribe);
        const expiry = Date.now() + 3000;
        f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: web.grant_id, user_id: 4, permissions, expires_at: expiry});
        const renewed = await ack(web.socket, 'reauth', {token: token(f.s, 'web', web.grant_id, expiry)});
        assert.equal(renewed.data.expires_at, expiry);
        await delay(200);
        assert.equal(web.socket.connected, true);
        const replacement = await f.connect();
        await ack(replacement, 'join_room', {token: token(f.s, 'web', web.grant_id, expiry)});
        await ack(replacement, 'relay_request', subscribe);
        const pushed = received(replacement, 'bot_event');
        assert.equal((await ack(node, 'bot_event', {forward_session_id: f.s.forward_session_id, bot_id: f.s.bot_id,
            context_generation: null, stream_id: randomUUID(), seq: 1, emitted_at: Date.now(),
            name: 'status_update', payload: {status: 'offline', context_exists: false}})).status, 'success');
        assert.equal((await pushed)[0].payload.status, 'offline');
        node.on('relay_request', (_value, reply) => reply(snapshot(f.s.bot_id)));
        assert.equal((await ack(replacement, 'relay_request', request('bot.snapshot'))).status, 'success');
        f.store.revoke_grant({forward_session_id: f.s.forward_session_id, grant_id: web.grant_id, reason: 'permission_removed'});
        assert.throws(() => f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: web.grant_id,
            user_id: 4, permissions, expires_at: expiry}), {code: 'FORBIDDEN'});
        f.store.revoke_session({forward_session_id: f.s.forward_session_id, reason: 'closed'});
        assert.throws(() => f.store.open({...f.s, lease_expires_at: expiry}), {code: 'SESSION_STALE'});
    } finally {await f.close();}
});

test('async join cannot commit after session revocation or disconnect, successful join ACK follows room membership', async () => {
    for (const action of ['revoke', 'disconnect', 'success']) {
        const f = await fixture();
        try {
            const client = await f.connect();
            const socket = f.namespace.sockets.get(client.id!)!;
            const join = socket.join.bind(socket);
            let release!: () => void;
            let entered!: () => void;
            const started = new Promise<void>(resolve => {entered = resolve;});
            socket.join = async room => {entered(); await new Promise<void>(resolve => {release = resolve;}); await join(room);};
            const joining = f.store.join(socket, {token: token(f.s, 'node')});
            await started;
            if (action === 'revoke') f.store.revoke_session({forward_session_id: f.s.forward_session_id, reason: 'revoked_while_joining'});
            if (action === 'disconnect') {client.disconnect(); await once(socket, 'disconnect');}
            release();
            if (action === 'success') {
                await joining;
                assert.equal(socket.rooms.size, 2);
            } else {
                await assert.rejects(joining, {code: action === 'revoke' ? 'SESSION_STALE' : 'CANCELLED'});
                await delay(0);
                assert.equal(socket.rooms.size, 0);
            }
        } finally {await f.close();}
    }
});

test('stalled async adapters are cancelled without waiting for join and late membership is removed', async () => {
    for (const action of ['disconnect', 'timeout']) {
        const f = await fixture({auth_timeout_ms: 45});
        try {
            const client = await f.connect();
            const socket = f.namespace.sockets.get(client.id!)!;
            const original = socket.join.bind(socket);
            let release!: () => void;
            let entered!: () => void;
            const started = new Promise<void>(resolve => {entered = resolve;});
            socket.join = async room => {entered(); await new Promise<void>(resolve => {release = resolve;}); await original(room);};
            const joining = f.store.join(socket, {token: token(f.s, 'node')});
            const rejected = assert.rejects(joining, {code: action === 'disconnect' ? 'CANCELLED' : 'UNAUTHORIZED'});
            await started;
            if (action === 'disconnect') client.disconnect();
            await rejected;
            release();
            await delay(0);
            assert.equal(socket.rooms.size, 0);
        } finally {await f.close();}
    }
});

test('capacity bounds reject new sessions, grants and sockets, including retained revocation records', async () => {
    const f = await fixture({max_sessions: 1, max_grants_per_session: 1, max_connections: 1});
    try {
        assert.throws(() => f.store.open({...scope(), bot_id: 9, lease_expires_at: Date.now() + 10_000}), {code: 'RATE_LIMITED'});
        const web = await f.web();
        assert.throws(() => f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: randomUUID(),
            user_id: 4, permissions, expires_at: Date.now() + 10_000}), {code: 'RATE_LIMITED'});
        const closed = received(web.socket, 'forward_session_closed');
        f.store.revoke_grant({forward_session_id: f.s.forward_session_id, grant_id: web.grant_id, reason: 'revoked'});
        await closed;
        assert.throws(() => f.store.authorize({forward_session_id: f.s.forward_session_id, grant_id: randomUUID(),
            user_id: 4, permissions, expires_at: Date.now() + 10_000}), {code: 'RATE_LIMITED'});
        const first = await f.connect();
        const rejected = await f.connect();
        assert.equal((await Promise.race([received(rejected, 'disconnect').then(() => true), delay(30).then(() => !rejected.connected)])), true);
        assert.equal(first.connected, true);
    } finally {await f.close();}
});

test('unauthenticated sockets and session leases expire, authentication is rate limited', async () => {
    const f = await fixture({auth_timeout_ms: 40, requests_per_second: 2});
    try {
        const idle = await f.connect();
        assert.equal((await received(idle, 'forward_session_closed'))[0].reason, 'authentication_timeout');
        const node = await f.connect();
        const credential = token(f.s, 'node');
        await ack(node, 'join_room', {token: credential});
        await ack(node, 'join_room', {token: credential});
        code(await ack(node, 'join_room', {token: credential}), 'RATE_LIMITED');
        assert.equal(f.namespace.sockets.get(node.id!)!.rooms.size, 2);
    } finally {await f.close();}
    const lease = await fixture();
    try {
        const short = {...scope(), node_id: 10};
        lease.store.open({...short, lease_expires_at: Date.now() + 70});
        const node = await lease.connect();
        const joined = await ack(node, 'join_room', {token: token(short, 'node')});
        assert.equal(joined.status, 'success');
        assert.ok(joined.data.expires_at <= Date.now() + 70);
        assert.equal((await received(node, 'forward_session_closed'))[0].code, 'TOKEN_EXPIRED');
        await delay(10);
        assert.equal(lease.store.size, 1);
    } finally {await lease.close();}
});

test('slow Node RPC queues and unauthenticated unknown-event ACK queues stay bounded', async () => {
    const f = await fixture({max_buffered_packets: 2, rpc_timeout_ms: 20});
    try {
        const node = await f.node(); const web = await f.web();
        const server_node = f.namespace.sockets.get(node.id!)!;
        server_node.conn.transport.writable = false;
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'TIMEOUT');
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'TIMEOUT');
        code(await ack(web.socket, 'relay_request', request('bot.snapshot')), 'RESYNC_REQUIRED');
        assert.equal(server_node.connected, false);
        const raw = await f.connect();
        const server_raw = f.namespace.sockets.get(raw.id!)!;
        server_raw.conn.transport.writable = false;
        for (let index = 0; index < 20; index++) raw.emit('unknown', {}, () => {});
        await delay(20);
        assert.equal(server_raw.connected, false);
        // Two bounded replies plus the explicit close notification and disconnect packet.
        assert.ok((server_raw.conn as unknown as {writeBuffer: unknown[]}).writeBuffer.length <= 4);
    } finally {await f.close();}
});