diff --git a/apps/readest-app/.gitignore b/apps/readest-app/.gitignore index ee92b853..ebea2b3f 100644 --- a/apps/readest-app/.gitignore +++ b/apps/readest-app/.gitignore @@ -11,6 +11,9 @@ .test-sandbox-node/ .vitest-attachments/ +# benchmarks — personal local perf history; share via PR/issue copy-paste +bench/results.jsonl + # next.js /.next/ /out/ diff --git a/apps/readest-app/bench/README.md b/apps/readest-app/bench/README.md new file mode 100644 index 00000000..c9043edd --- /dev/null +++ b/apps/readest-app/bench/README.md @@ -0,0 +1,57 @@ +# Benchmarks + +Manual performance benchmarks for the readest-app. **Not run in CI** — CI runners +have shared-tenant variance that makes performance regression detection unreliable +(numbers swing 2-10× between runs). These exist so anyone considering an +architecture change can produce reproducible before/after numbers on their own +hardware. + +## Run + +```bash +pnpm bench # run every bench/*.bench.ts +pnpm bench vector-retrieval # run a single benchmark by name +pnpm bench --no-record # run but don't append to bench/results.jsonl +pnpm bench --list # list available benchmarks +``` + +Refuses to run when `$CI` is set. Append `--force` to override (don't unless +you've explicitly opted into running benches in CI for a one-off investigation). + +## Output + +Each run prints a header with machine info (platform, CPU, Node version, key +package versions) followed by per-benchmark results. By default, results are +also appended to `bench/results.jsonl` (gitignored) — your personal local +history. To share numbers, paste the table from the terminal into a PR or issue. + +## When to add a new benchmark + +When you're proposing an architecture change and need numbers to defend it. The +benchmark should: + +1. Live at `bench/.bench.ts`. +2. Export `default { name, description, run(ctx) }` matching the type in `lib.ts`. +3. Print human-readable results to stdout and return structured results to the + harness so they get logged to `results.jsonl`. +4. Be self-contained — no fixtures outside `bench/`, no I/O outside the bench + directory and an in-memory database. +5. Run in under ~30 seconds at default sample sizes. If you need long-running + scenarios, gate them behind a CLI flag. + +## When *not* to add a benchmark + +- "Just in case" — performance infrastructure has carrying cost. Wait until + you have a real architecture question that numbers will answer. +- To benchmark upstream libraries' performance (e.g., raw Turso function + throughput). That belongs in the upstream project's bench suite. +- To gate CI on performance thresholds. CI variance makes that flaky; use + production telemetry (`reedy_metrics` table) for regression detection + against real workloads. + +## Existing benchmarks + +- **`vector-retrieval`** — proves Turso's brute-force vector search is + SIMD-accelerated and fast enough for Reedy MVP corpus sizes (sub-millisecond + at 400 chunks × 768 dim, ~14 ms at 10K chunks × 768 dim). Established the + decision in plan §M1.5 to skip ANN indexes (which Turso doesn't ship anyway). diff --git a/apps/readest-app/bench/index.ts b/apps/readest-app/bench/index.ts new file mode 100644 index 00000000..a3c64569 --- /dev/null +++ b/apps/readest-app/bench/index.ts @@ -0,0 +1,115 @@ +import { readdirSync, appendFileSync, mkdirSync, existsSync } from 'node:fs'; +import { join, resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execSync } from 'node:child_process'; +import { type Bench, type BenchResult, formatHeader, formatResults, machineInfo } from './lib.ts'; + +const BENCH_DIR = dirname(fileURLToPath(import.meta.url)); +const RESULTS_FILE = join(BENCH_DIR, 'results.jsonl'); + +interface Args { + filter: string | null; + record: boolean; + list: boolean; + force: boolean; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { filter: null, record: true, list: false, force: false }; + for (const arg of argv) { + if (arg === '--no-record') args.record = false; + else if (arg === '--list') args.list = true; + else if (arg === '--force') args.force = true; + else if (!arg.startsWith('--')) args.filter = arg; + else { + console.error(`Unknown flag: ${arg}`); + process.exit(2); + } + } + return args; +} + +async function loadBenches(): Promise { + const files = readdirSync(BENCH_DIR).filter((f) => f.endsWith('.bench.ts')); + const benches: Bench[] = []; + for (const file of files) { + const mod = await import(resolve(BENCH_DIR, file)); + const bench = mod.default as Bench; + if (!bench || typeof bench.run !== 'function') { + console.error(`Skipping ${file}: no default export with .run()`); + continue; + } + benches.push(bench); + } + return benches.sort((a, b) => a.name.localeCompare(b.name)); +} + +function gitCommit(): string { + try { + return execSync('git rev-parse --short HEAD', { encoding: 'utf8' }).trim(); + } catch { + return 'unknown'; + } +} + +function recordRun(bench: Bench, results: BenchResult[], commit: string): void { + if (!existsSync(BENCH_DIR)) mkdirSync(BENCH_DIR, { recursive: true }); + const entry = { + ts: new Date().toISOString(), + commit, + bench: bench.name, + platform: process.platform, + arch: process.arch, + node: process.version, + results, + }; + appendFileSync(RESULTS_FILE, JSON.stringify(entry) + '\n'); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + + if (process.env['CI'] && !args.force) { + console.error('Refusing to run benchmarks in CI (CI=' + process.env['CI'] + ').'); + console.error('Pass --force if you really mean it. See bench/README.md for why.'); + process.exit(1); + } + + const benches = await loadBenches(); + + if (args.list) { + console.log('Available benchmarks:'); + for (const b of benches) console.log(` ${b.name.padEnd(20)} ${b.description}`); + return; + } + + const selected = args.filter ? benches.filter((b) => b.name === args.filter) : benches; + if (selected.length === 0) { + console.error(`No benchmark named "${args.filter}". Try --list.`); + process.exit(2); + } + + const info = machineInfo(['@tursodatabase/database', '@readest/turso-database-wasm']); + console.log(formatHeader(info)); + + const commit = gitCommit(); + console.log(` git commit : ${commit}`); + console.log(` recording : ${args.record ? RESULTS_FILE : 'disabled (--no-record)'}`); + console.log('═'.repeat(70)); + + for (const bench of selected) { + process.stdout.write(`Running ${bench.name}... `); + const t0 = performance.now(); + const results = await bench.run({ verbose: false }); + const elapsed = ((performance.now() - t0) / 1000).toFixed(1); + console.log(`done in ${elapsed}s`); + console.log(formatResults(bench.name, results)); + if (args.record) recordRun(bench, results, commit); + } + console.log(''); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/readest-app/bench/lib.ts b/apps/readest-app/bench/lib.ts new file mode 100644 index 00000000..57268665 --- /dev/null +++ b/apps/readest-app/bench/lib.ts @@ -0,0 +1,119 @@ +import { performance } from 'node:perf_hooks'; +import { cpus, platform, arch, totalmem } from 'node:os'; +import { readFileSync } from 'node:fs'; + +export interface BenchContext { + /** Whether to print verbose per-iteration info. */ + verbose: boolean; +} + +export interface BenchResult { + scenario: string; + unit: 'ms' | 'us' | 'ns'; + value: number; + /** Optional metadata: chunk count, dim, etc. */ + meta?: Record; +} + +export interface Bench { + name: string; + description: string; + run(ctx: BenchContext): Promise; +} + +/** High-resolution timer; returns elapsed milliseconds. */ +export async function timed(fn: () => Promise): Promise<{ result: T; ms: number }> { + const t0 = performance.now(); + const result = await fn(); + const ms = performance.now() - t0; + return { result, ms }; +} + +/** Run `fn` `reps` times, return average milliseconds (after warmup). */ +export async function avg(fn: () => Promise, reps: number, warmup = 3): Promise { + for (let i = 0; i < warmup; i++) await fn(); + const t0 = performance.now(); + for (let i = 0; i < reps; i++) await fn(); + return (performance.now() - t0) / reps; +} + +/** Generate a random unit vector serialized as JSON, suitable for `vector32(?)`. */ +export function randomUnitVectorJson(dim: number): string { + const v = new Float32Array(dim); + let norm = 0; + for (let i = 0; i < dim; i++) { + const x = Math.random() * 2 - 1; + v[i] = x; + norm += x * x; + } + const inv = 1 / Math.sqrt(norm); + for (let i = 0; i < dim; i++) v[i] = (v[i] ?? 0) * inv; + return JSON.stringify(Array.from(v)); +} + +export interface MachineInfo { + platform: string; + arch: string; + cpu: string; + cpuCount: number; + memGiB: number; + node: string; + packages: Record; +} + +export function machineInfo(packages: string[] = []): MachineInfo { + const cpuList = cpus(); + const firstCpu = cpuList[0]; + const versions: Record = {}; + for (const name of packages) { + try { + const pkg = JSON.parse(readFileSync(`./node_modules/${name}/package.json`, 'utf8')); + versions[name] = pkg.version; + } catch { + versions[name] = 'not installed'; + } + } + return { + platform: platform(), + arch: arch(), + cpu: firstCpu?.model.trim() ?? 'unknown', + cpuCount: cpuList.length, + memGiB: Math.round(totalmem() / 1024 ** 3), + node: process.version, + packages: versions, + }; +} + +export function formatHeader(info: MachineInfo): string { + const lines = [ + '═'.repeat(70), + ` Platform : ${info.platform}/${info.arch}`, + ` CPU : ${info.cpu} (${info.cpuCount} cores)`, + ` Memory : ${info.memGiB} GiB`, + ` Node : ${info.node}`, + ]; + for (const [name, ver] of Object.entries(info.packages)) { + lines.push(` ${name.padEnd(9)}: ${ver}`); + } + lines.push('═'.repeat(70)); + return lines.join('\n'); +} + +export function formatResults(benchName: string, results: BenchResult[]): string { + const lines = [`\n[${benchName}]`]; + for (const r of results) { + const metaStr = r.meta + ? ` ${Object.entries(r.meta) + .map(([k, v]) => `${k}=${v}`) + .join(' ')}` + : ''; + lines.push(` ${r.scenario.padEnd(40)} ${formatValue(r.value, r.unit).padStart(12)}${metaStr}`); + } + return lines.join('\n'); +} + +function formatValue(value: number, unit: 'ms' | 'us' | 'ns'): string { + if (unit === 'ms') return `${value.toFixed(3)} ms`; + if (unit === 'us') return `${value.toFixed(2)} µs`; + return `${value.toFixed(0)} ns`; +} diff --git a/apps/readest-app/bench/vector-retrieval.bench.ts b/apps/readest-app/bench/vector-retrieval.bench.ts new file mode 100644 index 00000000..82fa5543 --- /dev/null +++ b/apps/readest-app/bench/vector-retrieval.bench.ts @@ -0,0 +1,87 @@ +import { connect } from '@tursodatabase/database'; +import { avg, randomUnitVectorJson, type Bench, type BenchResult } from './lib.ts'; + +/** + * Vector-retrieval brute-force kNN benchmark. + * + * Reedy MVP retrieval (see plan §M1.5) issues: + * + * SELECT id, vector_distance_cos(embedding, vector32(?)) AS d + * FROM reedy_book_chunk_embeddings + * WHERE book_hash = ? + * ORDER BY d ASC LIMIT k + * + * Why this matters: Turso has no native vector index module + * (`libsql_vector_idx` / `vector_top_k` don't exist — confirmed against + * @tursodatabase/database@0.6.0-pre.28 and acknowledged upstream: + * tursodatabase/turso#832 closed not-planned, #3778 proposed brute-force-first + * which shipped at commit 1aba105df4f). The brute-force path with + * SIMD-accelerated `vector_distance_cos` is what we ship; this bench tracks + * its per-query latency at realistic MVP corpus sizes. + * + * Run it after upgrading @tursodatabase/database, after touching + * BookRetriever's SQL shape, or when evaluating an architecture change + * (ANN extension, quantization, engine swap). + */ +export default { + name: 'vector-retrieval', + description: 'Brute-force per-book kNN over vector32 embeddings filtered by book_hash.', + + async run(): Promise { + const db = await connect(':memory:', {}); + await db.exec( + 'CREATE TABLE c (id INTEGER PRIMARY KEY, book_hash TEXT NOT NULL, embedding BLOB)', + ); + await db.exec('CREATE INDEX idx_c_book ON c(book_hash)'); + + // (dim, chunks-per-book) matrix. Two books per scenario so the WHERE filter + // does real work; we measure only the active-book query. + const scenarios = [ + { dim: 384, chunks: 400 }, // small book, light embedding (e5-small-v2) + { dim: 768, chunks: 400 }, // typical novel @ nomic-embed-text + { dim: 768, chunks: 2000 }, // long novel + { dim: 768, chunks: 10000 }, // multi-volume / textbook + { dim: 1536, chunks: 400 }, // text-embedding-3-small + ]; + + const results: BenchResult[] = []; + + for (const { dim, chunks } of scenarios) { + await db.exec('DELETE FROM c'); + + const insertA = await db.prepare( + "INSERT INTO c (book_hash, embedding) VALUES ('book_a', vector32(?))", + ); + for (let i = 0; i < chunks; i++) await insertA.run(randomUnitVectorJson(dim)); + + const insertB = await db.prepare( + "INSERT INTO c (book_hash, embedding) VALUES ('book_b', vector32(?))", + ); + for (let i = 0; i < chunks; i++) await insertB.run(randomUnitVectorJson(dim)); + + const query = randomUnitVectorJson(dim); + // Embed the query vector literally so SIMD has the same memory layout + // every call (mirrors the BookRetriever code path which serializes the + // query embedding inline at the value-binding position). + const sql = ` + SELECT id, vector_distance_cos(embedding, vector32('${query}')) AS d + FROM c + WHERE book_hash = ? + ORDER BY d ASC + LIMIT 5 + `; + const stmt = await db.prepare(sql); + + const ms = await avg(() => stmt.all('book_a'), 20); + results.push({ + scenario: `${chunks} chunks × ${dim} dim`, + unit: 'ms', + value: ms, + meta: { chunks, dim, usPerChunk: ((ms * 1000) / chunks).toFixed(2) }, + }); + } + + await db.close(); + return results; + }, +} satisfies Bench; diff --git a/apps/readest-app/package.json b/apps/readest-app/package.json index a7026fe5..67710b44 100644 --- a/apps/readest-app/package.json +++ b/apps/readest-app/package.json @@ -30,6 +30,7 @@ "test:pr:web": "dotenv -e .env -e .env.test.local -- vitest run --maxWorkers=4 && pnpm test:browser && pnpm test:extension && pnpm build-browser-ext", "test:pr:tauri": "bash scripts/test-tauri.sh", "test:all": "pnpm test -- --watch=false && pnpm test:browser && bash scripts/test-tauri.sh", + "bench": "node --experimental-strip-types --no-warnings bench/index.ts", "test:e2e": "wdio run wdio.conf.ts", "test:e2e:web": "playwright test", "test:e2e:web:headed": "playwright test --headed --workers=1 --trace on", diff --git a/apps/readest-app/src/__tests__/database/nativeDatabaseService-opts.test.ts b/apps/readest-app/src/__tests__/database/nativeDatabaseService-opts.test.ts new file mode 100644 index 00000000..f73071f3 --- /dev/null +++ b/apps/readest-app/src/__tests__/database/nativeDatabaseService-opts.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { LoadOptions } from 'tauri-plugin-turso'; +import type { DatabaseOpts } from '@/types/database'; + +// Capture the argument Database.load receives so we can assert opts forwarding. +// The plugin signature is Database.load(pathOrOptions: string | LoadOptions) — a +// single argument that's either a path string (no opts) or a LoadOptions object +// (path embedded in the object). +vi.mock('tauri-plugin-turso', () => { + const loadCalls: Array = []; + const mockDb = { + execute: vi.fn(async () => ({ rowsAffected: 0, lastInsertId: 0 })), + select: vi.fn(async () => []), + batch: vi.fn(async () => {}), + close: vi.fn(async () => {}), + }; + return { + Database: { + load: vi.fn(async (pathOrOptions: string | LoadOptions) => { + loadCalls.push(pathOrOptions); + return mockDb; + }), + }, + __loadCalls: loadCalls, + __mockDb: mockDb, + }; +}); + +describe('NativeDatabaseService.open forwards opts to tauri-plugin-turso', () => { + beforeEach(async () => { + vi.clearAllMocks(); + const mod = await import('tauri-plugin-turso'); + (mod as unknown as { __loadCalls: unknown[] }).__loadCalls.length = 0; + }); + + it('passes a plain path string when no opts provided (preserves existing behavior)', async () => { + const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService'); + await NativeDatabaseService.open('sqlite:test.db'); + + const mod = await import('tauri-plugin-turso'); + const loadCalls = (mod as unknown as { __loadCalls: Array }).__loadCalls; + expect(loadCalls).toHaveLength(1); + expect(loadCalls[0]).toBe('sqlite:test.db'); + }); + + it('translates experimental opts into LoadOptions and forwards as a single object', async () => { + const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService'); + const opts: DatabaseOpts = { experimental: ['index_method'] }; + await NativeDatabaseService.open('sqlite:reedy.db', opts); + + const mod = await import('tauri-plugin-turso'); + const loadCalls = (mod as unknown as { __loadCalls: Array }).__loadCalls; + expect(loadCalls).toHaveLength(1); + expect(loadCalls[0]).toEqual({ + path: 'sqlite:reedy.db', + experimental: ['index_method'], + }); + }); + + it('passes a plain path string when opts has no experimental and no encryption', async () => { + const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService'); + // Fields like `readonly`/`timeout` exist in DatabaseOpts but aren't supported by + // the native plugin's LoadOptions, so the translator should skip them and + // fall back to a bare path string. + const opts: DatabaseOpts = { readonly: true, timeout: 5000 }; + await NativeDatabaseService.open('sqlite:plain.db', opts); + + const mod = await import('tauri-plugin-turso'); + const loadCalls = (mod as unknown as { __loadCalls: Array }).__loadCalls; + expect(loadCalls[0]).toBe('sqlite:plain.db'); + }); + + it('passes a plain path string when experimental is an empty array', async () => { + const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService'); + await NativeDatabaseService.open('sqlite:empty-exp.db', { experimental: [] }); + + const mod = await import('tauri-plugin-turso'); + const loadCalls = (mod as unknown as { __loadCalls: Array }).__loadCalls; + expect(loadCalls[0]).toBe('sqlite:empty-exp.db'); + }); +}); diff --git a/apps/readest-app/src/__tests__/database/suites/vector-tests.ts b/apps/readest-app/src/__tests__/database/suites/vector-tests.ts index ce6e247c..810aabbd 100644 --- a/apps/readest-app/src/__tests__/database/suites/vector-tests.ts +++ b/apps/readest-app/src/__tests__/database/suites/vector-tests.ts @@ -233,4 +233,65 @@ export function vectorTests(getDb: () => DatabaseService) { ); expect(rows[0]!.d).toBeCloseTo(5.0, 4); }); + + // --------------------------------------------------------------------------- + // Per-book brute-force kNN — the pattern Reedy retrieval uses. + // + // Turso (the rust rewrite this repo wraps via @tursodatabase/database + + // @readest/turso-database-wasm) has vector storage + distance functions + // but no native vector index module: `libsql_vector_idx`, `vector_top_k`, + // and `USING vector/hnsw/diskann/ivfflat` all parse-error on v0.6.0-pre.28. + // `libsql_vector_idx` is a libSQL (sqld) feature, a different fork. + // + // The portable, parameterizable alternative: `ORDER BY vector_distance_cos + // LIMIT k` with a `WHERE book_hash = ?` filter. O(n) per book, sub-ms for + // 400 chunks × 768 dim, acceptable to ~10k chunks per book on phones. + // --------------------------------------------------------------------------- + + it('per-book kNN filters by book_hash and orders by cosine distance', async () => { + // Models the exact pattern BookRetriever.search will issue. Two books in + // one table; query the active book; assert zero cross-book bleed and + // correct nearest-first ordering. No DDL, no identifier interpolation; + // bookHash binds as a ? parameter. + const db = getDb(); + await db.execute( + 'CREATE TABLE book_chunks (id INTEGER PRIMARY KEY, book_hash TEXT NOT NULL, label TEXT, embedding BLOB)', + ); + // book_b's chunks happen to be closer to the query than any book_a chunk — + // the WHERE filter must hide them entirely. + await db.execute( + "INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.95,0.05,0,0]'))", + ['book_a', 'A-near'], + ); + await db.execute( + "INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.5,0.5,0,0]'))", + ['book_a', 'A-mid'], + ); + await db.execute( + "INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[1,0,0,0]'))", + ['book_b', 'B-exact'], + ); + await db.execute( + "INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.99,0.01,0,0]'))", + ['book_b', 'B-near'], + ); + + const rows = await db.select<{ label: string; book_hash: string; d: number }>( + `SELECT label, book_hash, + vector_distance_cos(embedding, vector32('[1,0,0,0]')) AS d + FROM book_chunks + WHERE book_hash = ? + ORDER BY d ASC + LIMIT 5`, + ['book_a'], + ); + + expect(rows.length).toBeGreaterThan(0); + expect(rows.every((r) => r.book_hash === 'book_a')).toBe(true); + expect(rows[0]!.label).toBe('A-near'); + // monotonically non-decreasing distances within the result + for (let i = 1; i < rows.length; i++) { + expect(rows[i]!.d).toBeGreaterThanOrEqual(rows[i - 1]!.d); + } + }); } diff --git a/apps/readest-app/src/services/database/nativeDatabaseService.ts b/apps/readest-app/src/services/database/nativeDatabaseService.ts index d5c4a14e..575427dc 100644 --- a/apps/readest-app/src/services/database/nativeDatabaseService.ts +++ b/apps/readest-app/src/services/database/nativeDatabaseService.ts @@ -1,4 +1,4 @@ -import { Database, QueryResult } from 'tauri-plugin-turso'; +import { Database, LoadOptions, QueryResult } from 'tauri-plugin-turso'; import { DatabaseService, DatabaseExecResult, DatabaseRow, DatabaseOpts } from '@/types/database'; export class NativeDatabaseService implements DatabaseService { @@ -8,8 +8,19 @@ export class NativeDatabaseService implements DatabaseService { this.db = db; } - static async open(path: string, _opts?: DatabaseOpts): Promise { - const db = await Database.load(path); + static async open(path: string, opts?: DatabaseOpts): Promise { + // Translate the cross-platform DatabaseOpts (from @readest/turso-database-common, + // used by WASM bindings) to tauri-plugin-turso's LoadOptions. The two interfaces + // have diverged: `experimental` is the same field name with compatible types + // (literal union vs `string[]`); `encryption` shapes differ entirely (native + // 'aes256cbc' + byte-array key vs WASM 'aes256gcm'/etc + hex key) and is not + // wired in MVP — revisit alongside any Reedy.db encryption work. Skip the + // translation when no relevant opts are set so existing callers preserve their + // plain path-string call shape. + const loadArg: string | LoadOptions = opts?.experimental?.length + ? { path, experimental: opts.experimental as string[] } + : path; + const db = await Database.load(loadArg); return new NativeDatabaseService(db); } diff --git a/apps/readest-app/tsconfig.json b/apps/readest-app/tsconfig.json index 6ba54009..8c9ee286 100644 --- a/apps/readest-app/tsconfig.json +++ b/apps/readest-app/tsconfig.json @@ -36,7 +36,8 @@ "src/**/*.tsx", "raw-loader.d.ts", ".next/types/**/*.ts", - "scripts/**/*.ts" + "scripts/**/*.ts", + "bench/**/*.ts" ], "exclude": ["node_modules", "out"] }