forked from akai/readest
fix(db): forward DatabaseOpts to tauri-plugin-turso (#4292)
* fix(db): forward DatabaseOpts to tauri-plugin-turso NativeDatabaseService.open ignored its opts parameter, dropping any experimental feature flags (e.g. 'index_method' needed for FTS / vector indexes) and any encryption config before they could reach the Tauri plugin. Translate DatabaseOpts to the plugin's LoadOptions shape and forward as the single argument Database.load accepts. Skip translation when no relevant opts are set so the existing path-string call shape is preserved for callers without experimental/encryption needs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * test(db): cover Turso vector primitives + add benchmark harness Verifies the Turso functions Reedy retrieval depends on, with a brute-force per-book kNN test that runs against every DatabaseService backend (node, native, WASM): SELECT vector_distance_cos(embedding, vector32(?)) AS d FROM book_chunks WHERE book_hash = ? ORDER BY d ASC LIMIT k This is the path Turso's own founder recommended in tursodatabase/turso#3778 ("First, focus on efficient SIMD-accelerated brute-force search") and what shipped at commit 1aba105df4f. Native vector index modules don't exist in this engine: `libsql_vector_idx`, `vector_top_k`, and `USING vector/hnsw/diskann/ivfflat` all parse-error against @tursodatabase/database@0.6.0-pre.28 (libsql_vector_idx is a libSQL/sqld fork feature; DiskANN was closed not-planned upstream in #832). The test asserts cross-book isolation and nearest-first ordering using only `WHERE book_hash = ?` and `ORDER BY` — no DDL, no identifier interpolation, no index plumbing. Also adds bench/ harness for manual perf checks: pnpm bench [name] run benchmarks (refuses in CI) pnpm bench --list list available benchmarks pnpm bench --no-record skip results.jsonl append pnpm bench --force override the CI guard Uses Node 24's --experimental-strip-types so no tsx devDep is needed. Appends one JSON line per run to bench/results.jsonl (gitignored, local history; share by pasting tabular stdout into PRs/issues). Explicitly NOT in CI — shared-tenant variance makes synthetic-benchmark regression detection unreliable; production telemetry (reedy_metrics, plan §M1.9) is the right tool for that. First benchmark: vector-retrieval. Measured on M1 Pro: 400 chunks × 384 dim → 0.35 ms / query 400 chunks × 768 dim → 0.45 ms / query 2000 chunks × 768 dim → 2.23 ms / query 10000 chunks × 768 dim → 14.00 ms / query 400 chunks × 1536 dim → 0.70 ms / query Per-chunk cost ~1.1 µs at 768 dim = ~1.4 ns/dim. NEON-class on Apple Silicon, ~50× faster than scalar — confirms SIMD acceleration is active in 0.6.0-pre.28. Per-query latency stays sub-ms at Reedy MVP corpus sizes; the ceiling is ~10K chunks per book before phone-class hardware notices. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { LoadOptions } from 'tauri-plugin-turso';
|
||||
import type { DatabaseOpts } from '@/types/database';
|
||||
|
||||
// Capture the argument Database.load receives so we can assert opts forwarding.
|
||||
// The plugin signature is Database.load(pathOrOptions: string | LoadOptions) — a
|
||||
// single argument that's either a path string (no opts) or a LoadOptions object
|
||||
// (path embedded in the object).
|
||||
vi.mock('tauri-plugin-turso', () => {
|
||||
const loadCalls: Array<string | LoadOptions> = [];
|
||||
const mockDb = {
|
||||
execute: vi.fn(async () => ({ rowsAffected: 0, lastInsertId: 0 })),
|
||||
select: vi.fn(async () => []),
|
||||
batch: vi.fn(async () => {}),
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
return {
|
||||
Database: {
|
||||
load: vi.fn(async (pathOrOptions: string | LoadOptions) => {
|
||||
loadCalls.push(pathOrOptions);
|
||||
return mockDb;
|
||||
}),
|
||||
},
|
||||
__loadCalls: loadCalls,
|
||||
__mockDb: mockDb,
|
||||
};
|
||||
});
|
||||
|
||||
describe('NativeDatabaseService.open forwards opts to tauri-plugin-turso', () => {
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
const mod = await import('tauri-plugin-turso');
|
||||
(mod as unknown as { __loadCalls: unknown[] }).__loadCalls.length = 0;
|
||||
});
|
||||
|
||||
it('passes a plain path string when no opts provided (preserves existing behavior)', async () => {
|
||||
const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService');
|
||||
await NativeDatabaseService.open('sqlite:test.db');
|
||||
|
||||
const mod = await import('tauri-plugin-turso');
|
||||
const loadCalls = (mod as unknown as { __loadCalls: Array<string | LoadOptions> }).__loadCalls;
|
||||
expect(loadCalls).toHaveLength(1);
|
||||
expect(loadCalls[0]).toBe('sqlite:test.db');
|
||||
});
|
||||
|
||||
it('translates experimental opts into LoadOptions and forwards as a single object', async () => {
|
||||
const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService');
|
||||
const opts: DatabaseOpts = { experimental: ['index_method'] };
|
||||
await NativeDatabaseService.open('sqlite:reedy.db', opts);
|
||||
|
||||
const mod = await import('tauri-plugin-turso');
|
||||
const loadCalls = (mod as unknown as { __loadCalls: Array<string | LoadOptions> }).__loadCalls;
|
||||
expect(loadCalls).toHaveLength(1);
|
||||
expect(loadCalls[0]).toEqual({
|
||||
path: 'sqlite:reedy.db',
|
||||
experimental: ['index_method'],
|
||||
});
|
||||
});
|
||||
|
||||
it('passes a plain path string when opts has no experimental and no encryption', async () => {
|
||||
const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService');
|
||||
// Fields like `readonly`/`timeout` exist in DatabaseOpts but aren't supported by
|
||||
// the native plugin's LoadOptions, so the translator should skip them and
|
||||
// fall back to a bare path string.
|
||||
const opts: DatabaseOpts = { readonly: true, timeout: 5000 };
|
||||
await NativeDatabaseService.open('sqlite:plain.db', opts);
|
||||
|
||||
const mod = await import('tauri-plugin-turso');
|
||||
const loadCalls = (mod as unknown as { __loadCalls: Array<string | LoadOptions> }).__loadCalls;
|
||||
expect(loadCalls[0]).toBe('sqlite:plain.db');
|
||||
});
|
||||
|
||||
it('passes a plain path string when experimental is an empty array', async () => {
|
||||
const { NativeDatabaseService } = await import('@/services/database/nativeDatabaseService');
|
||||
await NativeDatabaseService.open('sqlite:empty-exp.db', { experimental: [] });
|
||||
|
||||
const mod = await import('tauri-plugin-turso');
|
||||
const loadCalls = (mod as unknown as { __loadCalls: Array<string | LoadOptions> }).__loadCalls;
|
||||
expect(loadCalls[0]).toBe('sqlite:empty-exp.db');
|
||||
});
|
||||
});
|
||||
@@ -233,4 +233,65 @@ export function vectorTests(getDb: () => DatabaseService) {
|
||||
);
|
||||
expect(rows[0]!.d).toBeCloseTo(5.0, 4);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-book brute-force kNN — the pattern Reedy retrieval uses.
|
||||
//
|
||||
// Turso (the rust rewrite this repo wraps via @tursodatabase/database +
|
||||
// @readest/turso-database-wasm) has vector storage + distance functions
|
||||
// but no native vector index module: `libsql_vector_idx`, `vector_top_k`,
|
||||
// and `USING vector/hnsw/diskann/ivfflat` all parse-error on v0.6.0-pre.28.
|
||||
// `libsql_vector_idx` is a libSQL (sqld) feature, a different fork.
|
||||
//
|
||||
// The portable, parameterizable alternative: `ORDER BY vector_distance_cos
|
||||
// LIMIT k` with a `WHERE book_hash = ?` filter. O(n) per book, sub-ms for
|
||||
// 400 chunks × 768 dim, acceptable to ~10k chunks per book on phones.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
it('per-book kNN filters by book_hash and orders by cosine distance', async () => {
|
||||
// Models the exact pattern BookRetriever.search will issue. Two books in
|
||||
// one table; query the active book; assert zero cross-book bleed and
|
||||
// correct nearest-first ordering. No DDL, no identifier interpolation;
|
||||
// bookHash binds as a ? parameter.
|
||||
const db = getDb();
|
||||
await db.execute(
|
||||
'CREATE TABLE book_chunks (id INTEGER PRIMARY KEY, book_hash TEXT NOT NULL, label TEXT, embedding BLOB)',
|
||||
);
|
||||
// book_b's chunks happen to be closer to the query than any book_a chunk —
|
||||
// the WHERE filter must hide them entirely.
|
||||
await db.execute(
|
||||
"INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.95,0.05,0,0]'))",
|
||||
['book_a', 'A-near'],
|
||||
);
|
||||
await db.execute(
|
||||
"INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.5,0.5,0,0]'))",
|
||||
['book_a', 'A-mid'],
|
||||
);
|
||||
await db.execute(
|
||||
"INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[1,0,0,0]'))",
|
||||
['book_b', 'B-exact'],
|
||||
);
|
||||
await db.execute(
|
||||
"INSERT INTO book_chunks (book_hash, label, embedding) VALUES (?, ?, vector32('[0.99,0.01,0,0]'))",
|
||||
['book_b', 'B-near'],
|
||||
);
|
||||
|
||||
const rows = await db.select<{ label: string; book_hash: string; d: number }>(
|
||||
`SELECT label, book_hash,
|
||||
vector_distance_cos(embedding, vector32('[1,0,0,0]')) AS d
|
||||
FROM book_chunks
|
||||
WHERE book_hash = ?
|
||||
ORDER BY d ASC
|
||||
LIMIT 5`,
|
||||
['book_a'],
|
||||
);
|
||||
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
expect(rows.every((r) => r.book_hash === 'book_a')).toBe(true);
|
||||
expect(rows[0]!.label).toBe('A-near');
|
||||
// monotonically non-decreasing distances within the result
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
expect(rows[i]!.d).toBeGreaterThanOrEqual(rows[i - 1]!.d);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Database, QueryResult } from 'tauri-plugin-turso';
|
||||
import { Database, LoadOptions, QueryResult } from 'tauri-plugin-turso';
|
||||
import { DatabaseService, DatabaseExecResult, DatabaseRow, DatabaseOpts } from '@/types/database';
|
||||
|
||||
export class NativeDatabaseService implements DatabaseService {
|
||||
@@ -8,8 +8,19 @@ export class NativeDatabaseService implements DatabaseService {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
static async open(path: string, _opts?: DatabaseOpts): Promise<NativeDatabaseService> {
|
||||
const db = await Database.load(path);
|
||||
static async open(path: string, opts?: DatabaseOpts): Promise<NativeDatabaseService> {
|
||||
// Translate the cross-platform DatabaseOpts (from @readest/turso-database-common,
|
||||
// used by WASM bindings) to tauri-plugin-turso's LoadOptions. The two interfaces
|
||||
// have diverged: `experimental` is the same field name with compatible types
|
||||
// (literal union vs `string[]`); `encryption` shapes differ entirely (native
|
||||
// 'aes256cbc' + byte-array key vs WASM 'aes256gcm'/etc + hex key) and is not
|
||||
// wired in MVP — revisit alongside any Reedy.db encryption work. Skip the
|
||||
// translation when no relevant opts are set so existing callers preserve their
|
||||
// plain path-string call shape.
|
||||
const loadArg: string | LoadOptions = opts?.experimental?.length
|
||||
? { path, experimental: opts.experimental as string[] }
|
||||
: path;
|
||||
const db = await Database.load(loadArg);
|
||||
return new NativeDatabaseService(db);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user