diff --git a/apps/readest-app/src/__tests__/reedy/ReedyDb-memory.test.ts b/apps/readest-app/src/__tests__/reedy/ReedyDb-memory.test.ts new file mode 100644 index 00000000..6e6aca16 --- /dev/null +++ b/apps/readest-app/src/__tests__/reedy/ReedyDb-memory.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { DatabaseService } from '@/types/database'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import { ReedyDb } from '@/services/reedy/db/ReedyDb'; + +const DIM = 4; + +function unit(values: number[]): number[] { + const norm = Math.sqrt(values.reduce((s, v) => s + v * v, 0)) || 1; + return values.map((v) => v / norm); +} + +describe('ReedyDb · memory', () => { + let svc: DatabaseService; + let reedy: ReedyDb; + + beforeEach(async () => { + svc = await NodeDatabaseService.open(':memory:', { experimental: ['index_method'] }); + await migrate(svc, getMigrations('reedy')); + reedy = new ReedyDb(svc); + }); + + afterEach(async () => { + await svc.close(); + }); + + describe('migration', () => { + it('creates reedy_memory + idx_memory_scope', async () => { + const tables = await svc.select<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type='table' AND name='reedy_memory'", + ); + expect(tables).toHaveLength(1); + const indexes = await svc.select<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type='index' AND name='idx_memory_scope'", + ); + expect(indexes).toHaveLength(1); + }); + + it('does NOT create reedy_memory_embeddings at migration time (lazy)', async () => { + const tables = await svc.select<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type='table' AND name='reedy_memory_embeddings'", + ); + expect(tables).toHaveLength(0); + }); + }); + + describe('ensureMemoryEmbeddingsTable', () => { + it('creates the table on first call', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + const rows = await svc.select( + "SELECT name FROM sqlite_master WHERE name='reedy_memory_embeddings'", + ); + expect(rows).toHaveLength(1); + }); + + it('is idempotent at the same dim', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + await reedy.ensureMemoryEmbeddingsTable(DIM); + await reedy.ensureMemoryEmbeddingsTable(DIM); + const rows = await svc.select( + "SELECT name FROM sqlite_master WHERE name='reedy_memory_embeddings'", + ); + expect(rows).toHaveLength(1); + }); + + it('throws on dim mismatch', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + await expect(reedy.ensureMemoryEmbeddingsTable(DIM + 1)).rejects.toThrow(/dim/); + }); + }); + + describe('upsertMemory + getMemory', () => { + it('inserts a new memory row + returns the full record', async () => { + const row = await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'prefers-spoiler-free', + summary: 'User asked to avoid spoilers.', + sourceMessageId: 'msg-1', + }); + expect(row.id).toMatch(/^mem-/); + expect(row.summary).toBe('User asked to avoid spoilers.'); + expect(row.updatedAt).toBeGreaterThan(0); + const fetched = await reedy.getMemory('user', 'u1', 'prefers-spoiler-free'); + expect(fetched?.id).toBe(row.id); + }); + + it('upserting the same (scope, scopeKey, key) replaces the prior summary', async () => { + const first = await reedy.upsertMemory({ + scope: 'book', + scopeKey: 'bk1', + key: 'theme', + summary: 'first', + }); + await new Promise((r) => setTimeout(r, 2)); + const second = await reedy.upsertMemory({ + scope: 'book', + scopeKey: 'bk1', + key: 'theme', + summary: 'second', + }); + expect(second.id).toBe(first.id); + expect(second.summary).toBe('second'); + expect(second.updatedAt).toBeGreaterThanOrEqual(first.updatedAt); + }); + + it('stores an embedding when provided + upserts replace it', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + const row = await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'k', + summary: 'hello', + embedding: unit([1, 0, 0, 0]), + }); + const embRows = await svc.select<{ extracted: string }>( + `SELECT vector_extract(embedding) AS extracted FROM reedy_memory_embeddings WHERE memory_id = '${row.id}'`, + ); + expect(embRows).toHaveLength(1); + const parsed = JSON.parse(embRows[0]!.extracted) as number[]; + expect(parsed).toHaveLength(DIM); + + // Upsert with a different embedding replaces it. + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'k', + summary: 'hello again', + embedding: unit([0, 1, 0, 0]), + }); + const after = await svc.select<{ extracted: string }>( + `SELECT vector_extract(embedding) AS extracted FROM reedy_memory_embeddings WHERE memory_id = '${row.id}'`, + ); + const parsedAfter = JSON.parse(after[0]!.extracted) as number[]; + expect(Math.abs(parsedAfter[1]! - 1)).toBeLessThan(0.01); + }); + }); + + describe('listMemories', () => { + it('returns rows for the scope in recency-desc order', async () => { + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u1', key: 'a', summary: 'A' }); + await new Promise((r) => setTimeout(r, 2)); + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u1', key: 'b', summary: 'B' }); + await new Promise((r) => setTimeout(r, 2)); + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u1', key: 'c', summary: 'C' }); + const out = await reedy.listMemories('user', 'u1', 10); + expect(out.map((m) => m.key)).toEqual(['c', 'b', 'a']); + }); + + it('isolates by scope and scopeKey', async () => { + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u1', key: 'k', summary: 'U1' }); + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u2', key: 'k', summary: 'U2' }); + await reedy.upsertMemory({ scope: 'book', scopeKey: 'u1', key: 'k', summary: 'B1' }); + const out = await reedy.listMemories('user', 'u1', 10); + expect(out.map((m) => m.summary)).toEqual(['U1']); + }); + }); + + describe('deleteMemory', () => { + it('returns true when a row was deleted, false when nothing matched', async () => { + await reedy.upsertMemory({ scope: 'user', scopeKey: 'u1', key: 'k', summary: 'x' }); + expect(await reedy.deleteMemory('user', 'u1', 'k')).toBe(true); + expect(await reedy.deleteMemory('user', 'u1', 'k')).toBe(false); + }); + + it('cascade-deletes the embedding row', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + const row = await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'k', + summary: 'x', + embedding: unit([1, 0, 0, 0]), + }); + await reedy.deleteMemory('user', 'u1', 'k'); + const embs = await svc.select( + `SELECT memory_id FROM reedy_memory_embeddings WHERE memory_id = '${row.id}'`, + ); + expect(embs).toHaveLength(0); + }); + }); + + describe('searchMemories', () => { + beforeEach(async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'a', + summary: 'about cats', + embedding: unit([1, 0, 0, 0]), + }); + await new Promise((r) => setTimeout(r, 2)); + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'b', + summary: 'about dogs', + embedding: unit([0, 1, 0, 0]), + }); + await new Promise((r) => setTimeout(r, 2)); + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'c', + summary: 'about birds', + embedding: unit([0, 0, 1, 0]), + }); + }); + + it('ranks the vector-aligned memory first when query embedding points at it', async () => { + const out = await reedy.searchMemories({ + scope: 'user', + scopeKey: 'u1', + queryEmbedding: unit([0, 1, 0, 0]), + limit: 3, + }); + expect(out[0]!.key).toBe('b'); + expect(out[0]!.vectorDistance).toBeLessThan(0.01); + }); + + it('returns rows by recency when no queryEmbedding is provided', async () => { + const out = await reedy.searchMemories({ + scope: 'user', + scopeKey: 'u1', + limit: 3, + }); + // Most-recent first (insertion order: a, b, c so c is newest). + expect(out.map((m) => m.key)).toEqual(['c', 'b', 'a']); + for (const r of out) expect(r.vectorDistance).toBeNull(); + }); + + it('strictly isolates by scope_key — no other user shows up', async () => { + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u2', + key: 'x', + summary: 'other user about cats', + embedding: unit([1, 0, 0, 0]), + }); + const out = await reedy.searchMemories({ + scope: 'user', + scopeKey: 'u1', + queryEmbedding: unit([1, 0, 0, 0]), + limit: 5, + }); + for (const r of out) expect(r.scopeKey).toBe('u1'); + }); + }); + + describe('wipeAllData', () => { + it('clears memory + drops the memory embeddings table', async () => { + await reedy.ensureMemoryEmbeddingsTable(DIM); + await reedy.upsertMemory({ + scope: 'user', + scopeKey: 'u1', + key: 'k', + summary: 's', + embedding: unit([1, 0, 0, 0]), + }); + await reedy.wipeAllData(); + const rows = await svc.select('SELECT id FROM reedy_memory'); + expect(rows).toHaveLength(0); + const embTable = await svc.select<{ name: string }>( + "SELECT name FROM sqlite_master WHERE name='reedy_memory_embeddings'", + ); + expect(embTable).toHaveLength(0); + }); + }); +}); diff --git a/apps/readest-app/src/__tests__/reedy/memoryService.test.ts b/apps/readest-app/src/__tests__/reedy/memoryService.test.ts new file mode 100644 index 00000000..181df53f --- /dev/null +++ b/apps/readest-app/src/__tests__/reedy/memoryService.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { DatabaseService } from '@/types/database'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import { ReedyDb } from '@/services/reedy/db/ReedyDb'; +import { MemoryService } from '@/services/reedy/memory/MemoryService'; +import type { EmbeddingModel } from '@/services/reedy/models/EmbeddingModel'; + +const DIM = 4; + +function fakeEmbedding(text: string): number[] { + // Distinct per-text vectors: bucket the entire text content into one + // of four axes via a stable hash so semantically-similar fakes don't + // accidentally cluster (the char-position fake above clustered "about + // cats" / "about dogs" / "about birds" so the recency boost dominated + // the assertion). + const hash = Array.from(text).reduce((acc, c) => (acc * 31 + c.charCodeAt(0)) >>> 0, 0); + const axis = hash % 4; + const v = [0, 0, 0, 0]; + v[axis] = 1; + return v; +} + +function fakeModel(): EmbeddingModel { + return { + id: 'fake', + dim: DIM, + async embed(texts) { + return texts.map(fakeEmbedding); + }, + }; +} + +describe('MemoryService', () => { + let svc: DatabaseService; + let reedy: ReedyDb; + let memory: MemoryService; + + beforeEach(async () => { + svc = await NodeDatabaseService.open(':memory:', { experimental: ['index_method'] }); + await migrate(svc, getMigrations('reedy')); + reedy = new ReedyDb(svc); + memory = new MemoryService(reedy, fakeModel()); + }); + + afterEach(async () => { + await svc.close(); + }); + + it('write() embeds the summary + lazy-creates the embeddings table', async () => { + await memory.write({ + scope: 'user', + scopeKey: 'u1', + key: 'k', + summary: 'avoids spoilers', + }); + const tables = await svc.select( + "SELECT name FROM sqlite_master WHERE name='reedy_memory_embeddings'", + ); + expect(tables).toHaveLength(1); + const embs = await svc.select('SELECT memory_id FROM reedy_memory_embeddings'); + expect(embs).toHaveLength(1); + }); + + it('search(query) embeds the query and ranks vector-aligned memories first', async () => { + await memory.write({ scope: 'book', scopeKey: 'bk1', key: 'cats', summary: 'about cats' }); + await new Promise((r) => setTimeout(r, 2)); + await memory.write({ scope: 'book', scopeKey: 'bk1', key: 'dogs', summary: 'about dogs' }); + await new Promise((r) => setTimeout(r, 2)); + await memory.write({ scope: 'book', scopeKey: 'bk1', key: 'birds', summary: 'about birds' }); + + const out = await memory.search({ + scope: 'book', + scopeKey: 'bk1', + query: 'about cats', + limit: 3, + }); + expect(out[0]!.key).toBe('cats'); + }); + + it('search without query returns rows by recency', async () => { + await memory.write({ scope: 'user', scopeKey: 'u1', key: 'a', summary: 'A' }); + await new Promise((r) => setTimeout(r, 2)); + await memory.write({ scope: 'user', scopeKey: 'u1', key: 'b', summary: 'B' }); + const out = await memory.search({ scope: 'user', scopeKey: 'u1', limit: 5 }); + expect(out.map((m) => m.key)).toEqual(['b', 'a']); + }); + + it('write() without an embedding model still upserts the row', async () => { + const m = new MemoryService(reedy, null); + await m.write({ scope: 'user', scopeKey: 'u1', key: 'k', summary: 's' }); + const row = await m.get('user', 'u1', 'k'); + expect(row?.summary).toBe('s'); + // No embedding row since the model was null. + const tables = await svc.select( + "SELECT name FROM sqlite_master WHERE name='reedy_memory_embeddings'", + ); + expect(tables).toHaveLength(0); + }); + + it('delete() removes the memory + its embedding', async () => { + await memory.write({ scope: 'user', scopeKey: 'u1', key: 'k', summary: 's' }); + expect(await memory.delete('user', 'u1', 'k')).toBe(true); + expect(await memory.get('user', 'u1', 'k')).toBeNull(); + }); +}); diff --git a/apps/readest-app/src/__tests__/reedy/memoryTools.test.ts b/apps/readest-app/src/__tests__/reedy/memoryTools.test.ts new file mode 100644 index 00000000..7e20c95e --- /dev/null +++ b/apps/readest-app/src/__tests__/reedy/memoryTools.test.ts @@ -0,0 +1,181 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { ToolRegistry } from '@/services/reedy/tools/ToolRegistry'; +import type { ToolContext } from '@/services/reedy/tools/types'; +import { + createSearchBookMemoryTool, + createSearchSessionMemoryTool, + createSearchUserMemoryTool, + createWriteBookMemoryTool, + createWriteUserMemoryTool, +} from '@/services/reedy/tools/builtins'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { DatabaseService } from '@/types/database'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import { ReedyDb } from '@/services/reedy/db/ReedyDb'; +import { MemoryService } from '@/services/reedy/memory/MemoryService'; +import type { EmbeddingModel } from '@/services/reedy/models/EmbeddingModel'; + +const DIM = 4; +function unit(values: number[]): number[] { + const norm = Math.sqrt(values.reduce((s, v) => s + v * v, 0)) || 1; + return values.map((v) => v / norm); +} +function fakeModel(): EmbeddingModel { + return { + id: 'fake', + dim: DIM, + async embed(texts) { + // Map deterministic by text length so identical strings → same vec. + return texts.map((t) => unit([t.length, 1, 0, 0])); + }, + }; +} +function ctxFor(): ToolContext { + const controller = new AbortController(); + return { + bookHash: 'bk1', + sessionId: 's1', + assistantMessageId: 'm1', + signal: controller.signal, + requestPermission: vi.fn(async () => true), + }; +} + +describe('memory tools', () => { + let svc: DatabaseService; + let memory: MemoryService; + + beforeEach(async () => { + svc = await NodeDatabaseService.open(':memory:', { experimental: ['index_method'] }); + await migrate(svc, getMigrations('reedy')); + memory = new MemoryService(new ReedyDb(svc), fakeModel()); + }); + + afterEach(async () => { + await svc.close(); + }); + + describe('writeUserMemory', () => { + it('persists a user-scoped memory + returns the key', async () => { + const reg = new ToolRegistry(); + reg.register(createWriteUserMemoryTool({ service: memory, scopeKey: () => 'u1' })); + const out = (await reg.invoke( + 'writeUserMemory', + { key: 'taste:scifi', summary: 'User loves space opera.' }, + ctxFor(), + )) as { ok: true; key: string }; + expect(out).toEqual({ ok: true, key: 'taste:scifi' }); + const row = await memory.get('user', 'u1', 'taste:scifi'); + expect(row?.summary).toBe('User loves space opera.'); + }); + + it('rejects keys that match the injection blocklist', async () => { + const reg = new ToolRegistry(); + reg.register(createWriteUserMemoryTool({ service: memory, scopeKey: () => 'u1' })); + for (const badKey of ['system', 'system:role', 'policy', 'INJECTION', 'override-fix']) { + await expect( + reg.invoke('writeUserMemory', { key: badKey, summary: 'x' }, ctxFor()), + ).rejects.toMatchObject({ kind: 'tool_invalid_args' }); + } + }); + + it('rejects oversized summaries via the Zod schema', async () => { + const reg = new ToolRegistry(); + reg.register(createWriteUserMemoryTool({ service: memory, scopeKey: () => 'u1' })); + await expect( + reg.invoke('writeUserMemory', { key: 'k', summary: 'x'.repeat(2_001) }, ctxFor()), + ).rejects.toMatchObject({ kind: 'tool_invalid_args' }); + }); + + it('rejects keys with disallowed characters', async () => { + const reg = new ToolRegistry(); + reg.register(createWriteUserMemoryTool({ service: memory, scopeKey: () => 'u1' })); + await expect( + reg.invoke('writeUserMemory', { key: 'has spaces', summary: 'x' }, ctxFor()), + ).rejects.toMatchObject({ kind: 'tool_invalid_args' }); + }); + + it('passes sourceMessageId through to the service when configured', async () => { + const reg = new ToolRegistry(); + reg.register( + createWriteUserMemoryTool({ + service: memory, + scopeKey: () => 'u1', + sourceMessageId: () => 'msg-42', + }), + ); + await reg.invoke('writeUserMemory', { key: 'k', summary: 's' }, ctxFor()); + const row = await memory.get('user', 'u1', 'k'); + expect(row?.sourceMessageId).toBe('msg-42'); + }); + }); + + describe('searchUserMemory / searchBookMemory', () => { + it('returns memories scoped to the active user', async () => { + await memory.write({ scope: 'user', scopeKey: 'u1', key: 'a', summary: 'A' }); + await memory.write({ scope: 'user', scopeKey: 'u2', key: 'a', summary: 'OTHER' }); + const reg = new ToolRegistry(); + reg.register(createSearchUserMemoryTool({ service: memory, scopeKey: () => 'u1' })); + const out = (await reg.invoke('searchUserMemory', { limit: 5 }, ctxFor())) as { + memories: Array<{ summary: string }>; + }; + expect(out.memories.map((m) => m.summary)).toEqual(['A']); + }); + + it('book-scoped tool returns book-scope rows only', async () => { + await memory.write({ scope: 'book', scopeKey: 'bk1', key: 'theme', summary: 'identity' }); + await memory.write({ + scope: 'user', + scopeKey: 'bk1', + key: 'theme', + summary: 'should not appear', + }); + const reg = new ToolRegistry(); + reg.register(createSearchBookMemoryTool({ service: memory, scopeKey: () => 'bk1' })); + const out = (await reg.invoke('searchBookMemory', { limit: 5 }, ctxFor())) as { + memories: Array<{ summary: string }>; + }; + expect(out.memories.map((m) => m.summary)).toEqual(['identity']); + }); + }); + + describe('searchSessionMemory', () => { + it('is search-only and returns session-scope rows', async () => { + await memory.write({ scope: 'session', scopeKey: 's1', key: 'k', summary: 'session note' }); + const reg = new ToolRegistry(); + reg.register(createSearchSessionMemoryTool({ service: memory, scopeKey: () => 's1' })); + const out = (await reg.invoke('searchSessionMemory', { limit: 5 }, ctxFor())) as { + memories: Array<{ summary: string }>; + }; + expect(out.memories[0]!.summary).toBe('session note'); + }); + }); + + describe('writeBookMemory parallel serialization', () => { + it('serializes concurrent writes (parallelSafe=false) so the same key doesn’t race', async () => { + let active = 0; + let maxActive = 0; + // Wrap memory.write to count concurrency. + const original = memory.write.bind(memory); + memory.write = async (args) => { + active++; + maxActive = Math.max(maxActive, active); + try { + return await original(args); + } finally { + active--; + } + }; + const reg = new ToolRegistry(); + reg.register(createWriteBookMemoryTool({ service: memory, scopeKey: () => 'bk1' })); + const ctx = ctxFor(); + await Promise.all([ + reg.invoke('writeBookMemory', { key: 'a', summary: 'one' }, ctx), + reg.invoke('writeBookMemory', { key: 'b', summary: 'two' }, ctx), + reg.invoke('writeBookMemory', { key: 'c', summary: 'three' }, ctx), + ]); + expect(maxActive).toBe(1); + }); + }); +}); diff --git a/apps/readest-app/src/services/database/migrations/index.ts b/apps/readest-app/src/services/database/migrations/index.ts index e39f6e6e..51fb5171 100644 --- a/apps/readest-app/src/services/database/migrations/index.ts +++ b/apps/readest-app/src/services/database/migrations/index.ts @@ -96,6 +96,31 @@ const migrations: Record = { CREATE INDEX IF NOT EXISTS idx_metrics_session ON reedy_metrics (session_id, ts DESC); `, }, + { + // Memory store for the agent runtime (Phase 3.1). One table, three + // scopes: user / book / session. UNIQUE(scope, scope_key, key) + // gives us upsert semantics — writing the same key twice replaces + // the prior summary. Embeddings live in a sibling table created + // lazily by MemoryService (same single-model-lock pattern as + // reedy_book_chunk_embeddings) so the vector32 dim matches the + // active embedding model. + name: '2026052603_reedy_memory', + sql: ` + CREATE TABLE IF NOT EXISTS reedy_memory ( + id TEXT PRIMARY KEY, + scope TEXT NOT NULL, + scope_key TEXT NOT NULL, + key TEXT NOT NULL, + summary TEXT NOT NULL, + source_message_id TEXT, + updated_at INTEGER NOT NULL, + UNIQUE(scope, scope_key, key) + ); + + CREATE INDEX IF NOT EXISTS idx_memory_scope + ON reedy_memory (scope, scope_key, updated_at DESC); + `, + }, ], }; diff --git a/apps/readest-app/src/services/reedy/db/ReedyDb.ts b/apps/readest-app/src/services/reedy/db/ReedyDb.ts index ef3d0d43..d431a245 100644 --- a/apps/readest-app/src/services/reedy/db/ReedyDb.ts +++ b/apps/readest-app/src/services/reedy/db/ReedyDb.ts @@ -1,5 +1,16 @@ import type { DatabaseService } from '@/types/database'; -import type { BookMeta, ChunkRow, EmbeddingRow, IndexingStatus, ScoredChunk } from './types'; +import type { + BookMeta, + ChunkRow, + EmbeddingRow, + IndexingStatus, + MemoryRow, + MemoryScope, + MemorySearchArgs, + MemoryWriteArgs, + ScoredChunk, + ScoredMemoryRow, +} from './types'; /** * Typed wrapper around a Turso DatabaseService opened against reedy.db. @@ -246,13 +257,213 @@ export class ReedyDb { async wipeAllData(): Promise { await this.enqueue(async () => { await this.db.execute('DELETE FROM reedy_book_meta'); - // Drop embeddings table so a future ensureEmbeddingsTable(newDim) is free - // to recreate it with a different vector32 width. + // Drop embeddings tables so a future ensure*EmbeddingsTable(newDim) + // is free to recreate them with a different vector32 width. await this.db.execute('DROP TABLE IF EXISTS reedy_book_chunk_embeddings'); + await this.db.execute('DROP TABLE IF EXISTS reedy_memory_embeddings'); await this.db.execute('DELETE FROM reedy_book_chunks'); + await this.db.execute('DELETE FROM reedy_memory'); }); } + // --------------------------------------------------------------------------- + // memory (Phase 3.1) + // --------------------------------------------------------------------------- + + /** + * Lazy-create reedy_memory_embeddings the same way the chunk embeddings + * table is created — vector32() lives on a sibling row keyed on + * memory_id, with ON DELETE CASCADE so deleting a memory row also drops + * its vector. Same dim-mismatch contract as ensureEmbeddingsTable. + */ + async ensureMemoryEmbeddingsTable(dim: number): Promise { + if (!Number.isInteger(dim) || dim <= 0) { + throw new Error(`ensureMemoryEmbeddingsTable: dim must be a positive integer, got ${dim}`); + } + await this.enqueue(async () => { + const existing = await this.db.select<{ sql: string | null }>( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='reedy_memory_embeddings'", + ); + if (existing.length === 0) { + await this.db.batch([ + `CREATE TABLE reedy_memory_embeddings ( + memory_id TEXT PRIMARY KEY REFERENCES reedy_memory(id) ON DELETE CASCADE, + embedding vector32(${dim}) + )`, + ]); + return; + } + const m = existing[0]!.sql?.match(/vector32\s*\(\s*(\d+)\s*\)/i); + const existingDim = m ? parseInt(m[1]!, 10) : NaN; + if (existingDim !== dim) { + throw new Error( + `ensureMemoryEmbeddingsTable: dim mismatch — existing table is vector32(${existingDim}), requested vector32(${dim}). ` + + `Switching embedding models requires wipeAllData() first.`, + ); + } + }); + } + + /** + * Upsert one memory row by (scope, scope_key, key). Re-writing the same + * key replaces the prior summary and bumps updated_at. When + * `args.embedding` is provided, the matching reedy_memory_embeddings row + * is also upserted — caller must have called ensureMemoryEmbeddingsTable + * with the same dim already. + */ + async upsertMemory(args: MemoryWriteArgs): Promise { + const id = await this.upsertMemoryRow(args); + if (args.embedding != null) { + if (args.embedding.length === 0) { + throw new Error('upsertMemory: empty embedding array'); + } + await this.upsertMemoryEmbedding(id, args.embedding); + } + const row = await this.getMemoryById(id); + if (!row) throw new Error(`upsertMemory: row vanished after write (id=${id})`); + return row; + } + + private async upsertMemoryRow(args: MemoryWriteArgs): Promise { + const now = Date.now(); + return this.enqueue(async () => { + const existing = await this.db.select<{ id: string }>( + 'SELECT id FROM reedy_memory WHERE scope = ? AND scope_key = ? AND key = ?', + [args.scope, args.scopeKey, args.key], + ); + if (existing[0]) { + const id = existing[0].id; + await this.db.execute( + `UPDATE reedy_memory SET summary = ?, source_message_id = ?, updated_at = ? + WHERE id = ?`, + [args.summary, args.sourceMessageId ?? null, now, id], + ); + return id; + } + const id = randomMemoryId(); + await this.db.execute( + `INSERT INTO reedy_memory (id, scope, scope_key, key, summary, source_message_id, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [id, args.scope, args.scopeKey, args.key, args.summary, args.sourceMessageId ?? null, now], + ); + return id; + }); + } + + private async upsertMemoryEmbedding(memoryId: string, embedding: number[]): Promise { + await this.enqueue(() => + this.db.batch([ + `INSERT INTO reedy_memory_embeddings (memory_id, embedding) + VALUES (${sqlQuote(memoryId)}, vector32(${sqlQuote(serializeVector(embedding))})) + ON CONFLICT(memory_id) DO UPDATE SET embedding = excluded.embedding`, + ]), + ); + } + + async getMemory(scope: MemoryScope, scopeKey: string, key: string): Promise { + const rows = await this.db.select( + 'SELECT * FROM reedy_memory WHERE scope = ? AND scope_key = ? AND key = ?', + [scope, scopeKey, key], + ); + return rows[0] ? toMemoryRow(rows[0]) : null; + } + + async deleteMemory(scope: MemoryScope, scopeKey: string, key: string): Promise { + return this.enqueue(async () => { + const matched = await this.db.select<{ id: string }>( + 'SELECT id FROM reedy_memory WHERE scope = ? AND scope_key = ? AND key = ?', + [scope, scopeKey, key], + ); + if (matched.length === 0) return false; + const id = matched[0]!.id; + // We can't rely on PRAGMA foreign_keys = ON in callers' connections, + // so explicitly drop the embedding row first when the table exists. + const hasEmbTable = await this.db.select<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type='table' AND name='reedy_memory_embeddings'", + ); + if (hasEmbTable.length > 0) { + await this.db.execute('DELETE FROM reedy_memory_embeddings WHERE memory_id = ?', [id]); + } + await this.db.execute('DELETE FROM reedy_memory WHERE id = ?', [id]); + return true; + }); + } + + async listMemories(scope: MemoryScope, scopeKey: string, limit: number): Promise { + if (limit <= 0) return []; + const rows = await this.db.select( + `SELECT * FROM reedy_memory + WHERE scope = ? AND scope_key = ? + ORDER BY updated_at DESC + LIMIT ?`, + [scope, scopeKey, limit], + ); + return rows.map(toMemoryRow); + } + + /** + * Hybrid memory search — vector cosine + recency. When the caller + * provides a queryEmbedding, score = vectorDistance + recencyWeight * + * normalizedAge; otherwise rows are returned by recency only. Both + * components are positive distance-like terms so a lower score wins. + */ + async searchMemories(args: MemorySearchArgs): Promise { + const limit = Math.max(0, args.limit); + if (limit === 0) return []; + const recencyWeight = args.recencyWeight ?? 0.1; + const now = Date.now(); + + if (args.queryEmbedding && args.queryEmbedding.length > 0) { + // Pull a slightly wider pool than `limit` so the JS-side fusion has + // candidates to work with even if the top-K-by-distance and + // top-K-by-recency disagree. + const fetchK = Math.max(limit, limit * 3); + const rows = await this.db.select( + `SELECT m.*, vector_distance_cos(e.embedding, vector32(?)) AS vector_distance + FROM reedy_memory m + JOIN reedy_memory_embeddings e ON e.memory_id = m.id + WHERE m.scope = ? AND m.scope_key = ? + ORDER BY vector_distance ASC + LIMIT ?`, + [serializeVector(args.queryEmbedding), args.scope, args.scopeKey, fetchK], + ); + const maxAge = Math.max(1, ...rows.map((r) => Math.max(0, now - r.updated_at))); + const scored = rows.map((r) => { + const distance = Number.isFinite(r.vector_distance) ? r.vector_distance : 1; + const ageNorm = (now - r.updated_at) / maxAge; // 0 = newest, 1 = oldest in pool + return { + ...toMemoryRow(r), + score: distance + recencyWeight * ageNorm, + vectorDistance: distance, + } as ScoredMemoryRow; + }); + return scored.sort((a, b) => a.score - b.score).slice(0, limit); + } + + const rows = await this.db.select( + `SELECT * FROM reedy_memory + WHERE scope = ? AND scope_key = ? + ORDER BY updated_at DESC + LIMIT ?`, + [args.scope, args.scopeKey, limit], + ); + return rows.map( + (r): ScoredMemoryRow => ({ + ...toMemoryRow(r), + // Recency-only score: 0 for the newest, growing with age. + score: Math.max(0, (now - r.updated_at) / 1000), + vectorDistance: null, + }), + ); + } + + private async getMemoryById(id: string): Promise { + const rows = await this.db.select('SELECT * FROM reedy_memory WHERE id = ?', [ + id, + ]); + return rows[0] ? toMemoryRow(rows[0]) : null; + } + // --------------------------------------------------------------------------- // hybrid search (brute-force cosine + Tantivy FTS + RRF) // --------------------------------------------------------------------------- @@ -403,3 +614,35 @@ function sqlQuote(s: string): string { function sqlQuoteNullable(s: string | null): string { return s === null ? 'NULL' : sqlQuote(s); } + +// --------------------------------------------------------------------------- +// memory helpers +// --------------------------------------------------------------------------- + +interface MemoryRowSql { + id: string; + scope: string; + scope_key: string; + key: string; + summary: string; + source_message_id: string | null; + updated_at: number; + [key: string]: unknown; +} + +function toMemoryRow(row: MemoryRowSql): MemoryRow { + return { + id: row.id, + scope: row.scope as MemoryScope, + scopeKey: row.scope_key, + key: row.key, + summary: row.summary, + sourceMessageId: row.source_message_id, + updatedAt: row.updated_at, + }; +} + +function randomMemoryId(): string { + if (typeof crypto !== 'undefined' && 'randomUUID' in crypto) return `mem-${crypto.randomUUID()}`; + return `mem-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} diff --git a/apps/readest-app/src/services/reedy/db/types.ts b/apps/readest-app/src/services/reedy/db/types.ts index 767a5d00..04b795ee 100644 --- a/apps/readest-app/src/services/reedy/db/types.ts +++ b/apps/readest-app/src/services/reedy/db/types.ts @@ -45,3 +45,49 @@ export interface ScoredChunk extends ChunkRow { vectorRank: number | null; ftsRank: number | null; } + +// --------------------------------------------------------------------------- +// Memory (Phase 3.1) +// --------------------------------------------------------------------------- + +export type MemoryScope = 'user' | 'book' | 'session'; + +export interface MemoryRow { + id: string; + scope: MemoryScope; + scopeKey: string; + key: string; + summary: string; + sourceMessageId: string | null; + updatedAt: number; +} + +export interface MemoryWriteArgs { + scope: MemoryScope; + scopeKey: string; + key: string; + summary: string; + sourceMessageId?: string | null; + /** Optional embedding for the summary — caller embeds + supplies. */ + embedding?: number[] | null; +} + +export interface MemorySearchArgs { + scope: MemoryScope; + scopeKey: string; + /** When provided, ranks via vector cosine + recency boost. When omitted, recency only. */ + queryEmbedding?: number[]; + limit: number; + /** + * Weight for the recency component when both vector + recency are + * active. 0 = pure vector, 1 = pure recency. Default 0.1. + */ + recencyWeight?: number; +} + +export interface ScoredMemoryRow extends MemoryRow { + /** Fused score for the search; lower = more relevant (distance-based). */ + score: number; + /** Distance from queryEmbedding when vector search ran. null otherwise. */ + vectorDistance: number | null; +} diff --git a/apps/readest-app/src/services/reedy/memory/MemoryService.ts b/apps/readest-app/src/services/reedy/memory/MemoryService.ts new file mode 100644 index 00000000..cdabbaa7 --- /dev/null +++ b/apps/readest-app/src/services/reedy/memory/MemoryService.ts @@ -0,0 +1,93 @@ +import type { ReedyDb } from '../db/ReedyDb'; +import type { MemoryRow, MemoryScope, ScoredMemoryRow } from '../db/types'; +import type { EmbeddingModel } from '../models/EmbeddingModel'; + +/** + * Phase 3.1 — one MemoryService class wrapping ReedyDb's memory primitives + * for a fixed scope. The plan's original separate UserMemory / + * BookMemory / SessionMemory services collapse to one type: the only + * thing that differs is the scope label, and tests + tools are cleaner + * when we don't have three near-identical classes. + * + * Reads embed the query string through the active embedding model and + * call the underlying searchMemories(); writes embed the summary so the + * row participates in subsequent semantic search. + * + * Session memory is search-only by convention (the plan's §3.1) — full + * transcript already lives in the messages table, so explicit + * session-scope writes are redundant. The class still allows write() at + * any scope; callers (tools) enforce the convention. + */ +export class MemoryService { + constructor( + private readonly reedy: ReedyDb, + private readonly model: EmbeddingModel | null, + ) {} + + async write(args: { + scope: MemoryScope; + scopeKey: string; + key: string; + summary: string; + sourceMessageId?: string; + }): Promise { + if (this.model) { + // Embed first so we can lazy-create the embeddings table at the + // right dim before we issue the insert. + const [embedding] = await this.model.embed([args.summary]); + await this.reedy.ensureMemoryEmbeddingsTable(this.model.dim); + return this.reedy.upsertMemory({ + scope: args.scope, + scopeKey: args.scopeKey, + key: args.key, + summary: args.summary, + sourceMessageId: args.sourceMessageId, + embedding, + }); + } + return this.reedy.upsertMemory({ + scope: args.scope, + scopeKey: args.scopeKey, + key: args.key, + summary: args.summary, + sourceMessageId: args.sourceMessageId, + }); + } + + async search(args: { + scope: MemoryScope; + scopeKey: string; + query?: string; + limit: number; + recencyWeight?: number; + }): Promise { + if (args.query && args.query.trim().length > 0 && this.model) { + const [queryEmbedding] = await this.model.embed([args.query]); + return this.reedy.searchMemories({ + scope: args.scope, + scopeKey: args.scopeKey, + queryEmbedding, + limit: args.limit, + recencyWeight: args.recencyWeight, + }); + } + return this.reedy.searchMemories({ + scope: args.scope, + scopeKey: args.scopeKey, + limit: args.limit, + recencyWeight: args.recencyWeight, + }); + } + + list(scope: MemoryScope, scopeKey: string, limit: number): Promise { + return this.reedy.listMemories(scope, scopeKey, limit); + } + + get(scope: MemoryScope, scopeKey: string, key: string): Promise { + return this.reedy.getMemory(scope, scopeKey, key); + } + + delete(scope: MemoryScope, scopeKey: string, key: string): Promise { + return this.reedy.deleteMemory(scope, scopeKey, key); + } +} diff --git a/apps/readest-app/src/services/reedy/tools/builtins/index.ts b/apps/readest-app/src/services/reedy/tools/builtins/index.ts index 4b3f2ccc..2cac09a0 100644 --- a/apps/readest-app/src/services/reedy/tools/builtins/index.ts +++ b/apps/readest-app/src/services/reedy/tools/builtins/index.ts @@ -1,8 +1,8 @@ /** - * Phase 2.4 built-in tools — each factory takes its dependencies as an - * argument so callers (AgentRuntime, tests) wire whatever read/write/ - * navigate surfaces they have. Memory tools (families 3, 4, 5) ship after - * Phase 3. + * Built-in tools the agent runtime registers. Each factory takes its + * dependencies as an argument so callers (AgentRuntime, tests) wire + * whatever read/write/navigate surfaces they have. Memory families ship + * alongside Phase 3.1's MemoryService. */ export { createGetReadingContextTool } from './getReadingContext'; export { createGetSelectionTool } from './getSelection'; @@ -16,6 +16,19 @@ export { createCreateHighlightTool } from './createHighlight'; export type { CreateHighlightResult } from './createHighlight'; export { createCreateNoteTool } from './createNote'; export type { CreateNoteResult } from './createNote'; +export { + createSearchUserMemoryTool, + createWriteUserMemoryTool, + createSearchBookMemoryTool, + createWriteBookMemoryTool, + createSearchSessionMemoryTool, +} from './memoryTools'; +export type { + SearchMemoryResult, + SearchMemoryToolDeps, + WriteMemoryResult, + WriteMemoryToolDeps, +} from './memoryTools'; export type { AnnotationServices, CitationData, diff --git a/apps/readest-app/src/services/reedy/tools/builtins/memoryTools.ts b/apps/readest-app/src/services/reedy/tools/builtins/memoryTools.ts new file mode 100644 index 00000000..1c281955 --- /dev/null +++ b/apps/readest-app/src/services/reedy/tools/builtins/memoryTools.ts @@ -0,0 +1,178 @@ +import { z } from 'zod'; +import type { ReedyTool } from '../types'; +import type { MemoryService } from '../../memory/MemoryService'; +import type { MemoryScope } from '../../db/types'; + +/** + * Memory tools (Phase 2.4 families 3, 4, 5 — shipped now that Phase 3.1 + * has MemoryService). Five tools: + * + * - searchUserMemory (user scope, read) + * - writeUserMemory (user scope, read — internal scratchpad, not a + * navigate/write operation; key is allowlisted against injection) + * - searchBookMemory (book scope, read) + * - writeBookMemory (book scope, read; allowlisted key) + * - searchSessionMemory (session scope, read; no write — sessions + * re-derive from the message log per plan §3.1) + * + * `permission: 'read'` for writes because memory writes don't mutate + * anything external — they're the agent's own scratchpad. The + * /system|policy|prompt|injection|override/i key blocklist (plan D8) + * keeps the model from sneaking new policy into the system prompt via + * memory. + */ + +const MEMORY_KEY_BLOCKLIST = /system|policy|prompt|injection|override/i; +const MEMORY_KEY_PATTERN = /^[a-z0-9][a-z0-9_\-:.]{0,127}$/i; + +const writeInputSchema = z.object({ + key: z + .string() + .min(1) + .max(128) + .regex(MEMORY_KEY_PATTERN, 'key must match [a-zA-Z0-9_\\-:.]') + .refine((k) => !MEMORY_KEY_BLOCKLIST.test(k), { + message: 'key matches the policy-injection blocklist', + }), + summary: z.string().min(1).max(2_000), +}); + +const searchInputSchema = z.object({ + /** When omitted the tool returns the most recent N memories. */ + query: z.string().min(1).max(500).optional(), + limit: z.number().int().min(1).max(20).default(5), +}); + +export interface SearchMemoryResult { + memories: Array<{ + key: string; + summary: string; + updatedAt: number; + score: number; + }>; +} + +export interface WriteMemoryResult { + ok: true; + key: string; +} + +export interface WriteMemoryToolDeps { + service: MemoryService; + /** Resolves the scope_key for this turn (userId / bookHash / sessionId). */ + scopeKey: () => string; + /** Optional source message id wired up by the runtime. */ + sourceMessageId?: () => string | undefined; +} + +export interface SearchMemoryToolDeps { + service: MemoryService; + scopeKey: () => string; +} + +function createSearchTool( + name: string, + description: string, + scope: MemoryScope, + deps: SearchMemoryToolDeps, +): ReedyTool, SearchMemoryResult> { + return { + name, + description, + permission: 'read', + parallelSafe: true, + inputSchema: searchInputSchema, + async run(args) { + const parsed = searchInputSchema.parse(args); + const memories = await deps.service.search({ + scope, + scopeKey: deps.scopeKey(), + query: parsed.query, + limit: parsed.limit, + }); + return { + memories: memories.map((m) => ({ + key: m.key, + summary: m.summary, + updatedAt: m.updatedAt, + score: m.score, + })), + }; + }, + }; +} + +function createWriteTool( + name: string, + description: string, + scope: MemoryScope, + deps: WriteMemoryToolDeps, +): ReedyTool, WriteMemoryResult> { + return { + name, + description, + permission: 'read', // writes touch only the agent's own scratchpad + parallelSafe: false, // memory writes serialize per-scope to avoid + // racing on the same key inside one turn + inputSchema: writeInputSchema, + async run(args) { + await deps.service.write({ + scope, + scopeKey: deps.scopeKey(), + key: args.key, + summary: args.summary, + sourceMessageId: deps.sourceMessageId?.(), + }); + return { ok: true, key: args.key }; + }, + }; +} + +// --------------------------------------------------------------------------- +// Tool factories +// --------------------------------------------------------------------------- + +export function createSearchUserMemoryTool(deps: SearchMemoryToolDeps) { + return createSearchTool( + 'searchUserMemory', + "Search the agent's stable memory about THIS USER (preferences, taste, what topics they care about). Returns the top-K matches by semantic + recency hybrid. Omit `query` to list the most recent.", + 'user', + deps, + ); +} + +export function createWriteUserMemoryTool(deps: WriteMemoryToolDeps) { + return createWriteTool( + 'writeUserMemory', + "Save a stable fact about THIS USER (preferences, taste, prior conversations). Re-writing the same `key` replaces the prior summary. Don't store policy / system instructions — those keys are blocked.", + 'user', + deps, + ); +} + +export function createSearchBookMemoryTool(deps: SearchMemoryToolDeps) { + return createSearchTool( + 'searchBookMemory', + "Search the agent's memory about THIS BOOK (character arcs, themes, prior summaries). Returns the top-K matches. Omit `query` to list the most recent.", + 'book', + deps, + ); +} + +export function createWriteBookMemoryTool(deps: WriteMemoryToolDeps) { + return createWriteTool( + 'writeBookMemory', + "Save a stable fact about THIS BOOK (character notes, themes, plot summaries). Re-writing the same `key` replaces the prior summary. Don't store policy / system instructions — those keys are blocked.", + 'book', + deps, + ); +} + +export function createSearchSessionMemoryTool(deps: SearchMemoryToolDeps) { + return createSearchTool( + 'searchSessionMemory', + "Search the agent's memory specific to THIS CONVERSATION SESSION. Read-only — the session's full transcript already lives in the message log.", + 'session', + deps, + ); +}