forked from akai/readest
35b02c4efc
Supersedes #3156. Adds a reading-statistics system whose canonical data model is KOReader's own (book + page_stat_data), so stats round-trip losslessly between Readest and KOReader. - Storage: a cross-platform Turso statistics.db in KOReader's exact schema (web/Workers, desktop, iOS, Android) — replacing #3156's Node-only better-sqlite3 + statistics.json. - Tracking: per-page reading events (time-on-page, idle-capped) flushed on page-change/idle/hide/close — the KOReader model — not session aggregates. - Sync: legacy /api/sync extended with a stats type backed by self-contained Supabase tables (stat_books, stat_pages); union/longer-duration-wins merge keyed on book_hash. apps/readest.koplugin syncs through the same endpoint. - Scale & robustness: per-tab singleton connection (avoids OPFS lock conflicts) + explicit WAL checkpoint; transactional bulk apply; chunked resumable push; client-driven paged pull with trailing-ms completion; paginated/scoped server merge. Verified: 5668 unit tests, 155 koplugin busted tests, biome+tsgo + luacheck all green; web OPFS DB verified live. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
221 lines
8.3 KiB
TypeScript
221 lines
8.3 KiB
TypeScript
import { MigrationEntry, SchemaType } from '../migrate';
|
|
|
|
/**
|
|
* Migration definitions for each schema type.
|
|
*
|
|
* To add a new migration:
|
|
* 1. Append a new entry to the appropriate schema array below.
|
|
* 2. Use a date-based name: YYYYMMDDNN (NN = sequence within the day).
|
|
* 3. Never reorder or remove existing entries.
|
|
*
|
|
* To add a new schema type:
|
|
* 1. Add the type to SchemaType in migrate.ts.
|
|
* 2. Add a new key here with its migration array.
|
|
*/
|
|
const migrations: Record<SchemaType, MigrationEntry[]> = {
|
|
opds: [
|
|
{
|
|
name: '2026052701_opds_source_mappings',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS opds_source_mappings (
|
|
catalog_id TEXT NOT NULL,
|
|
source_url TEXT NOT NULL,
|
|
book_hash TEXT NOT NULL,
|
|
PRIMARY KEY (catalog_id, source_url)
|
|
);
|
|
`,
|
|
},
|
|
],
|
|
'hardcover-sync': [
|
|
{
|
|
name: '2026032901_hardcover_note_mappings',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS hardcover_note_mappings (
|
|
book_hash TEXT NOT NULL,
|
|
note_id TEXT NOT NULL,
|
|
hardcover_journal_id INTEGER NOT NULL,
|
|
payload_hash TEXT NOT NULL,
|
|
synced_at INTEGER NOT NULL,
|
|
PRIMARY KEY (book_hash, note_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_hardcover_note_mappings_synced_at
|
|
ON hardcover_note_mappings (synced_at);
|
|
`,
|
|
},
|
|
],
|
|
// The embeddings table is created lazily by BookIndexer because its
|
|
// vector32(<dim>) column needs the active embedding model's dim, which
|
|
// isn't known at migration time. Tantivy FTS lives on the chunks.text
|
|
// column directly (no virtual table). Writers MUST DELETE+INSERT chunk
|
|
// rows rather than UPDATE — Tantivy 0.25→0.26 has a known WASM-only
|
|
// UPDATE regression (see fts-tests.ts:306 FIXME). MVP indexing is
|
|
// write-once per book so this is naturally satisfied.
|
|
statistics: [
|
|
{
|
|
name: '2026061501_statistics_koreader_schema',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS book (
|
|
id integer PRIMARY KEY autoincrement,
|
|
title text, authors text, notes integer, last_open integer,
|
|
highlights integer, pages integer, series text, language text,
|
|
md5 text, total_read_time integer, total_read_pages integer
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
|
|
|
|
CREATE TABLE IF NOT EXISTS page_stat_data (
|
|
id_book integer,
|
|
page integer NOT NULL DEFAULT 0,
|
|
start_time integer NOT NULL DEFAULT 0,
|
|
duration integer NOT NULL DEFAULT 0,
|
|
total_pages integer NOT NULL DEFAULT 0,
|
|
UNIQUE (id_book, page, start_time)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time);
|
|
|
|
CREATE TABLE IF NOT EXISTS numbers (number INTEGER PRIMARY KEY);
|
|
|
|
INSERT OR IGNORE INTO numbers(number)
|
|
SELECT h.n * 100 + t.n * 10 + o.n + 1
|
|
FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) o,
|
|
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t,
|
|
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) h;
|
|
|
|
CREATE VIEW IF NOT EXISTS page_stat AS
|
|
SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration
|
|
FROM (
|
|
SELECT id_book, page, total_pages, pages, start_time, duration,
|
|
((page - 1) * pages) / total_pages + 1 AS first_page,
|
|
max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page,
|
|
idx
|
|
FROM page_stat_data
|
|
JOIN book ON book.id = id_book
|
|
JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS readest_page_ext (
|
|
book_hash text NOT NULL, page integer NOT NULL, start_time integer NOT NULL,
|
|
ext text, PRIMARY KEY (book_hash, page, start_time)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS readest_book_ext (
|
|
book_hash text PRIMARY KEY, ext text
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS readest_stat_sync_state (
|
|
key text PRIMARY KEY, value integer NOT NULL DEFAULT 0
|
|
);
|
|
`,
|
|
},
|
|
],
|
|
reedy: [
|
|
{
|
|
name: '2026052601_reedy_init',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS reedy_book_meta (
|
|
book_hash TEXT PRIMARY KEY,
|
|
indexing_status TEXT NOT NULL,
|
|
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
embedding_model TEXT NOT NULL,
|
|
embedding_dim INTEGER NOT NULL,
|
|
indexed_at INTEGER,
|
|
error TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS reedy_book_chunks (
|
|
id TEXT PRIMARY KEY,
|
|
book_hash TEXT NOT NULL,
|
|
section_index INTEGER NOT NULL,
|
|
chapter_title TEXT,
|
|
start_cfi TEXT NOT NULL,
|
|
end_cfi TEXT NOT NULL,
|
|
position_index INTEGER NOT NULL,
|
|
text TEXT NOT NULL,
|
|
token_count INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_chunks_book_position
|
|
ON reedy_book_chunks (book_hash, position_index);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_chunks_fts
|
|
ON reedy_book_chunks USING fts (text) WITH (tokenizer = 'ngram');
|
|
`,
|
|
},
|
|
{
|
|
// MVP measurement (plan §M1.9). Local-only by default — no network
|
|
// egress; the user can manually export a 90-day JSON bundle from
|
|
// settings to share. `app_version` + `schema_version` are captured
|
|
// per row so future bundle replay still parses cleanly after we
|
|
// evolve the event shape.
|
|
name: '2026052602_reedy_metrics',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS reedy_metrics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
ts INTEGER NOT NULL,
|
|
event TEXT NOT NULL,
|
|
book_hash TEXT,
|
|
session_id TEXT,
|
|
turn_id TEXT,
|
|
message_id TEXT,
|
|
app_version TEXT NOT NULL,
|
|
schema_version INTEGER NOT NULL,
|
|
payload TEXT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_metrics_ts ON reedy_metrics (ts DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_metrics_session ON reedy_metrics (session_id, ts DESC);
|
|
`,
|
|
},
|
|
{
|
|
// Memory store for the agent runtime (Phase 3.1). One table, three
|
|
// scopes: user / book / session. UNIQUE(scope, scope_key, key)
|
|
// gives us upsert semantics — writing the same key twice replaces
|
|
// the prior summary. Embeddings live in a sibling table created
|
|
// lazily by MemoryService (same single-model-lock pattern as
|
|
// reedy_book_chunk_embeddings) so the vector32 dim matches the
|
|
// active embedding model.
|
|
name: '2026052603_reedy_memory',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS reedy_memory (
|
|
id TEXT PRIMARY KEY,
|
|
scope TEXT NOT NULL,
|
|
scope_key TEXT NOT NULL,
|
|
key TEXT NOT NULL,
|
|
summary TEXT NOT NULL,
|
|
source_message_id TEXT,
|
|
updated_at INTEGER NOT NULL,
|
|
UNIQUE(scope, scope_key, key)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_memory_scope
|
|
ON reedy_memory (scope, scope_key, updated_at DESC);
|
|
`,
|
|
},
|
|
{
|
|
// Skill catalog for the agent runtime (Phase 5.1). Built-in skills
|
|
// are seeded on first SkillRegistry boot; user-defined skills
|
|
// (post-MVP) live in the same table with builtin=0. tool_allowlist
|
|
// is a JSON-encoded string array applied by the runtime when
|
|
// building the per-turn ToolSet.
|
|
name: '2026052604_reedy_skills',
|
|
sql: `
|
|
CREATE TABLE IF NOT EXISTS reedy_skills (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
description TEXT NOT NULL,
|
|
instructions TEXT NOT NULL,
|
|
tool_allowlist TEXT,
|
|
builtin INTEGER NOT NULL DEFAULT 1,
|
|
enabled INTEGER NOT NULL DEFAULT 1
|
|
);
|
|
`,
|
|
},
|
|
],
|
|
};
|
|
|
|
export function getMigrations(schema: SchemaType): MigrationEntry[] {
|
|
return migrations[schema] ?? [];
|
|
}
|