Files
readest/apps/readest-app/src/services/database/migrations/index.ts
T
Huang Xin 35b02c4efc feat(statistics): KOReader-compatible reading stats with cross-device sync (#4606)
Supersedes #3156. Adds a reading-statistics system whose canonical data model
is KOReader's own (book + page_stat_data), so stats round-trip losslessly
between Readest and KOReader.

- Storage: a cross-platform Turso statistics.db in KOReader's exact schema
  (web/Workers, desktop, iOS, Android) — replacing #3156's Node-only
  better-sqlite3 + statistics.json.
- Tracking: per-page reading events (time-on-page, idle-capped) flushed on
  page-change/idle/hide/close — the KOReader model — not session aggregates.
- Sync: legacy /api/sync extended with a stats type backed by self-contained
  Supabase tables (stat_books, stat_pages); union/longer-duration-wins merge
  keyed on book_hash. apps/readest.koplugin syncs through the same endpoint.
- Scale & robustness: per-tab singleton connection (avoids OPFS lock
  conflicts) + explicit WAL checkpoint; transactional bulk apply; chunked
  resumable push; client-driven paged pull with trailing-ms completion;
  paginated/scoped server merge.

Verified: 5668 unit tests, 155 koplugin busted tests, biome+tsgo + luacheck
all green; web OPFS DB verified live.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 10:42:39 +02:00

221 lines
8.3 KiB
TypeScript

import { MigrationEntry, SchemaType } from '../migrate';
/**
* Migration definitions for each schema type.
*
* To add a new migration:
* 1. Append a new entry to the appropriate schema array below.
* 2. Use a date-based name: YYYYMMDDNN (NN = sequence within the day).
* 3. Never reorder or remove existing entries.
*
* To add a new schema type:
* 1. Add the type to SchemaType in migrate.ts.
* 2. Add a new key here with its migration array.
*/
const migrations: Record<SchemaType, MigrationEntry[]> = {
opds: [
{
name: '2026052701_opds_source_mappings',
sql: `
CREATE TABLE IF NOT EXISTS opds_source_mappings (
catalog_id TEXT NOT NULL,
source_url TEXT NOT NULL,
book_hash TEXT NOT NULL,
PRIMARY KEY (catalog_id, source_url)
);
`,
},
],
'hardcover-sync': [
{
name: '2026032901_hardcover_note_mappings',
sql: `
CREATE TABLE IF NOT EXISTS hardcover_note_mappings (
book_hash TEXT NOT NULL,
note_id TEXT NOT NULL,
hardcover_journal_id INTEGER NOT NULL,
payload_hash TEXT NOT NULL,
synced_at INTEGER NOT NULL,
PRIMARY KEY (book_hash, note_id)
);
CREATE INDEX IF NOT EXISTS idx_hardcover_note_mappings_synced_at
ON hardcover_note_mappings (synced_at);
`,
},
],
// The embeddings table is created lazily by BookIndexer because its
// vector32(<dim>) column needs the active embedding model's dim, which
// isn't known at migration time. Tantivy FTS lives on the chunks.text
// column directly (no virtual table). Writers MUST DELETE+INSERT chunk
// rows rather than UPDATE — Tantivy 0.25→0.26 has a known WASM-only
// UPDATE regression (see fts-tests.ts:306 FIXME). MVP indexing is
// write-once per book so this is naturally satisfied.
statistics: [
{
name: '2026061501_statistics_koreader_schema',
sql: `
CREATE TABLE IF NOT EXISTS book (
id integer PRIMARY KEY autoincrement,
title text, authors text, notes integer, last_open integer,
highlights integer, pages integer, series text, language text,
md5 text, total_read_time integer, total_read_pages integer
);
CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
CREATE TABLE IF NOT EXISTS page_stat_data (
id_book integer,
page integer NOT NULL DEFAULT 0,
start_time integer NOT NULL DEFAULT 0,
duration integer NOT NULL DEFAULT 0,
total_pages integer NOT NULL DEFAULT 0,
UNIQUE (id_book, page, start_time)
);
CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time);
CREATE TABLE IF NOT EXISTS numbers (number INTEGER PRIMARY KEY);
INSERT OR IGNORE INTO numbers(number)
SELECT h.n * 100 + t.n * 10 + o.n + 1
FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) o,
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t,
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) h;
CREATE VIEW IF NOT EXISTS page_stat AS
SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration
FROM (
SELECT id_book, page, total_pages, pages, start_time, duration,
((page - 1) * pages) / total_pages + 1 AS first_page,
max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page,
idx
FROM page_stat_data
JOIN book ON book.id = id_book
JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1)
);
CREATE TABLE IF NOT EXISTS readest_page_ext (
book_hash text NOT NULL, page integer NOT NULL, start_time integer NOT NULL,
ext text, PRIMARY KEY (book_hash, page, start_time)
);
CREATE TABLE IF NOT EXISTS readest_book_ext (
book_hash text PRIMARY KEY, ext text
);
CREATE TABLE IF NOT EXISTS readest_stat_sync_state (
key text PRIMARY KEY, value integer NOT NULL DEFAULT 0
);
`,
},
],
reedy: [
{
name: '2026052601_reedy_init',
sql: `
CREATE TABLE IF NOT EXISTS reedy_book_meta (
book_hash TEXT PRIMARY KEY,
indexing_status TEXT NOT NULL,
chunk_count INTEGER NOT NULL DEFAULT 0,
embedding_model TEXT NOT NULL,
embedding_dim INTEGER NOT NULL,
indexed_at INTEGER,
error TEXT
);
CREATE TABLE IF NOT EXISTS reedy_book_chunks (
id TEXT PRIMARY KEY,
book_hash TEXT NOT NULL,
section_index INTEGER NOT NULL,
chapter_title TEXT,
start_cfi TEXT NOT NULL,
end_cfi TEXT NOT NULL,
position_index INTEGER NOT NULL,
text TEXT NOT NULL,
token_count INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_chunks_book_position
ON reedy_book_chunks (book_hash, position_index);
CREATE INDEX IF NOT EXISTS idx_chunks_fts
ON reedy_book_chunks USING fts (text) WITH (tokenizer = 'ngram');
`,
},
{
// MVP measurement (plan §M1.9). Local-only by default — no network
// egress; the user can manually export a 90-day JSON bundle from
// settings to share. `app_version` + `schema_version` are captured
// per row so future bundle replay still parses cleanly after we
// evolve the event shape.
name: '2026052602_reedy_metrics',
sql: `
CREATE TABLE IF NOT EXISTS reedy_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
event TEXT NOT NULL,
book_hash TEXT,
session_id TEXT,
turn_id TEXT,
message_id TEXT,
app_version TEXT NOT NULL,
schema_version INTEGER NOT NULL,
payload TEXT
);
CREATE INDEX IF NOT EXISTS idx_metrics_ts ON reedy_metrics (ts DESC);
CREATE INDEX IF NOT EXISTS idx_metrics_session ON reedy_metrics (session_id, ts DESC);
`,
},
{
// Memory store for the agent runtime (Phase 3.1). One table, three
// scopes: user / book / session. UNIQUE(scope, scope_key, key)
// gives us upsert semantics — writing the same key twice replaces
// the prior summary. Embeddings live in a sibling table created
// lazily by MemoryService (same single-model-lock pattern as
// reedy_book_chunk_embeddings) so the vector32 dim matches the
// active embedding model.
name: '2026052603_reedy_memory',
sql: `
CREATE TABLE IF NOT EXISTS reedy_memory (
id TEXT PRIMARY KEY,
scope TEXT NOT NULL,
scope_key TEXT NOT NULL,
key TEXT NOT NULL,
summary TEXT NOT NULL,
source_message_id TEXT,
updated_at INTEGER NOT NULL,
UNIQUE(scope, scope_key, key)
);
CREATE INDEX IF NOT EXISTS idx_memory_scope
ON reedy_memory (scope, scope_key, updated_at DESC);
`,
},
{
// Skill catalog for the agent runtime (Phase 5.1). Built-in skills
// are seeded on first SkillRegistry boot; user-defined skills
// (post-MVP) live in the same table with builtin=0. tool_allowlist
// is a JSON-encoded string array applied by the runtime when
// building the per-turn ToolSet.
name: '2026052604_reedy_skills',
sql: `
CREATE TABLE IF NOT EXISTS reedy_skills (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL,
instructions TEXT NOT NULL,
tool_allowlist TEXT,
builtin INTEGER NOT NULL DEFAULT 1,
enabled INTEGER NOT NULL DEFAULT 1
);
`,
},
],
};
export function getMigrations(schema: SchemaType): MigrationEntry[] {
return migrations[schema] ?? [];
}