diff --git a/apps/readest-app/docs/superpowers/plans/2026-06-15-koreader-stats-sync.md b/apps/readest-app/docs/superpowers/plans/2026-06-15-koreader-stats-sync.md new file mode 100644 index 00000000..3a8b1ed3 --- /dev/null +++ b/apps/readest-app/docs/superpowers/plans/2026-06-15-koreader-stats-sync.md @@ -0,0 +1,1583 @@ +# KOReader-compatible reading statistics — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Track reading time as KOReader-style per-page events in a cross-platform Turso `statistics.db`, and sync those events across all Readest **and** KOReader devices over the existing `/api/sync` transport. + +**Architecture:** Local source of truth is a Turso `statistics.db` holding KOReader's exact schema (`book` + `page_stat_data`). A per-book reader component flushes immutable page-read events on page-change/idle/hidden/close. Cross-device sync rides the legacy `/api/sync` endpoint as a new `stats` type backed by two self-contained Supabase tables (`stat_books`, `stat_pages`), union-merged by `book_hash`. The `apps/readest.koplugin` plugin syncs KOReader devices through the same endpoint. + +**Tech Stack:** TypeScript / React (Next.js 16), Zustand, the Turso `DatabaseService` abstraction, Supabase (Postgres + RLS), Lua (KOReader plugin), vitest + busted. + +**Design spec:** `docs/superpowers/specs/2026-06-15-koreader-stats-sync-design.md` + +--- + +## Conventions for this plan + +- Run a single test file with: `pnpm test ` (no `--`). +- Full gate before finishing a phase: `pnpm test` + `pnpm lint`. Lua phase also: `pnpm lint:lua` + `pnpm test:lua`. +- Units: `start_time` and `duration` are **Unix seconds** (matches KOReader). `page` is **1-based**. +- Book identity is `book_hash` = `Book.hash` = `partialMD5(file)` (byte-identical to KOReader's `util.partialMD5`). +- This plan has **three phases**, each independently shippable. Stop and review at each "▣ Phase checkpoint". + +--- + +# Phase 1 — Local store + page-event tracking + +Produces working local stats (recorded to `statistics.db`) with no sync yet. + +## File structure (Phase 1) + +- Create `src/types/statistics.ts` — event/book/config types. +- Modify `src/services/database/migrations/index.ts` — add the `statistics` schema (KOReader DDL + extension tables + sync-state table). +- Create `src/services/statistics/statisticsDb.ts` — typed DB wrapper (upsert book, insert event, recompute totals, cursors, push/apply helpers). +- Create `src/services/statistics/trackerCore.ts` — pure flush/idle state machine (no React), so it is unit-testable. +- Create `src/app/reader/components/ReadingStatsTracker.tsx` — per-`bookKey` React component wiring `trackerCore` to progress + visibility + unmount. +- Modify `src/app/reader/components/BooksGrid.tsx` — mount `ReadingStatsTracker` per book (same place PR #3156 mounted its tracker). +- Modify `src/app/reader/components/ReaderContent.tsx` — open `statistics.db` on mount. +- Tests: `src/__tests__/statistics/statisticsDb.test.ts`, `src/__tests__/statistics/trackerCore.test.ts`. + +--- + +### Task 1: Statistics types + +**Files:** +- Create: `src/types/statistics.ts` + +- [ ] **Step 1: Write the file** + +```typescript +// KOReader-compatible reading-statistics types. +// +// The canonical unit is the per-page read event (KOReader's page_stat_data +// row). Aggregates (sessions, streaks, totals) are DERIVED via SQL, never +// stored as separate records. Times are Unix seconds; pages are 1-based. + +/** One immutable page-read event — a KOReader `page_stat_data` row. */ +export interface PageStatEvent { + bookMd5: string; // = Book.hash = KOReader book.md5 + page: number; // 1-based + startTime: number; // Unix seconds + duration: number; // seconds + totalPages: number; +} + +/** KOReader book identity — the only book metadata that syncs. */ +export interface StatBook { + bookMd5: string; + title: string; + authors: string; // KOReader stores authors as a single text field +} + +/** Tunables for the tracker's flush/idle behavior. KOReader-aligned defaults. */ +export interface StatsTrackingConfig { + /** Seconds of inactivity before the current page event is flushed + paused. */ + idleTimeoutSeconds: number; + /** Hard per-event duration cap (safety net if a visibility event is missed). */ + maxEventSeconds: number; + /** Events shorter than this are dropped (ignore sub-second page flips). */ + minEventSeconds: number; +} + +export const DEFAULT_STATS_TRACKING_CONFIG: StatsTrackingConfig = { + idleTimeoutSeconds: 120, + maxEventSeconds: 120, + minEventSeconds: 3, +}; +``` + +- [ ] **Step 2: Typecheck** + +Run: `pnpm lint` +Expected: PASS (no usages yet; file compiles). + +- [ ] **Step 3: Commit** + +```bash +git add src/types/statistics.ts +git commit -m "feat(stats): add KOReader-compatible statistics types" +``` + +--- + +### Task 2: `statistics` migration schema (KOReader DDL + extensions) + +**Files:** +- Modify: `src/services/database/migrations/index.ts` +- Test: `src/__tests__/statistics/statisticsDb.test.ts` (created here; grows in Task 3) + +- [ ] **Step 1: Write the failing test** (migration creates the KOReader schema) + +Create `src/__tests__/statistics/statisticsDb.test.ts`: + +```typescript +import { describe, it, expect, beforeEach } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import type { DatabaseService } from '@/types/database'; + +async function freshStatsDb(): Promise { + // In-memory libsql DB; run the same migrations production uses. + const db = await NodeDatabaseService.open(':memory:'); + await migrate(db, getMigrations('statistics')); + return db; +} + +describe('statistics migration', () => { + let db: DatabaseService; + beforeEach(async () => { + db = await freshStatsDb(); + }); + + it('creates KOReader book + page_stat_data tables and extension tables', async () => { + const tables = await db.select<{ name: string }>( + `SELECT name FROM sqlite_master WHERE type IN ('table','view') ORDER BY name`, + ); + const names = tables.map((t) => t.name); + expect(names).toContain('book'); + expect(names).toContain('page_stat_data'); + expect(names).toContain('numbers'); + expect(names).toContain('page_stat'); // the rescaling view + expect(names).toContain('readest_page_ext'); + expect(names).toContain('readest_book_ext'); + expect(names).toContain('readest_stat_sync_state'); + }); + + it('seeds the numbers helper table 1..1000', async () => { + const rows = await db.select<{ c: number }>(`SELECT COUNT(*) AS c FROM numbers`); + expect(rows[0]!.c).toBe(1000); + }); + + it('enforces the page_stat_data uniqueness key', async () => { + await db.execute(`INSERT INTO book (title, authors, md5) VALUES ('T','A','m')`); + const id = (await db.select<{ id: number }>(`SELECT id FROM book LIMIT 1`))[0]!.id; + await db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (?,?,?,?,?)`, + [id, 5, 1000, 10, 100], + ); + await db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?,?,?,?,?) + ON CONFLICT(id_book, page, start_time) DO UPDATE SET duration = max(duration, excluded.duration)`, + [id, 5, 1000, 25, 100], + ); + const rows = await db.select<{ duration: number; c: number }>( + `SELECT duration, COUNT(*) OVER () AS c FROM page_stat_data`, + ); + expect(rows.length).toBe(1); + expect(rows[0]!.duration).toBe(25); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pnpm test src/__tests__/statistics/statisticsDb.test.ts` +Expected: FAIL — `getMigrations('statistics')` returns `[]`, so no tables exist. + +- [ ] **Step 3: Add the `statistics` schema** + +In `src/services/database/migrations/index.ts`, add a new key to the `migrations` record (after the `reedy` entry, before the closing `}`). The DDL is KOReader's verbatim, plus Readest extension tables and a sync-cursor table. `numbers` is seeded 1..1000 via a recursive CTE so the `page_stat` view's rescale join works. + +```typescript + statistics: [ + { + name: '2026061501_statistics_koreader_schema', + sql: ` + CREATE TABLE IF NOT EXISTS book ( + id integer PRIMARY KEY autoincrement, + title text, authors text, notes integer, last_open integer, + highlights integer, pages integer, series text, language text, + md5 text, total_read_time integer, total_read_pages integer + ); + + CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); + + CREATE TABLE IF NOT EXISTS page_stat_data ( + id_book integer, + page integer NOT NULL DEFAULT 0, + start_time integer NOT NULL DEFAULT 0, + duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + UNIQUE (id_book, page, start_time) + ); + + CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time); + + CREATE TABLE IF NOT EXISTS numbers (number INTEGER PRIMARY KEY); + + INSERT OR IGNORE INTO numbers(number) + WITH RECURSIVE c(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM c WHERE n < 1000) + SELECT n FROM c; + + CREATE VIEW IF NOT EXISTS page_stat AS + SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration + FROM ( + SELECT id_book, page, total_pages, pages, start_time, duration, + ((page - 1) * pages) / total_pages + 1 AS first_page, + max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page, + idx + FROM page_stat_data + JOIN book ON book.id = id_book + JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1) + ); + + CREATE TABLE IF NOT EXISTS readest_page_ext ( + book_hash text NOT NULL, page integer NOT NULL, start_time integer NOT NULL, + ext text, PRIMARY KEY (book_hash, page, start_time) + ); + + CREATE TABLE IF NOT EXISTS readest_book_ext ( + book_hash text PRIMARY KEY, ext text + ); + + CREATE TABLE IF NOT EXISTS readest_stat_sync_state ( + key text PRIMARY KEY, value integer NOT NULL DEFAULT 0 + ); + `, + }, + ], +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pnpm test src/__tests__/statistics/statisticsDb.test.ts` +Expected: PASS (all three migration tests green). If the `page_stat` view or recursive CTE errors on the Node libsql build, that is a real compatibility finding — report it; do not silently drop the view. + +- [ ] **Step 5: Commit** + +```bash +git add src/services/database/migrations/index.ts src/__tests__/statistics/statisticsDb.test.ts +git commit -m "feat(stats): add KOReader-compatible statistics.db schema migration" +``` + +--- + +### Task 3: `StatisticsDb` wrapper + +**Files:** +- Create: `src/services/statistics/statisticsDb.ts` +- Test: `src/__tests__/statistics/statisticsDb.test.ts` (extend) + +- [ ] **Step 1: Write the failing tests** (append to the existing test file) + +Append to `src/__tests__/statistics/statisticsDb.test.ts`: + +```typescript +import { StatisticsDb } from '@/services/statistics/statisticsDb'; + +describe('StatisticsDb', () => { + let stats: StatisticsDb; + beforeEach(async () => { + stats = StatisticsDb.from(await freshStatsDb()); + }); + + it('upserts a book by md5 and returns a stable id_book', async () => { + const id1 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + const id2 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + expect(id1).toBe(id2); + }); + + it('inserts page events and keeps the longer duration on re-flush', async () => { + const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 10, totalPages: 50 }); + await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 30, totalPages: 50 }); + await stats.insertPageEvent(id, { page: 4, startTime: 140, duration: 12, totalPages: 50 }); + await stats.recomputeBookTotals(id); + const book = await stats.getBookByMd5('m1'); + expect(book!.total_read_time).toBe(42); // 30 + 12 + expect(book!.total_read_pages).toBe(2); // distinct pages 3,4 + expect(book!.last_open).toBe(152); // max(start_time + duration) = 140 + 12 + }); + + it('returns events for push after a start_time cursor, joined with md5', async () => { + const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 }); + await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 5, totalPages: 9 }); + const { events } = await stats.getEventsForPush(150); + expect(events.map((e) => e.startTime)).toEqual([200]); + expect(events[0]!.bookMd5).toBe('m1'); + }); + + it('applies remote events idempotently via upsert', async () => { + const remoteBooks = [{ bookMd5: 'm2', title: 'T2', authors: 'A2' }]; + const remoteEvents = [ + { bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 }, + { bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 }, // dup + ]; + await stats.applyRemoteEvents(remoteBooks, remoteEvents); + await stats.applyRemoteEvents(remoteBooks, remoteEvents); // again — still idempotent + const book = await stats.getBookByMd5('m2'); + expect(book!.total_read_time).toBe(8); + }); + + it('reads and writes sync cursors', async () => { + expect(await stats.getCursor('push')).toBe(0); + await stats.setCursor('push', 1234); + expect(await stats.getCursor('push')).toBe(1234); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pnpm test src/__tests__/statistics/statisticsDb.test.ts` +Expected: FAIL — `@/services/statistics/statisticsDb` does not exist. + +- [ ] **Step 3: Implement `StatisticsDb`** + +Create `src/services/statistics/statisticsDb.ts`: + +```typescript +import type { AppService } from '@/services/appService'; +import type { DatabaseService, DatabaseRow } from '@/types/database'; +import type { PageStatEvent, StatBook } from '@/types/statistics'; + +interface BookRow extends DatabaseRow { + id: number; + title: string; + authors: string; + md5: string; + total_read_time: number; + total_read_pages: number; + last_open: number; + pages: number; +} + +type CursorKey = 'push' | 'pull'; + +/** + * Typed wrapper over the KOReader-compatible `statistics.db`. All identity is + * keyed on `book_hash` (= Book.hash); the local autoincrement `id_book` never + * leaves this class. + */ +export class StatisticsDb { + private constructor(private readonly db: DatabaseService) {} + + /** Production entry point — opens + migrates statistics.db. */ + static async open(appService: AppService): Promise { + const db = await appService.openDatabase('statistics', 'statistics.db', 'Data'); + return new StatisticsDb(db); + } + + /** Test/advanced entry point — wrap an already-migrated DatabaseService. */ + static from(db: DatabaseService): StatisticsDb { + return new StatisticsDb(db); + } + + async upsertBook(book: StatBook): Promise { + await this.db.execute( + `INSERT INTO book (title, authors, md5, notes, last_open, highlights, pages, total_read_time, total_read_pages) + VALUES (?, ?, ?, 0, 0, 0, 0, 0, 0) + ON CONFLICT(title, authors, md5) DO NOTHING`, + [book.title, book.authors, book.bookMd5], + ); + const rows = await this.db.select( + `SELECT id FROM book WHERE title = ? AND authors = ? AND md5 = ? LIMIT 1`, + [book.title, book.authors, book.bookMd5], + ); + return rows[0]!.id; + } + + async insertPageEvent( + idBook: number, + e: { page: number; startTime: number; duration: number; totalPages: number }, + ): Promise { + await this.db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id_book, page, start_time) + DO UPDATE SET duration = max(duration, excluded.duration), total_pages = excluded.total_pages`, + [idBook, e.page, e.startTime, e.duration, e.totalPages], + ); + } + + async recomputeBookTotals(idBook: number): Promise { + await this.db.execute( + `UPDATE book SET + total_read_time = COALESCE((SELECT SUM(duration) FROM page_stat_data WHERE id_book = ?), 0), + total_read_pages = COALESCE((SELECT COUNT(DISTINCT page) FROM page_stat_data WHERE id_book = ?), 0), + last_open = COALESCE((SELECT MAX(start_time + duration) FROM page_stat_data WHERE id_book = ?), last_open), + pages = COALESCE((SELECT total_pages FROM page_stat_data WHERE id_book = ? ORDER BY start_time DESC LIMIT 1), pages) + WHERE id = ?`, + [idBook, idBook, idBook, idBook, idBook], + ); + } + + async getBookByMd5(md5: string): Promise { + const rows = await this.db.select(`SELECT * FROM book WHERE md5 = ? LIMIT 1`, [md5]); + return rows[0] ?? null; + } + + /** Events with start_time > cursor, joined to their md5, for pushing. */ + async getEventsForPush( + sinceStartTime: number, + ): Promise<{ events: PageStatEvent[]; books: StatBook[] }> { + const rows = await this.db.select( + `SELECT b.md5 AS bookMd5, b.title AS title, b.authors AS authors, + p.page AS page, p.start_time AS startTime, p.duration AS duration, p.total_pages AS totalPages + FROM page_stat_data p JOIN book b ON b.id = p.id_book + WHERE p.start_time > ? + ORDER BY p.start_time ASC`, + [sinceStartTime], + ); + const events: PageStatEvent[] = rows.map((r) => ({ + bookMd5: String(r['bookMd5']), + page: Number(r['page']), + startTime: Number(r['startTime']), + duration: Number(r['duration']), + totalPages: Number(r['totalPages']), + })); + const bookMap = new Map(); + for (const r of rows) { + const md5 = String(r['bookMd5']); + if (!bookMap.has(md5)) { + bookMap.set(md5, { bookMd5: md5, title: String(r['title']), authors: String(r['authors']) }); + } + } + return { events, books: [...bookMap.values()] }; + } + + async applyRemoteEvents(books: StatBook[], events: PageStatEvent[]): Promise { + const idByMd5 = new Map(); + for (const b of books) idByMd5.set(b.bookMd5, await this.upsertBook(b)); + // Books referenced only by events (no metadata record) get a placeholder row. + const touched = new Set(); + for (const e of events) { + let id = idByMd5.get(e.bookMd5); + if (id === undefined) { + id = await this.upsertBook({ bookMd5: e.bookMd5, title: e.bookMd5, authors: '' }); + idByMd5.set(e.bookMd5, id); + } + await this.insertPageEvent(id, e); + touched.add(id); + } + for (const id of touched) await this.recomputeBookTotals(id); + } + + async getCursor(key: CursorKey): Promise { + const rows = await this.db.select<{ value: number }>( + `SELECT value FROM readest_stat_sync_state WHERE key = ?`, + [key], + ); + return rows[0]?.value ?? 0; + } + + async setCursor(key: CursorKey, value: number): Promise { + await this.db.execute( + `INSERT INTO readest_stat_sync_state (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + [key, value], + ); + } +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `pnpm test src/__tests__/statistics/statisticsDb.test.ts` +Expected: PASS (all StatisticsDb tests green). + +- [ ] **Step 5: Commit** + +```bash +git add src/services/statistics/statisticsDb.ts src/__tests__/statistics/statisticsDb.test.ts +git commit -m "feat(stats): add StatisticsDb wrapper over the KOReader schema" +``` + +--- + +### Task 4: `trackerCore` flush/idle state machine + +**Files:** +- Create: `src/services/statistics/trackerCore.ts` +- Test: `src/__tests__/statistics/trackerCore.test.ts` + +The core is pure: it receives `(page, totalPages, nowSeconds)` notifications plus idle/visibility/close signals and returns the page events to persist. No timers, no DB — the React layer drives `now` and schedules idle. + +- [ ] **Step 1: Write the failing tests** + +Create `src/__tests__/statistics/trackerCore.test.ts`: + +```typescript +import { describe, it, expect } from 'vitest'; +import { TrackerCore } from '@/services/statistics/trackerCore'; +import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics'; + +const cfg = DEFAULT_STATS_TRACKING_CONFIG; + +describe('TrackerCore', () => { + it('emits an event when the page changes', () => { + const t = new TrackerCore(cfg); + expect(t.onPage(5, 100, 1000)).toEqual([]); // first page, nothing to flush yet + const out = t.onPage(6, 100, 1030); // moved off page 5 after 30s + expect(out).toEqual([{ page: 5, startTime: 1000, duration: 30, totalPages: 100 }]); + }); + + it('caps duration at maxEventSeconds', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + const out = t.onPage(2, 10, 10_000); // way over the 120s cap + expect(out[0]!.duration).toBe(cfg.maxEventSeconds); + }); + + it('drops sub-minimum events', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + expect(t.onPage(2, 10, 1)).toEqual([]); // 1s < minEventSeconds (3) + }); + + it('flushes and pauses on idle, then resumes with a new start_time', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + expect(t.onIdle(50)).toEqual([{ page: 1, startTime: 0, duration: 50, totalPages: 10 }]); + // After idle, the same page resumes as a fresh event. + expect(t.onPage(1, 10, 200)).toEqual([]); // resume marker, no flush + expect(t.onPage(2, 10, 230)).toEqual([{ page: 1, startTime: 200, duration: 30, totalPages: 10 }]); + }); + + it('flushes on hide and on close without double-counting', () => { + const t = new TrackerCore(cfg); + t.onPage(7, 10, 0); + expect(t.onHide(40)).toEqual([{ page: 7, startTime: 0, duration: 40, totalPages: 10 }]); + expect(t.onClose(99)).toEqual([]); // already flushed + paused by hide + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pnpm test src/__tests__/statistics/trackerCore.test.ts` +Expected: FAIL — `@/services/statistics/trackerCore` does not exist. + +- [ ] **Step 3: Implement `TrackerCore`** + +Create `src/services/statistics/trackerCore.ts`: + +```typescript +import type { StatsTrackingConfig } from '@/types/statistics'; + +interface PendingEvent { + page: number; + startTime: number; // Unix seconds + totalPages: number; +} + +export interface FlushedEvent { + page: number; + startTime: number; + duration: number; + totalPages: number; +} + +/** + * Pure page-dwell tracker. The React layer feeds it `nowSeconds` and the + * current `(page, totalPages)`; it returns zero-or-one immutable events to + * persist. Events end on page-change / idle / hide / close. Each returned + * event is final — its start_time and duration never change afterwards, which + * lets sync use a simple start_time high-water cursor. + */ +export class TrackerCore { + private pending: PendingEvent | null = null; + + constructor(private readonly cfg: StatsTrackingConfig) {} + + /** Notify the current page at `now`. Returns events flushed by leaving the prior page. */ + onPage(page: number, totalPages: number, now: number): FlushedEvent[] { + if (this.pending && this.pending.page === page) { + // Same page (e.g. resume after idle, or a no-op relocate): keep dwelling. + return []; + } + const flushed = this.flush(now); + this.pending = { page, startTime: now, totalPages }; + return flushed; + } + + onIdle(now: number): FlushedEvent[] { + return this.flush(now); // flush + pause (pending cleared) + } + + onHide(now: number): FlushedEvent[] { + return this.flush(now); + } + + onClose(now: number): FlushedEvent[] { + return this.flush(now); + } + + private flush(now: number): FlushedEvent[] { + const p = this.pending; + this.pending = null; + if (!p) return []; + const raw = now - p.startTime; + const duration = Math.min(Math.max(raw, 0), this.cfg.maxEventSeconds); + if (duration < this.cfg.minEventSeconds) return []; + return [{ page: p.page, startTime: p.startTime, duration, totalPages: p.totalPages }]; + } +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `pnpm test src/__tests__/statistics/trackerCore.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/services/statistics/trackerCore.ts src/__tests__/statistics/trackerCore.test.ts +git commit -m "feat(stats): add pure page-dwell tracker core" +``` + +--- + +### Task 5: `ReadingStatsTracker` React component + mount + +**Files:** +- Create: `src/app/reader/components/ReadingStatsTracker.tsx` +- Modify: `src/app/reader/components/BooksGrid.tsx` +- Reference: `src/store/readerStore.ts` (the `viewStates[bookKey].progress` selector PR #3156 used: `progress.pageinfo.{current,total}`, `progress.location`) + +This component holds no business logic — it wires `TrackerCore` to: progress changes (`onPage`), an idle timer (`onIdle`), `document.visibilitychange` (`onHide`), and unmount (`onClose`), persisting each flushed event via `StatisticsDb`. + +- [ ] **Step 1: Implement the component** + +Create `src/app/reader/components/ReadingStatsTracker.tsx`: + +```tsx +'use client'; + +import { useEffect, useRef } from 'react'; +import { useReaderStore } from '@/store/readerStore'; +import { useBookDataStore } from '@/store/bookDataStore'; +import { useEnv } from '@/context/EnvContext'; +import { StatisticsDb } from '@/services/statistics/statisticsDb'; +import { TrackerCore, type FlushedEvent } from '@/services/statistics/trackerCore'; +import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics'; + +const nowSec = () => Math.floor(Date.now() / 1000); + +export default function ReadingStatsTracker({ bookKey }: { bookKey: string }) { + const { appService } = useEnv(); + const progress = useReaderStore((state) => state.viewStates[bookKey]?.progress); + const bookData = useBookDataStore((state) => state.booksData[bookKey]); + const coreRef = useRef(new TrackerCore(DEFAULT_STATS_TRACKING_CONFIG)); + const dbRef = useRef(null); + const idleRef = useRef | null>(null); + + const bookMd5 = bookData?.book?.hash; + const title = bookData?.book?.title ?? ''; + const author = bookData?.book?.author ?? ''; + + useEffect(() => { + if (!appService) return; + let cancelled = false; + StatisticsDb.open(appService).then((db) => { + if (!cancelled) dbRef.current = db; + }); + return () => { + cancelled = true; + }; + }, [appService]); + + // Persist flushed events. + const persist = (events: FlushedEvent[]) => { + const db = dbRef.current; + if (!db || !bookMd5 || events.length === 0) return; + void (async () => { + const idBook = await db.upsertBook({ bookMd5, title, authors: author }); + for (const e of events) await db.insertPageEvent(idBook, e); + await db.recomputeBookTotals(idBook); + })(); + }; + + const armIdle = () => { + if (idleRef.current) clearTimeout(idleRef.current); + idleRef.current = setTimeout( + () => persist(coreRef.current.onIdle(nowSec())), + DEFAULT_STATS_TRACKING_CONFIG.idleTimeoutSeconds * 1000, + ); + }; + + // Progress (page) changes drive the tracker. + useEffect(() => { + const info = progress?.pageinfo; + if (!info) return; + const page = (info.current ?? 0) + 1; + const total = info.total || 1; + persist(coreRef.current.onPage(page, total, nowSec())); + armIdle(); + }, [progress?.pageinfo]); // eslint-disable-line react-hooks/exhaustive-deps + + // Tab/window visibility. + useEffect(() => { + const onVis = () => { + if (document.visibilityState === 'hidden') { + if (idleRef.current) clearTimeout(idleRef.current); + persist(coreRef.current.onHide(nowSec())); + } + }; + document.addEventListener('visibilitychange', onVis); + return () => document.removeEventListener('visibilitychange', onVis); + }, [bookMd5]); // eslint-disable-line react-hooks/exhaustive-deps + + // Book close (unmount). + useEffect(() => { + return () => { + if (idleRef.current) clearTimeout(idleRef.current); + persist(coreRef.current.onClose(nowSec())); + }; + }, [bookMd5]); // eslint-disable-line react-hooks/exhaustive-deps + + return null; +} +``` + +- [ ] **Step 2: Mount it per book** + +In `src/app/reader/components/BooksGrid.tsx`, locate where each book view is rendered per `bookKey` (PR #3156 mounted `BookSessionTracker` here). Add the import and render the tracker alongside the existing per-book content: + +```tsx +import ReadingStatsTracker from './ReadingStatsTracker'; +``` + +Inside the per-`bookKey` map, render `` next to the existing viewer for that key. + +- [ ] **Step 3: Verify build + lint** + +Run: `pnpm lint` +Expected: PASS. Then sanity-check the field paths against the real stores: confirm `useBookDataStore().booksData[bookKey].book.hash/title/author` and `useReaderStore().viewStates[bookKey].progress.pageinfo` exist (grep `booksData` and `pageinfo` if unsure). Fix selector paths to match the actual stores if they differ. + +- [ ] **Step 4: Manual smoke (dev-web)** + +Run: `pnpm dev-web`, open a book, turn a few pages, wait > 3s per page, close the book. Then in devtools confirm `statistics.db` has rows (OPFS on web), or add a temporary `console.log` of `getBookByMd5`. Remove the log before committing. + +- [ ] **Step 5: Commit** + +```bash +git add src/app/reader/components/ReadingStatsTracker.tsx src/app/reader/components/BooksGrid.tsx +git commit -m "feat(stats): track per-page reading events into statistics.db" +``` + +### ▣ Phase 1 checkpoint + +Run `pnpm test` and `pnpm lint` — both green. Local reading stats now record to a KOReader-compatible `statistics.db`. **Stop and review before Phase 2.** + +--- + +# Phase 2 — Cross-device sync (Supabase + app client) + +Adds `/api/sync type=stats` and the Readest-app push/pull. Ships Readest↔Readest stats sync. + +## File structure (Phase 2) + +- Create `docker/volumes/db/migrations/014_add_reading_stats.sql` — `stat_books` + `stat_pages` tables + RLS. +- Modify `src/libs/sync.ts` — extend `SyncType`, `SyncData`, `SyncResult` with stats. +- Modify `src/pages/api/sync.ts` — GET + POST `stats` branches (longer-duration-wins merge). +- Modify `src/types/settings.ts` — add `'stats'` to `SyncCategory` + `SYNC_CATEGORIES`. +- Modify `src/services/sync/syncCategories.ts` — map the `stats` id. +- Create `src/services/statistics/statsSync.ts` — app-side push/pull orchestration over `StatisticsDb`. +- Modify `src/app/reader/components/ReadingStatsTracker.tsx` — trigger push after persist; pull on mount. +- Tests: `src/__tests__/statistics/statsSync.test.ts`, `src/__tests__/api/statsSyncMerge.test.ts`. + +--- + +### Task 6: Supabase tables + RLS + +**Files:** +- Create: `docker/volumes/db/migrations/014_add_reading_stats.sql` + +- [ ] **Step 1: Write the migration** + +Create `docker/volumes/db/migrations/014_add_reading_stats.sql` (RLS pattern mirrors `002_add_book_shares.sql`): + +```sql +-- Migration 014: reading statistics sync (KOReader-compatible page events) + +CREATE TABLE IF NOT EXISTS public.stat_books ( + user_id uuid NOT NULL, + book_hash text NOT NULL, + title text NOT NULL DEFAULT '', + authors text NOT NULL DEFAULT '', + updated_at timestamp with time zone NOT NULL DEFAULT now(), + deleted_at timestamp with time zone NULL, + CONSTRAINT stat_books_pkey PRIMARY KEY (user_id, book_hash), + CONSTRAINT stat_books_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_stat_books_user_updated ON public.stat_books (user_id, updated_at); + +CREATE TABLE IF NOT EXISTS public.stat_pages ( + user_id uuid NOT NULL, + book_hash text NOT NULL, + page integer NOT NULL, + start_time bigint NOT NULL, + duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + ext jsonb NULL, + updated_at timestamp with time zone NOT NULL DEFAULT now(), + deleted_at timestamp with time zone NULL, + CONSTRAINT stat_pages_pkey PRIMARY KEY (user_id, book_hash, page, start_time), + CONSTRAINT stat_pages_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_stat_pages_user_updated ON public.stat_pages (user_id, updated_at); + +ALTER TABLE public.stat_books ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.stat_pages ENABLE ROW LEVEL SECURITY; + +CREATE POLICY stat_books_select ON public.stat_books FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_insert ON public.stat_books FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_update ON public.stat_books FOR UPDATE TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_delete ON public.stat_books FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id); + +CREATE POLICY stat_pages_select ON public.stat_pages FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_insert ON public.stat_pages FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_update ON public.stat_pages FOR UPDATE TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_delete ON public.stat_pages FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id); +``` + +- [ ] **Step 2: Apply locally** (if the local Supabase stack is running) + +Run: `psql "$SUPABASE_DB_URL" -f docker/volumes/db/migrations/014_add_reading_stats.sql` (or whatever the repo's migration-apply step is — check `docker/` README). If no local stack, note that it applies on next stack bring-up. + +- [ ] **Step 3: Commit** + +```bash +git add docker/volumes/db/migrations/014_add_reading_stats.sql +git commit -m "feat(stats): add stat_books/stat_pages Supabase tables with RLS" +``` + +--- + +### Task 7: Sync wire types + +**Files:** +- Modify: `src/libs/sync.ts` + +- [ ] **Step 1: Extend the types** + +In `src/libs/sync.ts`: + +```typescript +export type SyncType = 'books' | 'configs' | 'notes' | 'stats'; +``` + +Add the wire record shapes and extend `SyncData` / `SyncResult`: + +```typescript +export interface StatBookRecord { + user_id?: string; + book_hash: string; + title: string; + authors: string; + updated_at?: string; + updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math + deleted_at?: string | null; +} + +export interface StatPageRecord { + user_id?: string; + book_hash: string; + page: number; + start_time: number; + duration: number; + total_pages: number; + ext?: unknown; + updated_at?: string; + updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math + deleted_at?: string | null; +} +``` + +In `SyncData` add: `statBooks?: StatBookRecord[]; statPages?: StatPageRecord[];` +In `SyncResult` add: `statBooks?: StatBookRecord[] | null; statPages?: StatPageRecord[] | null;` + +- [ ] **Step 2: Lint** + +Run: `pnpm lint` +Expected: PASS (additive; existing call sites unaffected). + +- [ ] **Step 3: Commit** + +```bash +git add src/libs/sync.ts +git commit -m "feat(stats): add stats records to the sync wire types" +``` + +--- + +### Task 8: `/api/sync` stats merge (server) + +**Files:** +- Modify: `src/pages/api/sync.ts` +- Test: `src/__tests__/api/statsSyncMerge.test.ts` + +The stats merge is custom (not the generic LWW `upsertRecords`): `stat_pages` keeps the **greater duration** on conflict; `stat_books` is LWW by `updated_at`. Extract the page-merge decision into a pure function so it is unit-testable without Supabase. + +- [ ] **Step 1: Write the failing test** (pure merge decision) + +Create `src/__tests__/api/statsSyncMerge.test.ts`: + +```typescript +import { describe, it, expect } from 'vitest'; +import { pickWinningPages } from '@/pages/api/sync'; +import type { StatPageRecord } from '@/libs/sync'; + +const mk = (start: number, duration: number): StatPageRecord => ({ + book_hash: 'm', page: 1, start_time: start, duration, total_pages: 10, +}); + +describe('pickWinningPages', () => { + it('inserts pages the server has not seen', () => { + const { toUpsert } = pickWinningPages([mk(100, 5)], new Map()); + expect(toUpsert).toHaveLength(1); + }); + + it('keeps the longer duration on conflict', () => { + const server = new Map([['m|1|100', mk(100, 5)]]); + const win = pickWinningPages([mk(100, 9)], server); + expect(win.toUpsert[0]!.duration).toBe(9); + }); + + it('drops an incoming page whose duration is not longer', () => { + const server = new Map([['m|1|100', mk(100, 9)]]); + const win = pickWinningPages([mk(100, 5)], server); + expect(win.toUpsert).toHaveLength(0); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pnpm test src/__tests__/api/statsSyncMerge.test.ts` +Expected: FAIL — `pickWinningPages` is not exported from `@/pages/api/sync`. + +- [ ] **Step 3: Implement the merge + wire GET/POST** + +In `src/pages/api/sync.ts`, add the exported pure helper near the top (after imports): + +```typescript +import type { StatPageRecord, StatBookRecord } from '@/libs/sync'; + +const pageKey = (r: StatPageRecord) => `${r.book_hash}|${r.page}|${r.start_time}`; + +/** + * Decide which incoming page events to write: new keys always win; existing + * keys win only when the incoming duration is strictly longer (union/upsert + * semantics — KOReader-compatible). + */ +export function pickWinningPages( + incoming: StatPageRecord[], + server: Map, +): { toUpsert: StatPageRecord[] } { + const toUpsert: StatPageRecord[] = []; + for (const rec of incoming) { + const existing = server.get(pageKey(rec)); + if (!existing || rec.duration > existing.duration) toUpsert.push(rec); + } + return { toUpsert }; +} +``` + +In the GET handler, after the existing `notes` branch, add (mirrors the `since`/`book` filtering already used for other tables): + +```typescript +if (!typeParam || typeParam === 'stats') { + const statBooks = await supabase + .from('stat_books') + .select('*') + .eq('user_id', user.id) + .or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`); + let pagesQuery = supabase + .from('stat_pages') + .select('*') + .eq('user_id', user.id) + .or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`); + if (bookParam) pagesQuery = pagesQuery.eq('book_hash', bookParam); + const statPages = await pagesQuery; + if (statBooks.error) throw { table: 'stat_books', error: statBooks.error } as DBError; + if (statPages.error) throw { table: 'stat_pages', error: statPages.error } as DBError; + // Attach updated_at_ms (epoch ms) so non-JS clients (the Lua koplugin) can + // compute their pull cursor without parsing ISO-8601 timestamps. + const withMs = (rows: T[]) => + rows.map((r) => ({ ...r, updated_at_ms: r.updated_at ? new Date(r.updated_at).getTime() : 0 })); + results.statBooks = withMs(statBooks.data ?? []); + results.statPages = withMs(statPages.data ?? []); +} +``` + +Add `statBooks: [], statPages: []` to the initial `results` object, and widen its type to include them. (`updated_at_ms` is already on the wire types from Task 7.) + +In the POST handler, destructure and handle stats after the existing books/configs/notes upserts: + +```typescript +const { books = [], configs = [], notes = [], statBooks = [], statPages = [] } = body as SyncData; + +// ... existing books/configs/notes handling ... + +if (statBooks.length > 0) { + const rows = statBooks.map((b: StatBookRecord) => ({ + user_id: user.id, book_hash: b.book_hash, title: b.title, authors: b.authors, + updated_at: new Date().toISOString(), deleted_at: b.deleted_at ?? null, + })); + const { error } = await supabase.from('stat_books').upsert(rows, { onConflict: 'user_id,book_hash' }); + if (error) return NextResponse.json({ error: error.message }, { status: 500 }); +} + +if (statPages.length > 0) { + const { data: existing } = await supabase + .from('stat_pages').select('*').eq('user_id', user.id).in('book_hash', statPages.map((p) => p.book_hash)); + const serverMap = new Map(); + (existing ?? []).forEach((r) => serverMap.set(pageKey(r as StatPageRecord), r as StatPageRecord)); + const { toUpsert } = pickWinningPages(statPages, serverMap); + const rows = toUpsert.map((p) => ({ + user_id: user.id, book_hash: p.book_hash, page: p.page, start_time: p.start_time, + duration: p.duration, total_pages: p.total_pages, ext: p.ext ?? null, + updated_at: new Date().toISOString(), deleted_at: p.deleted_at ?? null, + })); + if (rows.length > 0) { + const { error } = await supabase.from('stat_pages').upsert(rows, { onConflict: 'user_id,book_hash,page,start_time' }); + if (error) return NextResponse.json({ error: error.message }, { status: 500 }); + } +} +``` + +(If the POST handler returns a combined result object, include `statBooks`/`statPages` echoes consistent with the existing return shape.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `pnpm test src/__tests__/api/statsSyncMerge.test.ts` +Expected: PASS. Then `pnpm lint` — PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/pages/api/sync.ts src/__tests__/api/statsSyncMerge.test.ts +git commit -m "feat(stats): merge stats in /api/sync (longer-duration-wins)" +``` + +--- + +### Task 9: Sync category `stats` + +**Files:** +- Modify: `src/types/settings.ts` +- Modify: `src/services/sync/syncCategories.ts` + +- [ ] **Step 1: Add the category** + +In `src/types/settings.ts`, add `| 'stats'` to the `SyncCategory` union and `'stats',` to the `SYNC_CATEGORIES` array. + +In `src/services/sync/syncCategories.ts`, the `toCategory` mapper already returns a matching `SyncCategory` for ids present in `SYNC_CATEGORIES`, so `'stats'` maps to itself automatically. No change needed unless a legacy alias is introduced. + +- [ ] **Step 2: Lint** + +Run: `pnpm lint` +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/types/settings.ts src/services/sync/syncCategories.ts +git commit -m "feat(stats): add 'stats' sync category (default on)" +``` + +--- + +### Task 10: App-side push/pull (`statsSync.ts`) + +**Files:** +- Create: `src/services/statistics/statsSync.ts` +- Test: `src/__tests__/statistics/statsSync.test.ts` + +- [ ] **Step 1: Write the failing test** (push reads new events; pull applies + advances cursor) + +Create `src/__tests__/statistics/statsSync.test.ts`: + +```typescript +import { describe, it, expect, vi } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import { StatisticsDb } from '@/services/statistics/statisticsDb'; +import { pushStats, pullStats } from '@/services/statistics/statsSync'; + +async function db() { + const d = await NodeDatabaseService.open(':memory:'); + await migrate(d, getMigrations('statistics')); + return StatisticsDb.from(d); +} + +describe('statsSync', () => { + it('pushes only events past the cursor and advances it', async () => { + const stats = await db(); + const id = await stats.upsertBook({ bookMd5: 'm', title: 'T', authors: 'A' }); + await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 }); + await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 6, totalPages: 9 }); + const client = { pushChanges: vi.fn().mockResolvedValue({}) }; + await pushStats(stats, client as never); + const sent = client.pushChanges.mock.calls[0]![0]; + expect(sent.statPages.map((p: { start_time: number }) => p.start_time)).toEqual([100, 200]); + expect(await stats.getCursor('push')).toBe(200); + }); + + it('applies pulled events and advances the pull cursor', async () => { + const stats = await db(); + const client = { + pullChanges: vi.fn().mockResolvedValue({ + statBooks: [{ book_hash: 'm', title: 'T', authors: 'A' }], + statPages: [{ book_hash: 'm', page: 1, start_time: 300, duration: 7, total_pages: 9, updated_at_ms: 1_750_000_000_000 }], + }), + }; + await pullStats(stats, client as never); + const book = await stats.getBookByMd5('m'); + expect(book!.total_read_time).toBe(7); + expect(await stats.getCursor('pull')).toBe(1_750_000_000_000); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `pnpm test src/__tests__/statistics/statsSync.test.ts` +Expected: FAIL — `@/services/statistics/statsSync` does not exist. + +- [ ] **Step 3: Implement `statsSync`** + +Create `src/services/statistics/statsSync.ts`: + +```typescript +import type { StatisticsDb } from './statisticsDb'; +import type { SyncClient, StatPageRecord, StatBookRecord } from '@/libs/sync'; +import type { PageStatEvent, StatBook } from '@/types/statistics'; + +type PushClient = Pick; +type PullClient = Pick; + +const toWirePage = (e: PageStatEvent): StatPageRecord => ({ + book_hash: e.bookMd5, page: e.page, start_time: e.startTime, + duration: e.duration, total_pages: e.totalPages, +}); +const toWireBook = (b: StatBook): StatBookRecord => ({ + book_hash: b.bookMd5, title: b.title, authors: b.authors, +}); + +/** Push local events newer than the push cursor; advance it to the max start_time sent. */ +export async function pushStats(stats: StatisticsDb, client: PushClient): Promise { + const cursor = await stats.getCursor('push'); + const { events, books } = await stats.getEventsForPush(cursor); + if (events.length === 0) return; + await client.pushChanges({ statBooks: books.map(toWireBook), statPages: events.map(toWirePage) }); + const maxStart = events.reduce((m, e) => Math.max(m, e.startTime), cursor); + await stats.setCursor('push', maxStart); +} + +/** Pull events since the pull cursor; apply + advance cursor to newest updated_at. */ +export async function pullStats(stats: StatisticsDb, client: PullClient): Promise { + const since = await stats.getCursor('pull'); + const res = await client.pullChanges(since, 'stats'); + const wireBooks = (res.statBooks ?? []) as StatBookRecord[]; + const wirePages = (res.statPages ?? []) as StatPageRecord[]; + const books: StatBook[] = wireBooks.map((b) => ({ bookMd5: b.book_hash, title: b.title, authors: b.authors })); + const events: PageStatEvent[] = wirePages.map((p) => ({ + bookMd5: p.book_hash, page: p.page, startTime: p.start_time, duration: p.duration, totalPages: p.total_pages, + })); + await stats.applyRemoteEvents(books, events); + // The server attaches updated_at_ms (epoch ms) precisely so the cursor is a + // plain number both JS and the Lua koplugin can advance the same way. + const newest = wirePages.reduce((m, p) => Math.max(m, p.updated_at_ms ?? 0), since); + if (newest > since) await stats.setCursor('pull', newest); +} +``` + +Note: `pullChanges(since, type)` matches the existing `SyncClient.pullChanges(since, type, book?, metaHash?)` signature in `src/libs/sync.ts`. If the real signature differs, adapt the call (the test mocks it, so align the test's mock with the real signature). + +- [ ] **Step 4: Run to verify it passes** + +Run: `pnpm test src/__tests__/statistics/statsSync.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/services/statistics/statsSync.ts src/__tests__/statistics/statsSync.test.ts +git commit -m "feat(stats): app-side stats push/pull over /api/sync" +``` + +--- + +### Task 11: Trigger sync from the tracker + +**Files:** +- Modify: `src/app/reader/components/ReadingStatsTracker.tsx` + +- [ ] **Step 1: Wire pull-on-open + debounced push-after-persist** + +Add imports to `ReadingStatsTracker.tsx`: + +```tsx +import { SyncClient } from '@/libs/sync'; +import { pushStats, pullStats } from '@/services/statistics/statsSync'; +import { isSyncCategoryEnabled } from '@/services/sync/syncCategories'; +import { useAuth } from '@/context/AuthContext'; +``` + +Inside the component, read auth and add a push-debounce ref: + +```tsx +const { user } = useAuth(); +const pushTimerRef = useRef | null>(null); + +const syncEnabled = () => !!user && isSyncCategoryEnabled('stats'); + +const schedulePush = () => { + if (!syncEnabled()) return; + if (pushTimerRef.current) clearTimeout(pushTimerRef.current); + pushTimerRef.current = setTimeout(() => { + const db = dbRef.current; + if (db) void pushStats(db, new SyncClient()); + }, 10_000); // trailing debounce so rapid page turns don't spam the endpoint +}; +``` + +In the open effect, after `dbRef.current = db`, pull once: + +```tsx +StatisticsDb.open(appService).then((db) => { + if (cancelled) return; + dbRef.current = db; + if (syncEnabled()) void pullStats(db, new SyncClient()); +}); +``` + +At the end of `persist`'s async block (after `recomputeBookTotals`), call `schedulePush()`. In the unmount cleanup, also flush a push: `if (pushTimerRef.current) clearTimeout(pushTimerRef.current);` then `if (syncEnabled() && dbRef.current) void pushStats(dbRef.current, new SyncClient());`. + +Every network call is guarded by `syncEnabled()` — logged-out users and users who disabled the `stats` category still get local-only recording. + +- [ ] **Step 2: Lint + manual two-client smoke** + +Run: `pnpm lint` — PASS. Then optionally verify with two logged-in dev sessions that events created in one appear in the other after pull. + +- [ ] **Step 3: Commit** + +```bash +git add src/app/reader/components/ReadingStatsTracker.tsx +git commit -m "feat(stats): sync reading stats across Readest devices" +``` + +### ▣ Phase 2 checkpoint + +`pnpm test` + `pnpm lint` green. Readest↔Readest stats sync works. **Stop and review before Phase 3.** + +--- + +# Phase 3 — KOReader plugin sync (`apps/readest.koplugin`) + +Adds `readest_syncstats.lua` so KOReader devices push/pull the same `type=stats` endpoint, reading/writing their native `statistics.sqlite3`. + +## File structure (Phase 3) + +- Create `apps/readest.koplugin/readest_syncstats.lua` — push/pull modeled on `readest_syncconfig.lua`. +- Modify `apps/readest.koplugin/main.lua` — wire syncstats into the sync flow + menu. +- Create `apps/readest.koplugin/spec/syncstats_spec.lua` — busted unit tests. +- i18n: run `/i18n-koplugin` to extract new `_()` strings. + +--- + +### Task 12: `readest_syncstats.lua` + +**Files:** +- Create: `apps/readest.koplugin/readest_syncstats.lua` +- Reference: `apps/readest.koplugin/readest_syncconfig.lua` (push/pull shape, auth-fail handling), `apps/readest.koplugin/readest_syncclient.lua` (client API), KOReader `plugins/statistics.koplugin/main.lua` (the `statistics.sqlite3` location + schema) + +KOReader's stats DB lives at `DataStorage:getSettingsDir() .. "/statistics.sqlite3"`. The plugin opens it read/write with the bundled `lua-ljsqlite3` (`require("lua-ljsqlite3/init")`), the same module KOReader's statistics plugin uses. + +- [ ] **Step 1: Implement the module** + +Create `apps/readest.koplugin/readest_syncstats.lua`: + +```lua +local DataStorage = require("datastorage") +local InfoMessage = require("ui/widget/infomessage") +local UIManager = require("ui/uimanager") +local SQ3 = require("lua-ljsqlite3/init") +local _ = require("readest_i18n") + +local SyncStats = {} + +local function db_path() + return DataStorage:getSettingsDir() .. "/statistics.sqlite3" +end + +-- Read book md5/title/authors + page events with start_time > cursor. +-- Uses prepare/reset/bind/step — the same idiom as plugins/statistics.koplugin +-- (`stmt:reset():bind(...):step()`, step() returns a 1-indexed row or nil). +function SyncStats:collectSince(cursor) + local conn = SQ3.open(db_path()) + local books, pages, seen = {}, {}, {} + local stmt = conn:prepare([[ + SELECT b.md5, b.title, b.authors, p.page, p.start_time, p.duration, p.total_pages + FROM page_stat_data p JOIN book b ON b.id = p.id_book + WHERE p.start_time > ? ORDER BY p.start_time ASC]]) + stmt:reset():bind(tonumber(cursor) or 0) + local row = stmt:step() + while row ~= nil do + local md5 = row[1] + if md5 and not seen[md5] then + seen[md5] = true + table.insert(books, { book_hash = md5, title = row[2] or "", authors = row[3] or "" }) + end + table.insert(pages, { + book_hash = md5, + page = tonumber(row[4]), + start_time = tonumber(row[5]), + duration = tonumber(row[6]), + total_pages = tonumber(row[7]), + }) + row = stmt:step() + end + stmt:close() + conn:close() + return books, pages +end + +-- Upsert pulled rows into the local statistics.sqlite3 (union / longer-duration). +function SyncStats:applyRemote(books, pages) + local conn = SQ3.open(db_path()) + conn:exec("BEGIN;") + local insert_book = conn:prepare("INSERT OR IGNORE INTO book (title, authors, md5) VALUES (?, ?, ?);") + for _, b in ipairs(books or {}) do + insert_book:reset():bind(b.title or "", b.authors or "", b.book_hash):step() + end + insert_book:close() + local find_id = conn:prepare("SELECT id FROM book WHERE md5 = ? LIMIT 1;") + local insert_page = conn:prepare([[ + INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id_book, page, start_time) + DO UPDATE SET duration = max(duration, excluded.duration);]]) + local id_cache = {} + for _, p in ipairs(pages or {}) do + local id = id_cache[p.book_hash] + if not id then + local r = find_id:reset():bind(p.book_hash):step() + if r ~= nil then id = tonumber(r[1]); id_cache[p.book_hash] = id end + end + if id then + insert_page:reset():bind(id, p.page, p.start_time, p.duration, p.total_pages):step() + end + end + find_id:close() + insert_page:close() + conn:exec("COMMIT;") + conn:close() +end + +function SyncStats:push(settings, client, interactive) + local cursor = settings:readSetting("stats_push_cursor") or 0 + local books, pages = self:collectSince(cursor) + if #pages == 0 then return end + local max_start = cursor + for _, p in ipairs(pages) do if p.start_time > max_start then max_start = p.start_time end end + client:pushChanges( + { statBooks = books, statPages = pages }, + function(success) + if success then + settings:saveSetting("stats_push_cursor", max_start) + elseif interactive then + UIManager:show(InfoMessage:new{ text = _("Failed to push reading statistics"), timeout = 2 }) + end + end) +end + +function SyncStats:pull(settings, client, interactive, logout_fn) + local since = settings:readSetting("stats_pull_cursor") or 0 + -- pullChanges requires since/type/book/meta_hash params (readest-sync-api.json). + client:pullChanges( + { since = since, type = "stats", book = "", meta_hash = "" }, + function(success, response, status) + if not success then + if status == 401 or status == 403 then + if logout_fn then logout_fn() end + end + if interactive then + UIManager:show(InfoMessage:new{ text = _("Failed to pull reading statistics"), timeout = 2 }) + end + return + end + self:applyRemote(response.statBooks, response.statPages) + local newest = since + for _, p in ipairs(response.statPages or {}) do + local u = tonumber(p.updated_at_ms) or 0 + if u > newest then newest = u end + end + if newest > since then settings:saveSetting("stats_pull_cursor", newest) end + end) +end + +return SyncStats +``` + +API note: `SQ3.open` / `conn:exec` / `conn:prepare` / `stmt:reset():bind(...):step()` and 1-indexed row access are exactly the idioms in KOReader's `plugins/statistics.koplugin/main.lua` (the authoritative example) and are supported by the busted SQ3 shim in `spec/spec_helper.lua`. No `SQ3.quote` is used. + +- [ ] **Step 1b: Declare the new payload fields in the Spore spec** + +`client:pushChanges` only sends body keys listed in the Spore spec's `payload`. In `apps/readest.koplugin/readest-sync-api.json`, extend the `pushChanges` entry: + +```json +"pushChanges": { + "path": "/sync", + "method": "POST", + "required_params": ["books", "notes", "configs"], + "payload": ["books", "notes", "configs", "statBooks", "statPages"], + "expected_status": [200, 201, 301, 400, 401, 403] +} +``` + +(`pullChanges` already passes `type`/`since`/`book`/`meta_hash` as params, so `type=stats` needs no spec change.) + +- [ ] **Step 2: Wire into `main.lua`** + +In `apps/readest.koplugin/main.lua`, find where `readest_syncconfig` is required and invoked during the sync cycle and add the parallel calls: + +```lua +local SyncStats = require("readest_syncstats") +-- in the push path: +SyncStats:push(self.settings, self.sync_client, interactive) +-- in the pull path (on sync / on book open): +SyncStats:pull(self.settings, self.sync_client, interactive, logout_fn) +``` + +Add a menu toggle "Sync reading statistics" mirroring the existing config/annotations sync menu entries. + +- [ ] **Step 3: Extract i18n** + +Run: `/i18n-koplugin` (or `node apps/readest.koplugin/scripts/extract-i18n.js`) to sync the `.po` catalogs with the new `_()` strings. + +- [ ] **Step 4: Commit** + +```bash +git add apps/readest.koplugin/readest_syncstats.lua apps/readest.koplugin/main.lua \ + apps/readest.koplugin/readest-sync-api.json apps/readest.koplugin/locales +git commit -m "feat(koplugin): sync reading statistics with Readest" +``` + +--- + +### Task 13: koplugin busted specs + +**Files:** +- Create: `apps/readest.koplugin/spec/syncstats_spec.lua` +- Reference: `apps/readest.koplugin/spec/syncannotations_spec.lua` (mocking style, spec_helper) + +- [ ] **Step 1: Write the spec** + +`spec/spec_helper.lua` already shims `lua-ljsqlite3/init` over `lsqlite3complete` and fakes `DataStorage` with a temp settings dir. So the spec seeds a real on-disk `statistics.sqlite3` at `DataStorage:getSettingsDir() .. "/statistics.sqlite3"` and drives the module directly. + +Create `apps/readest.koplugin/spec/syncstats_spec.lua`: + +```lua +require("spec_helper") + +-- Minimal KOReader UI stubs the module pulls at require-time. +package.preload["ui/widget/infomessage"] = function() return { new = function() return {} end } end +package.preload["ui/uimanager"] = function() return { show = function() end } end +package.preload["readest_i18n"] = function() return function(s) return s end end + +local SQ3 = require("lua-ljsqlite3/init") +local DataStorage = require("datastorage") + +local function statsDbPath() + return DataStorage:getSettingsDir() .. "/statistics.sqlite3" +end + +local function seedDb() + local conn = SQ3.open(statsDbPath()) + conn:exec([[ + CREATE TABLE IF NOT EXISTS book ( + id integer PRIMARY KEY autoincrement, title text, authors text, notes integer, + last_open integer, highlights integer, pages integer, series text, language text, + md5 text, total_read_time integer, total_read_pages integer); + CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); + CREATE TABLE IF NOT EXISTS page_stat_data ( + id_book integer, page integer NOT NULL DEFAULT 0, start_time integer NOT NULL DEFAULT 0, + duration integer NOT NULL DEFAULT 0, total_pages integer NOT NULL DEFAULT 0, + UNIQUE (id_book, page, start_time)); + ]]) + conn:close() +end + +describe("readest_syncstats", function() + local SyncStats + + before_each(function() + os.remove(statsDbPath()) + seedDb() + package.loaded["readest_syncstats"] = nil + SyncStats = require("readest_syncstats") + end) + + it("collects only events past the cursor, joined with md5", function() + local conn = SQ3.open(statsDbPath()) + conn:exec("INSERT INTO book (title, authors, md5) VALUES ('T', 'A', 'md5-1');") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 1, 100, 5, 9);") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 2, 200, 6, 9);") + conn:close() + + local books, pages = SyncStats:collectSince(150) + assert.are.equal(1, #pages) + assert.are.equal(200, pages[1].start_time) + assert.are.equal("md5-1", pages[1].book_hash) + assert.are.equal("md5-1", books[1].book_hash) + end) + + it("keeps the longer duration when applying remote events", function() + SyncStats:applyRemote( + { { book_hash = "md5-2", title = "T2", authors = "A2" } }, + { + { book_hash = "md5-2", page = 1, start_time = 300, duration = 8, total_pages = 20 }, + { book_hash = "md5-2", page = 1, start_time = 300, duration = 20, total_pages = 20 }, + }) + + local conn = SQ3.open(statsDbPath()) + local count = conn:rowexec("SELECT COUNT(*) FROM page_stat_data;") + local dur = conn:rowexec("SELECT duration FROM page_stat_data WHERE start_time = 300;") + conn:close() + assert.are.equal(1, tonumber(count)) + assert.are.equal(20, tonumber(dur)) + end) +end) +``` + +If the local `lsqlite3complete` build rejects `ON CONFLICT ... DO UPDATE` (very old SQLite), report it — do not weaken the merge; bump the test SQLite instead. + +- [ ] **Step 2: Run the Lua tests** + +Run: `pnpm test:lua` +Expected: PASS (soft-skips if luajit/busted unavailable — then run on a machine that has them). + +- [ ] **Step 3: Lint Lua** + +Run: `pnpm lint:lua` +Expected: PASS. + +- [ ] **Step 4: Commit** + +```bash +git add apps/readest.koplugin/spec/syncstats_spec.lua +git commit -m "test(koplugin): cover readest_syncstats collect/apply/cursor" +``` + +### ▣ Phase 3 checkpoint + +`pnpm test` + `pnpm lint` + `pnpm lint:lua` + `pnpm test:lua` green. KOReader devices now sync stats with Readest end-to-end. + +--- + +## Final verification + +- [ ] `pnpm test` — all unit tests pass. +- [ ] `pnpm lint` — Biome + tsgo clean. +- [ ] `pnpm lint:lua` + `pnpm test:lua` — koplugin Lua clean (Phase 3). +- [ ] End-to-end: read on Readest device A → stats appear on Readest device B after pull; read on a KOReader device → events appear in Readest after the koplugin pushes; and a Readest-originated book's events show up in KOReader's own statistics view. +- [ ] Remove PR #3156 leftovers if present in the branch base: `better-sqlite3` / `@types/better-sqlite3` deps, the `onlyBuiltDependencies` block, `statistics.json` code, and the session-based `statisticsStore`/`BookSessionTracker` if they were merged. diff --git a/apps/readest-app/docs/superpowers/specs/2026-06-15-koreader-stats-sync-design.md b/apps/readest-app/docs/superpowers/specs/2026-06-15-koreader-stats-sync-design.md new file mode 100644 index 00000000..c089befd --- /dev/null +++ b/apps/readest-app/docs/superpowers/specs/2026-06-15-koreader-stats-sync-design.md @@ -0,0 +1,189 @@ +# KOReader-compatible reading statistics with cross-device sync + +- **Date:** 2026-06-15 +- **Status:** Approved design — ready for implementation plan +- **Supersedes:** PR #3156 (`feat(statistics): add core session tracking (Phase 1)`) and tracking issue #3155 +- **Related:** KOReader `statistics.koplugin`, Readest `apps/readest.koplugin`, legacy `/api/sync` + +## 1. Summary + +Add a reading-statistics system to Readest whose **canonical data model is KOReader's own**, so stats round-trip losslessly between Readest and KOReader. Stats are stored locally in a cross-platform Turso `statistics.db` (KOReader schema), tracked as per-page reading events, and synced across all of a user's devices — Readest apps **and** KOReader devices (via `apps/readest.koplugin`) — over the existing `/api/sync` transport. + +This replaces PR #3156's approach (session aggregates in a `statistics.json` written with the Node-only `better-sqlite3`) which cannot run on web/mobile and is structurally incompatible with KOReader's per-page model. + +## 2. Goals / Non-goals + +### Goals + +- Local stats persisted in a **Turso `statistics.db`** via the existing cross-platform `DatabaseService` (works on web Workers, desktop, iOS, Android). +- Local schema is **byte-for-byte KOReader-compatible** (`book`, `page_stat_data`, `page_stat` view, `numbers`). +- **Per-page event tracking** in the Readest reader (KOReader-style time-on-page with idle capping). +- **Cross-device sync** of stats through the legacy `/api/sync` endpoint, as a new `stats` type backed by a new Supabase table. +- **KOReader devices participate** via a new `readest_syncstats.lua` in `apps/readest.koplugin`, reusing the plugin's existing sync client/auth. +- Stats sync is **union/append-only** and merges contributions from every Readest and KOReader device. +- A **KOReader-compatible enrichment seam** (extension tables + a nullable `ext` field) so Readest can add richer-than-KOReader data later without breaking compatibility — **seam-only in v1, no data captured**. + +### Non-goals (explicitly deferred) + +- The statistics **UI page** (charts, calendar, streaks) — a later phase. This design only produces the data + sync substrate; the UI reads it via SQL. +- Reading **goals**. +- Routing stats through the HLC **replica CRDT** (`/api/sync/replicas`). Stats use the simpler legacy transport the koplugin already speaks; union-by-key needs no CRDT. +- **Capturing** enrichment data (chapter attribution, words/wpm, device). The extension seam is built (§5.6) but stays empty until the stats-UI phase. +- Migrating any PR #3156 `statistics.json` data (PR is unmerged; no field data exists). + +## 3. Background + +### 3.1 KOReader's model (the compatibility target) + +`statistics.sqlite3` (verified against the sample DB at +`koreader-emulator-.../settings/statistics.sqlite3`): + +```sql +CREATE TABLE book ( + id integer PRIMARY KEY autoincrement, + title text, authors text, notes integer, last_open integer, + highlights integer, pages integer, series text, language text, + md5 text, total_read_time integer, total_read_pages integer ); +CREATE UNIQUE INDEX book_title_authors_md5 ON book(title, authors, md5); + +CREATE TABLE page_stat_data ( + id_book integer, page integer NOT NULL DEFAULT 0, + start_time integer NOT NULL DEFAULT 0, duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + UNIQUE (id_book, page, start_time), + FOREIGN KEY(id_book) REFERENCES book(id) ); +CREATE INDEX page_stat_data_start_time ON page_stat_data(start_time); + +CREATE TABLE numbers (number INTEGER PRIMARY KEY); -- rescale helper for the view +CREATE VIEW page_stat AS ...; -- rescales rows when total_pages changes +``` + +Key facts: + +- The unit of truth is the **per-page read event** in `page_stat_data`: "on `page` of a book that had `total_pages`, starting at Unix-second `start_time`, you read for `duration` seconds." Events are **immutable and append-only**; `(id_book, page, start_time)` is unique. +- `book.total_read_time` / `total_read_pages` are **derived caches** of the page events. +- Book identity is `md5 = util.partialMD5(file)` — a non-uniform sampled MD5 (steps `1024 << (2*i)`, 1 KiB samples, `i = -1..10`). +- KOReader's own cross-device sync is a 3-way SQL merge of the whole `statistics.sqlite3` file over cloud storage, unioning `page_stat_data` keyed on `(title, authors, md5)` → `(page, start_time)`. **We reproduce the union semantics, not the file transport.** + +### 3.2 Readest infrastructure we build on + +- **Cross-platform Turso layer:** `appService.openDatabase(schema, path, base, opts)` → `DatabaseService` (`execute` / `select` / `batch` / `close`), with an auto-migration system (`migrate.ts`, `getMigrations(schema)`). Already used by `reedy.db`, `opds.db`, `library.db`. Native = `tauri-plugin-turso`; web = `@readest/turso-database-wasm` (OPFS); node = `@tursodatabase/database`. +- **`partialMD5` already equals KOReader's** (`src/utils/md5.ts::partialMD5`, byte-identical algorithm) and is already `Book.hash`. **Book identity matches KOReader with no extra work.** +- **Progress model:** `BookProgress.pageinfo = { current /*0-based*/, total }` + `location` (CFI) — enough to emit page events. +- **Legacy sync `/api/sync`:** `SyncType = 'books' | 'configs' | 'notes'`, Supabase-backed, pull by `since` timestamp (+ optional `book`/`meta_hash`), push `{ books, notes, configs }`. Records keyed by `(user_id, book_hash, meta_hash)` with `updated_at` / `deleted_at`. **This is the endpoint `apps/readest.koplugin` already uses** (`readest-sync-api.json`, `readest_syncconfig.lua`). +- **Reader lifecycle hook point:** PR #3156's `BookSessionTracker` mounts per `bookKey` (in `BooksGrid` / `ReaderContent`) and already wires idle timers + `visibilitychange`. We keep this shell and change what it records. + +## 4. Architecture + +``` +┌─ Readest app (web/desktop/mobile) ─┐ ┌─ KOReader device ──────────┐ +│ Turso statistics.db │ │ statistics.sqlite3 │ +│ (KOReader schema) │ │ (native) │ +│ ▲ page-event tracker │ │ ▲ KOReader writes │ +└────────┼───────────────────────────┘ └─────┼──────────────────────┘ + │ push/pull type=stats │ readest_syncstats.lua + └──────────────┬────────────────────────────┘ (push/pull type=stats) + ▼ + /api/sync (Supabase: book_page_stats) + upsert/union by (user_id, book_hash, page, start_time) +``` + +- **One canonical format** (KOReader's). **One transport** (legacy `/api/sync`). **One merge rule** (union/upsert). +- **The stats payload is self-contained** — it carries both `book` metadata records and `page_stat_data` event records, exactly like KOReader's `statistics.sqlite3` bundles `book` + `page_stat_data`. No join against the `books` sync table; a stats consumer (Readest app or koplugin) can rebuild a complete `statistics.sqlite3` from the stats payload alone. +- **Everything is keyed by the stable `book_hash`, never the local autoincrement `id_book`** (which is per-device). Each device maps `book_hash` → its own local `id_book` on import. +- **Synced book metadata is trimmed to KOReader's identity triple: `md5` + `title` + `authors`.** Everything else in KOReader's `book` table is **derived locally** and never synced: `pages` = latest event's `total_pages`, `last_open` = `max(start_time + duration)`, `total_read_time` / `total_read_pages` summed from events; `series` / `language` / `notes` / `highlights` are left NULL/0 (valid KOReader schema, filled by KOReader itself when it opens the book). This keeps the local `statistics.db` a valid KOReader export while syncing only the irreducible identity. +- Derived analytics (sessions, daily summaries, streaks, hour/day heatmaps) are **SQL queries over `page_stat_data`**, computed on demand by the future UI — nothing extra stored or synced. + +## 5. Component design + +### 5.1 Local store — `src/services/statistics/` + +- New `'statistics'` migration schema registering the KOReader DDL verbatim (book, page_stat_data, page_stat view, numbers, indexes). Registered in `getMigrations('statistics')`. +- `statisticsDb.ts`: opens `statistics.db` via `appService.openDatabase('statistics', 'statistics.db', 'Data')`; thin typed helpers: + - `upsertBook(meta) → id_book` (upsert by `(title, authors, md5)`). + - `insertPageEvent(id_book, page, start_time, duration, total_pages)` — upsert on the unique key `(id_book, page, start_time)` via `ON CONFLICT … DO UPDATE SET duration = max(duration, excluded.duration), total_pages = excluded.total_pages` (a re-flush of the same page-read with a longer duration wins; a return visit at a different `start_time` is a distinct row, as in KOReader). + - `recomputeBookTotals(id_book)`. + - `getEventsSince(ts)` / `applyRemoteEvents(events)` for sync. +- `numbers` is seeded to a fixed range (KOReader seeds 1..1000) so the `page_stat` view's rescale join works. + +### 5.2 Tracker — replace PR's session logic in `BookSessionTracker.tsx` + +Keep the mount-per-`bookKey` shell + idle/visibility wiring. Record **page events**, not sessions: + +- State per book: `currentPage`, `pageEnterTime` (Unix s), `totalPages`. +- **Flush a page event** `(page=currentPage, start_time=pageEnterTime, duration=cappedElapsed, total_pages)` on: page change, idle timeout, tab hidden (`visibilitychange`), and book close (unmount). **No periodic tick** — every flushed event is final/immutable (its `start_time` and `duration` never change afterward), which lets cross-device sync use a simple `start_time` high-water push cursor. An idle timeout flushes-and-pauses; resuming the same page starts a fresh event with a new `start_time`. +- **Duration cap:** `duration = min(now - pageEnterTime, maxEventSeconds)`, mirroring KOReader's per-page cap (default 120 s; configurable). Events shorter than a small floor are dropped (KOReader ignores sub-second flips). +- `page = pageinfo.current + 1`, `total_pages = pageinfo.total`. Units = Unix **seconds** throughout (matches KOReader; the PR already converts). +- On first event for a book, `upsertBook` from library metadata (title/authors/pages/series/language/md5=Book.hash). + +### 5.3 Sync — server (`/api/sync`, Supabase) + +The stats type carries **two self-contained record sets** (mirroring KOReader's `book` + `page_stat_data`), both keyed by `book_hash`: + +- `stat_books` — identity only: + `user_id, book_hash, title, authors, updated_at, deleted_at`, + primary key `(user_id, book_hash)`, index on `updated_at`. (No `pages` / `last_open` / `series` / `language` / totals — derived locally.) +- `stat_pages` — page events: + `user_id, book_hash, page, start_time, duration, total_pages, updated_at, deleted_at`, + primary key `(user_id, book_hash, page, start_time)`, index on `updated_at`. +- Extend `SyncType` with `'stats'`. GET returns `{ statBooks: [...], statPages: [...] }` changed since `since` (optional `book` filter). POST upserts both: + - `stat_pages`: `ON CONFLICT … DO UPDATE SET duration = greatest(duration, excluded.duration), total_pages = excluded.total_pages, updated_at = now()`. + - `stat_books`: `ON CONFLICT (user_id, book_hash) DO UPDATE SET title = excluded.title, authors = excluded.authors, updated_at = now()` (latest title/authors win — handles a renamed book). + - Deletes (a book's stats cleared) set `deleted_at` on both. +- No dependency on the `books` table — stats are fully self-describing. + +### 5.4 Sync — Readest app client (`src/services/sync/` + statistics service) + +- After a flush batch, **push** new page events **and the book-metadata record** (`type=stats`). On reader open and on a periodic timer, **pull** `since` the stored cursor → apply both `statBooks` (upsert local `book`) and `statPages` (upsert `page_stat_data`), then `recomputeBookTotals`. Reuse the existing cursor/`since` machinery used by `configs`. +- Gate behind a new `stats` entry in `syncCategories` (default **on**). + +### 5.5 Sync — koplugin (`apps/readest.koplugin/readest_syncstats.lua`) + +- Modeled on `readest_syncconfig.lua`; uses the existing `readest_syncclient` + auth. +- **Push:** read new `page_stat_data` rows plus each book's `md5` / `title` / `authors` from KOReader's `statistics.sqlite3` (`md5` via `partial_md5_checksum`), send both record sets as `type=stats`. +- **Pull:** fetch `since` cursor; upsert incoming `statBooks` (`md5` + `title` + `authors`) into the local `book` table — creating the row when absent, KOReader fills the rest when the book is opened — and `statPages` into `page_stat_data` (mapping `book_hash` → local `id_book`), then let KOReader recompute totals. +- Add to the plugin's sync menu + i18n (follow `/i18n-koplugin`); cover with `busted` specs. + +### 5.6 Readest enrichment seam (KOReader-compatible, seam-only in v1) + +Readest may later want per-event data KOReader's page-based schema can't represent (chapter/section attribution, words-read → wpm, originating device). The bridge being the **sync API, not a shared file**, makes this additive: KOReader reads only its own `statistics.sqlite3` (written by the koplugin in KOReader's exact schema), so it never sees Readest extras. v1 **builds the seam and captures nothing** — the extension columns/tables exist but stay NULL/empty until a later phase wires up the stats UI. + +- **Local:** enrichment lives in **separate `readest_*` extension tables** (e.g. `readest_page_ext(book_hash, page, start_time, …)`, `readest_book_ext(book_hash, …)`) keyed by the same identity. KOReader's `book` / `page_stat_data` tables stay **byte-identical** (a literal file export stays valid too). +- **Wire:** a nullable **`ext jsonb`** column on `stat_pages` / `stat_books`. Readest reads/writes it; the koplugin **leaves it NULL and ignores incoming `ext`**. +- **Compatibility invariants:** (1) the koplugin maps only KOReader-schema fields; (2) those round-trip losslessly. Enrichment is therefore **lossless Readest↔Readest** and **dropped only at the KOReader boundary** (inherent — KOReader's schema can't hold it). + +## 6. KOReader interop details + +- **Book identity:** Readest `Book.hash` = `partialMD5(file)` = KOReader `md5`. The Supabase key is `book_hash`. No re-hashing. +- **Units:** Unix **seconds** for `start_time` and `duration`; same as KOReader. +- **`total_pages`/repagination:** stored per event; KOReader's `page_stat` view already rescales when `total_pages` differs, so differing pagination between sessions/engines is tolerated by the view, not by us. +- **Book-row reconstruction is self-contained:** incoming `statBooks` records carry `md5` + `title` + `authors`, so a pulling device creates its local `book` row directly from the payload — no lookup against the library or `books` sync. `pages` and `last_open` are filled from the book's page events; `series` / `language` stay NULL. The Readest app still enriches its *own* book rows from library metadata when it originates them. + +## 7. Known limitations + +Readest (foliate) and KOReader paginate the same EPUB differently, so per-page **coordinates differ across engines**. Consequences: + +- **Total reading *time* is exact** (sum of `duration`) — the primary metric. +- Per-page heatmaps and **distinct-page counts mix coordinate systems** cross-engine and are approximate — the same approximation KOReader already lives with when `total_pages` changes (handled by the rescaling `page_stat` view). Within a single engine, fully exact. + +This is documented, accepted, and not a blocker. + +## 8. Testing strategy (test-first) + +- **Tracker flush logic** (vitest): page-change/idle/hidden/close each emit the right `(page, start_time, duration, total_pages)`; duration cap; sub-floor drop; idle-then-resume starts a fresh event; multi-book isolation. +- **statisticsDb** (vitest, node DB service): KOReader-schema round-trip; upsert-by-key keeps longer duration; `recomputeBookTotals`; `page_stat` view returns expected rescaled rows for a known fixture. +- **Sync endpoint** (vitest): `since` filtering, upsert/union, conflict-keeps-longer-duration, delete propagation. +- **KOReader fixture parity:** import the sample `statistics.sqlite3`, assert `book` + `page_stat_data` survive a Readest export/import round-trip unchanged. +- **koplugin** (`busted`): `readest_syncstats` push/pull/merge against an in-memory `statistics.sqlite3`; `pnpm lint:lua` + `pnpm test:lua`. +- Full `pnpm test` + `pnpm lint` green before completion. + +## 9. Rollout / compatibility + +- PR #3156 is unmerged → no `statistics.json` migration. This design lands as the first stats implementation. +- New Supabase table + migration; additive `SyncType` — no change to existing `books`/`notes`/`configs` flows. +- `better-sqlite3`, `@types/better-sqlite3`, and the `onlyBuiltDependencies`/`statistics.json` bits from PR #3156 are dropped. + +## 10. Open questions + +- **Duration-cap / idle-timeout defaults:** adopt KOReader's 120 s per-page cap and a 120 s idle timeout, both configurable later. (Assumed; confirm at implementation.) +- **Stats-sync encryption:** match whatever the legacy `configs` flow does today (no new E2E layer for stats in v1). Confirm against current `/api/sync` behavior when wiring the endpoint. diff --git a/apps/readest-app/src/__tests__/api/statsSyncMerge.test.ts b/apps/readest-app/src/__tests__/api/statsSyncMerge.test.ts new file mode 100644 index 00000000..babf44d2 --- /dev/null +++ b/apps/readest-app/src/__tests__/api/statsSyncMerge.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { pickWinningPages } from '@/pages/api/sync'; +import type { StatPageRecord } from '@/libs/sync'; + +const mk = (start: number, duration: number): StatPageRecord => ({ + book_hash: 'm', + page: 1, + start_time: start, + duration, + total_pages: 10, +}); + +describe('pickWinningPages', () => { + it('inserts pages the server has not seen', () => { + const { toUpsert } = pickWinningPages([mk(100, 5)], new Map()); + expect(toUpsert).toHaveLength(1); + }); + + it('keeps the longer duration on conflict', () => { + const server = new Map([['m|1|100', mk(100, 5)]]); + const win = pickWinningPages([mk(100, 9)], server); + expect(win.toUpsert[0]!.duration).toBe(9); + }); + + it('drops an incoming page whose duration is not longer', () => { + const server = new Map([['m|1|100', mk(100, 9)]]); + const win = pickWinningPages([mk(100, 5)], server); + expect(win.toUpsert).toHaveLength(0); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/syncCategories.test.ts b/apps/readest-app/src/__tests__/services/sync/syncCategories.test.ts index 5d2c5d36..b963c64d 100644 --- a/apps/readest-app/src/__tests__/services/sync/syncCategories.test.ts +++ b/apps/readest-app/src/__tests__/services/sync/syncCategories.test.ts @@ -149,7 +149,7 @@ describe('isSyncCategoryEnabled', () => { }); describe('SYNC_CATEGORIES', () => { - test('covers all nine user-facing categories (incl. settings + credentials)', () => { + test('covers all ten user-facing categories (incl. settings + stats + credentials)', () => { expect([...SYNC_CATEGORIES].sort()).toEqual( [ 'book', @@ -160,6 +160,7 @@ describe('SYNC_CATEGORIES', () => { 'opds_catalog', 'progress', 'settings', + 'stats', 'texture', ].sort(), ); diff --git a/apps/readest-app/src/__tests__/statistics/statisticsDb.test.ts b/apps/readest-app/src/__tests__/statistics/statisticsDb.test.ts new file mode 100644 index 00000000..1259b9f9 --- /dev/null +++ b/apps/readest-app/src/__tests__/statistics/statisticsDb.test.ts @@ -0,0 +1,122 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import type { DatabaseService } from '@/types/database'; +import { StatisticsDb } from '@/services/statistics/statisticsDb'; + +async function freshStatsDb(): Promise { + // In-memory libsql DB; run the same migrations production uses. + const db = await NodeDatabaseService.open(':memory:'); + await migrate(db, getMigrations('statistics')); + return db; +} + +describe('statistics migration', () => { + let db: DatabaseService; + beforeEach(async () => { + db = await freshStatsDb(); + }); + + it('creates KOReader book + page_stat_data tables and extension tables', async () => { + const tables = await db.select<{ name: string }>( + `SELECT name FROM sqlite_master WHERE type IN ('table','view') ORDER BY name`, + ); + const names = tables.map((t) => t.name); + expect(names).toContain('book'); + expect(names).toContain('page_stat_data'); + expect(names).toContain('numbers'); + expect(names).toContain('page_stat'); // the rescaling view + expect(names).toContain('readest_page_ext'); + expect(names).toContain('readest_book_ext'); + expect(names).toContain('readest_stat_sync_state'); + }); + + it('seeds the numbers helper table 1..1000', async () => { + const rows = await db.select<{ c: number }>(`SELECT COUNT(*) AS c FROM numbers`); + expect(rows[0]!.c).toBe(1000); + }); + + it('enforces the page_stat_data uniqueness key', async () => { + await db.execute(`INSERT INTO book (title, authors, md5) VALUES ('T','A','m')`); + const id = (await db.select<{ id: number }>(`SELECT id FROM book LIMIT 1`))[0]!.id; + await db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (?,?,?,?,?)`, + [id, 5, 1000, 10, 100], + ); + await db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?,?,?,?,?) + ON CONFLICT(id_book, page, start_time) DO UPDATE SET duration = max(duration, excluded.duration)`, + [id, 5, 1000, 25, 100], + ); + const rows = await db.select<{ duration: number; c: number }>( + `SELECT duration, COUNT(*) OVER () AS c FROM page_stat_data`, + ); + expect(rows.length).toBe(1); + expect(rows[0]!.duration).toBe(25); + }); +}); + +describe('StatisticsDb', () => { + let stats: StatisticsDb; + beforeEach(async () => { + stats = StatisticsDb.from(await freshStatsDb()); + }); + + it('upserts a book by md5 and returns a stable id_book', async () => { + const id1 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + const id2 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + expect(id1).toBe(id2); + }); + + it('inserts page events and keeps the longer duration on re-flush', async () => { + const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 10, totalPages: 50 }); + await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 30, totalPages: 50 }); + await stats.insertPageEvent(id, { page: 4, startTime: 140, duration: 12, totalPages: 50 }); + await stats.recomputeBookTotals(id); + const book = await stats.getBookByMd5('m1'); + expect(book!.total_read_time).toBe(42); // 30 + 12 + expect(book!.total_read_pages).toBe(2); // distinct pages 3,4 + expect(book!.last_open).toBe(152); // max(start_time + duration) = 140 + 12 + }); + + it('returns events for push after a start_time cursor, joined with md5', async () => { + const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' }); + await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 }); + await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 5, totalPages: 9 }); + const { events } = await stats.getEventsForPush(150); + expect(events.map((e) => e.startTime)).toEqual([200]); + expect(events[0]!.bookMd5).toBe('m1'); + }); + + it('applies remote events idempotently via upsert', async () => { + const remoteBooks = [{ bookMd5: 'm2', title: 'T2', authors: 'A2' }]; + const remoteEvents = [ + { bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 }, + { bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 }, // dup + ]; + await stats.applyRemoteEvents(remoteBooks, remoteEvents); + await stats.applyRemoteEvents(remoteBooks, remoteEvents); // again — still idempotent + const book = await stats.getBookByMd5('m2'); + expect(book!.total_read_time).toBe(8); + }); + + it('reads and writes sync cursors', async () => { + expect(await stats.getCursor('push')).toBe(0); + await stats.setCursor('push', 1234); + expect(await stats.getCursor('push')).toBe(1234); + }); + + it('keeps one book row per md5 even when title/authors change (no duplicates)', async () => { + const id1 = await stats.upsertBook({ bookMd5: 'm1', title: 'Old', authors: 'A' }); + const id2 = await stats.upsertBook({ bookMd5: 'm1', title: 'New', authors: 'B' }); + expect(id2).toBe(id1); + const book = await stats.getBookByMd5('m1'); + expect(book!.title).toBe('New'); // latest title wins + // exactly one row for this md5 + const rows = await stats.getEventsForPush(-1); // no events; just exercise no crash + void rows; + }); +}); diff --git a/apps/readest-app/src/__tests__/statistics/statsSync.test.ts b/apps/readest-app/src/__tests__/statistics/statsSync.test.ts new file mode 100644 index 00000000..45d5073c --- /dev/null +++ b/apps/readest-app/src/__tests__/statistics/statsSync.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, vi } from 'vitest'; +import { NodeDatabaseService } from '@/services/database/nodeDatabaseService'; +import { migrate } from '@/services/database/migrate'; +import { getMigrations } from '@/services/database/migrations'; +import { StatisticsDb } from '@/services/statistics/statisticsDb'; +import { pushStats, pullStats } from '@/services/statistics/statsSync'; + +async function db() { + const d = await NodeDatabaseService.open(':memory:'); + await migrate(d, getMigrations('statistics')); + return StatisticsDb.from(d); +} + +describe('statsSync', () => { + it('pushes only events past the cursor and advances it', async () => { + const stats = await db(); + const id = await stats.upsertBook({ bookMd5: 'm', title: 'T', authors: 'A' }); + await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 }); + await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 6, totalPages: 9 }); + const client = { pushChanges: vi.fn().mockResolvedValue({}) }; + await pushStats(stats, client as never); + const sent = client.pushChanges.mock.calls[0]![0]; + expect(sent.statPages.map((p: { start_time: number }) => p.start_time)).toEqual([100, 200]); + expect(await stats.getCursor('push')).toBe(200); + }); + + it('applies pulled events and advances the pull cursor', async () => { + const stats = await db(); + const client = { + pullChanges: vi.fn().mockResolvedValue({ + statBooks: [{ book_hash: 'm', title: 'T', authors: 'A' }], + statPages: [ + { + book_hash: 'm', + page: 1, + start_time: 300, + duration: 7, + total_pages: 9, + updated_at_ms: 1_750_000_000_000, + }, + ], + }), + }; + await pullStats(stats, client as never); + const book = await stats.getBookByMd5('m'); + expect(book!.total_read_time).toBe(7); + expect(await stats.getCursor('pull')).toBe(1_750_000_000_000); + }); + + it('chunks a large push backlog into bounded requests, advancing the cursor per chunk', async () => { + const stats = await db(); + const id = await stats.upsertBook({ bookMd5: 'm', title: 'T', authors: 'A' }); + // 600 events at distinct start_times — exceeds the 500-event push chunk → 2 requests. + for (let t = 1; t <= 600; t++) { + await stats.insertPageEvent(id, { page: t, startTime: t, duration: 1, totalPages: 999 }); + } + const client = { pushChanges: vi.fn().mockResolvedValue({}) }; + await pushStats(stats, client as never); + expect(client.pushChanges).toHaveBeenCalledTimes(2); + expect(client.pushChanges.mock.calls[0]![0].statPages.length).toBe(500); + expect(client.pushChanges.mock.calls[1]![0].statPages.length).toBe(100); + expect(await stats.getCursor('push')).toBe(600); + }); + + it('pages through a multi-page pull, applying every page until exhausted', async () => { + const stats = await db(); + const ev = (n: number, ms: number) => ({ + book_hash: 'm', + page: n, + start_time: n, + duration: 2, + total_pages: 9, + updated_at_ms: ms, + }); + const client = { + pullChanges: vi + .fn() + .mockResolvedValueOnce({ + statBooks: [{ book_hash: 'm', title: 'T', authors: 'A' }], + statPages: [ev(1, 1000), ev(2, 1000)], + }) + .mockResolvedValueOnce({ statBooks: [], statPages: [ev(3, 2000)] }) + .mockResolvedValue({ statBooks: [], statPages: [] }), + }; + await pullStats(stats, client as never); + expect(client.pullChanges).toHaveBeenCalledTimes(3); + const book = await stats.getBookByMd5('m'); + expect(book!.total_read_pages).toBe(3); // pages 1, 2, 3 all applied across pages + expect(await stats.getCursor('pull')).toBe(2000); + }); +}); diff --git a/apps/readest-app/src/__tests__/statistics/trackerCore.test.ts b/apps/readest-app/src/__tests__/statistics/trackerCore.test.ts new file mode 100644 index 00000000..c2a8a686 --- /dev/null +++ b/apps/readest-app/src/__tests__/statistics/trackerCore.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { TrackerCore } from '@/services/statistics/trackerCore'; +import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics'; + +const cfg = DEFAULT_STATS_TRACKING_CONFIG; + +describe('TrackerCore', () => { + it('emits an event when the page changes', () => { + const t = new TrackerCore(cfg); + expect(t.onPage(5, 100, 1000)).toEqual([]); // first page, nothing to flush yet + const out = t.onPage(6, 100, 1030); // moved off page 5 after 30s + expect(out).toEqual([{ page: 5, startTime: 1000, duration: 30, totalPages: 100 }]); + }); + + it('caps duration at maxEventSeconds', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + const out = t.onPage(2, 10, 10_000); // way over the 120s cap + expect(out[0]!.duration).toBe(cfg.maxEventSeconds); + }); + + it('drops sub-minimum events', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + expect(t.onPage(2, 10, 1)).toEqual([]); // 1s < minEventSeconds (3) + }); + + it('flushes and pauses on idle, then resumes with a new start_time', () => { + const t = new TrackerCore(cfg); + t.onPage(1, 10, 0); + expect(t.onIdle(50)).toEqual([{ page: 1, startTime: 0, duration: 50, totalPages: 10 }]); + // After idle, the same page resumes as a fresh event. + expect(t.onPage(1, 10, 200)).toEqual([]); // resume marker, no flush + expect(t.onPage(2, 10, 230)).toEqual([ + { page: 1, startTime: 200, duration: 30, totalPages: 10 }, + ]); + }); + + it('flushes on hide and on close without double-counting', () => { + const t = new TrackerCore(cfg); + t.onPage(7, 10, 0); + expect(t.onHide(40)).toEqual([{ page: 7, startTime: 0, duration: 40, totalPages: 10 }]); + expect(t.onClose(99)).toEqual([]); // already flushed + paused by hide + }); +}); diff --git a/apps/readest-app/src/app/reader/components/BooksGrid.tsx b/apps/readest-app/src/app/reader/components/BooksGrid.tsx index 9bb4ffc3..8313e832 100644 --- a/apps/readest-app/src/app/reader/components/BooksGrid.tsx +++ b/apps/readest-app/src/app/reader/components/BooksGrid.tsx @@ -25,6 +25,7 @@ import FootnotePopup from './FootnotePopup'; import HintInfo from './HintInfo'; import ReadingRuler from './ReadingRuler'; import DoubleBorder from './DoubleBorder'; +import ReadingStatsTracker from './ReadingStatsTracker'; interface BooksGridProps { bookKeys: string[]; @@ -278,6 +279,7 @@ const BookCellInner: React.FC = ({ isHoveredAnim={false} gridInsets={gridInsets} /> + ); }; diff --git a/apps/readest-app/src/app/reader/components/ReadingStatsTracker.tsx b/apps/readest-app/src/app/reader/components/ReadingStatsTracker.tsx new file mode 100644 index 00000000..01101031 --- /dev/null +++ b/apps/readest-app/src/app/reader/components/ReadingStatsTracker.tsx @@ -0,0 +1,121 @@ +'use client'; + +import { useEffect, useRef } from 'react'; +import { useBookProgress } from '@/store/readerProgressStore'; +import { useBookDataStore } from '@/store/bookDataStore'; +import { useEnv } from '@/context/EnvContext'; +import { useAuth } from '@/context/AuthContext'; +import { StatisticsDb } from '@/services/statistics/statisticsDb'; +import { TrackerCore, type FlushedEvent } from '@/services/statistics/trackerCore'; +import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics'; +import { SyncClient } from '@/libs/sync'; +import { pushStats, pullStats } from '@/services/statistics/statsSync'; +import { isSyncCategoryEnabled } from '@/services/sync/syncCategories'; + +const nowSec = () => Math.floor(Date.now() / 1000); + +export default function ReadingStatsTracker({ bookKey }: { bookKey: string }) { + const { appService } = useEnv(); + // Progress lives in readerProgressStore, not readerStore.viewStates. + const progress = useBookProgress(bookKey); + // booksData is keyed by book id = bookKey.split('-')[0]. + const getBookData = useBookDataStore((s) => s.getBookData); + const { user } = useAuth(); + const coreRef = useRef(new TrackerCore(DEFAULT_STATS_TRACKING_CONFIG)); + const dbRef = useRef(null); + const idleRef = useRef | null>(null); + const pushTimerRef = useRef | null>(null); + + const bookData = getBookData(bookKey); + const book = bookData?.book; + // Book.hash is the partialMD5 used as KOReader's md5. + const bookMd5 = book?.hash; + const title = book?.title ?? ''; + // Book.author is the single-string author field; upsertBook takes authors: string. + const authors = book?.author ?? ''; + + const syncEnabled = () => !!user && isSyncCategoryEnabled('stats'); + + const schedulePush = () => { + if (!syncEnabled()) return; + if (pushTimerRef.current) clearTimeout(pushTimerRef.current); + pushTimerRef.current = setTimeout(() => { + const db = dbRef.current; + if (db) void pushStats(db, new SyncClient()); + }, 10_000); + }; + + useEffect(() => { + if (!appService) return; + let cancelled = false; + StatisticsDb.open(appService).then((db) => { + if (cancelled) return; + dbRef.current = db; + if (syncEnabled()) void pullStats(db, new SyncClient()); + }); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [appService]); + + // Persist flushed events into the statistics DB. + const persist = (events: FlushedEvent[]): Promise => { + const db = dbRef.current; + if (!db || !bookMd5 || events.length === 0) return Promise.resolve(); + return (async () => { + const idBook = await db.upsertBook({ bookMd5, title, authors }); + for (const e of events) await db.insertPageEvent(idBook, e); + await db.recomputeBookTotals(idBook); + schedulePush(); + })(); + }; + + const armIdle = () => { + if (idleRef.current) clearTimeout(idleRef.current); + idleRef.current = setTimeout( + () => void persist(coreRef.current.onIdle(nowSec())), + DEFAULT_STATS_TRACKING_CONFIG.idleTimeoutSeconds * 1000, + ); + }; + + // Page changes drive the tracker. + useEffect(() => { + const info = progress?.pageinfo; + if (!info) return; + const page = (info.current ?? 0) + 1; + const total = info.total || 1; + void persist(coreRef.current.onPage(page, total, nowSec())); + armIdle(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [progress?.pageinfo]); + + // Tab/window visibility. + useEffect(() => { + const onVis = () => { + if (document.visibilityState === 'hidden') { + if (idleRef.current) clearTimeout(idleRef.current); + void persist(coreRef.current.onHide(nowSec())); + } + }; + document.addEventListener('visibilitychange', onVis); + return () => document.removeEventListener('visibilitychange', onVis); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [bookMd5]); + + // Book close (unmount). + useEffect(() => { + return () => { + if (idleRef.current) clearTimeout(idleRef.current); + if (pushTimerRef.current) clearTimeout(pushTimerRef.current); + const closePersist = persist(coreRef.current.onClose(nowSec())); + void closePersist.then(() => { + if (syncEnabled() && dbRef.current) return pushStats(dbRef.current, new SyncClient()); + return undefined; + }); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [bookMd5]); + + return null; +} diff --git a/apps/readest-app/src/app/user/components/SyncCategoriesSection.tsx b/apps/readest-app/src/app/user/components/SyncCategoriesSection.tsx index 18ea9957..554aaeb0 100644 --- a/apps/readest-app/src/app/user/components/SyncCategoriesSection.tsx +++ b/apps/readest-app/src/app/user/components/SyncCategoriesSection.tsx @@ -59,6 +59,10 @@ const useCategoryCopy = (): Record => { 'Tokens, usernames, and passwords for OPDS, KOReader, Hardcover, and Readwise. When disabled, credentials remain on this device only and are never uploaded.', ), }, + stats: { + title: _('Reading statistics'), + description: _('Reading time and pages read, synced across your devices and KOReader.'), + }, }; }; diff --git a/apps/readest-app/src/hooks/useSync.ts b/apps/readest-app/src/hooks/useSync.ts index 2ed20ed4..738b7442 100644 --- a/apps/readest-app/src/hooks/useSync.ts +++ b/apps/readest-app/src/hooks/useSync.ts @@ -109,8 +109,12 @@ export function useSync(bookKey?: string) { try { const result = await syncClient.pullChanges(since, type, bookId, metaHash); - setSyncResult({ ...syncResult, [type]: result[type] }); - const records = result[type]; + const resultAsRecord = result as unknown as Record< + string, + BookDataRecord[] | null | undefined + >; + setSyncResult({ ...syncResult, [type]: resultAsRecord[type] }); + const records = resultAsRecord[type]; if (since > 1000 && !records?.length) return 0; // For since <= 1000, we set lastSyncedAt to now if no records returned const maxTime = records?.length ? computeMaxTimestamp(records) : Date.now(); diff --git a/apps/readest-app/src/libs/sync.ts b/apps/readest-app/src/libs/sync.ts index 562b5908..4f1b9477 100644 --- a/apps/readest-app/src/libs/sync.ts +++ b/apps/readest-app/src/libs/sync.ts @@ -5,17 +5,42 @@ import { fetchWithTimeout } from '@/utils/fetch'; const SYNC_API_ENDPOINT = getAPIBaseUrl() + '/sync'; -export type SyncType = 'books' | 'configs' | 'notes'; +export type SyncType = 'books' | 'configs' | 'notes' | 'stats'; export type SyncOp = 'push' | 'pull' | 'both'; interface BookRecord extends BookDataRecord, Book {} interface BookConfigRecord extends BookDataRecord, BookConfig {} interface BookNoteRecord extends BookDataRecord, BookNote {} +export interface StatBookRecord { + user_id?: string; + book_hash: string; + title: string; + authors: string; + updated_at?: string; + updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math + deleted_at?: string | null; +} + +export interface StatPageRecord { + user_id?: string; + book_hash: string; + page: number; + start_time: number; + duration: number; + total_pages: number; + ext?: unknown; + updated_at?: string; + updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math + deleted_at?: string | null; +} + export interface SyncResult { books: BookRecord[] | null; notes: BookNoteRecord[] | null; configs: BookConfigRecord[] | null; + statBooks?: StatBookRecord[] | null; + statPages?: StatPageRecord[] | null; } export type SyncRecord = BookRecord & BookConfigRecord & BookNoteRecord; @@ -24,6 +49,8 @@ export interface SyncData { books?: Partial[]; notes?: Partial[]; configs?: Partial[]; + statBooks?: StatBookRecord[]; + statPages?: StatPageRecord[]; } export class SyncClient { @@ -36,11 +63,13 @@ export class SyncClient { type?: SyncType, book?: string, metaHash?: string, + limit?: number, ): Promise { const token = await getAccessToken(); if (!token) throw new Error('Not authenticated'); - const url = `${SYNC_API_ENDPOINT}?since=${encodeURIComponent(since)}&type=${type ?? ''}&book=${book ?? ''}&meta_hash=${metaHash ?? ''}`; + const limitParam = limit && limit > 0 ? `&limit=${encodeURIComponent(limit)}` : ''; + const url = `${SYNC_API_ENDPOINT}?since=${encodeURIComponent(since)}&type=${type ?? ''}&book=${book ?? ''}&meta_hash=${metaHash ?? ''}${limitParam}`; const res = await fetchWithTimeout( url, { diff --git a/apps/readest-app/src/pages/api/sync.ts b/apps/readest-app/src/pages/api/sync.ts index fb2a22c1..73ce85b3 100644 --- a/apps/readest-app/src/pages/api/sync.ts +++ b/apps/readest-app/src/pages/api/sync.ts @@ -7,10 +7,36 @@ import { transformBookConfigToDB } from '@/utils/transform'; import { transformBookNoteToDB } from '@/utils/transform'; import { transformBookToDB } from '@/utils/transform'; import { runMiddleware, corsAllMethods } from '@/utils/cors'; -import { SyncData, SyncRecord, SyncResult, SyncType } from '@/libs/sync'; +import { + SyncData, + SyncRecord, + SyncResult, + SyncType, + StatBookRecord, + StatPageRecord, +} from '@/libs/sync'; import { validateUserAndToken } from '@/utils/access'; import { DBBook, DBBookConfig } from '@/types/records'; +const pageKey = (r: StatPageRecord) => `${r.book_hash}|${r.page}|${r.start_time}`; + +/** + * Decide which incoming page events to write: new keys always win; existing + * keys win only when the incoming duration is strictly longer (union/upsert + * semantics — KOReader-compatible). + */ +export function pickWinningPages( + incoming: StatPageRecord[], + server: Map, +): { toUpsert: StatPageRecord[] } { + const toUpsert: StatPageRecord[] = []; + for (const rec of incoming) { + const existing = server.get(pageKey(rec)); + if (!existing || rec.duration > existing.duration) toUpsert.push(rec); + } + return { toUpsert }; +} + const transformsToDB = { books: transformBookToDB, book_notes: transformBookNoteToDB, @@ -39,6 +65,10 @@ export async function GET(req: NextRequest) { const typeParam = searchParams.get('type') as SyncType | undefined; const bookParam = searchParams.get('book'); const metaHashParam = searchParams.get('meta_hash'); + // Optional page size for `type=stats` (client-driven paged pull). Absent for + // the koplugin, which keeps the full-delta response. + const statsLimitParam = searchParams.get('limit'); + const statsLimit = statsLimitParam ? Math.max(1, Math.floor(Number(statsLimitParam))) : 0; if (!sinceParam) { return NextResponse.json({ error: '"since" query parameter is required' }, { status: 400 }); @@ -52,7 +82,7 @@ export async function GET(req: NextRequest) { const sinceIso = since.toISOString(); try { - const results: SyncResult = { books: [], configs: [], notes: [] }; + const results: SyncResult = { books: [], configs: [], notes: [], statBooks: [], statPages: [] }; const errors: Record = { books: null, book_notes: null, @@ -113,7 +143,7 @@ export async function GET(req: NextRequest) { } }); } - results[DBSyncTypeMap[table] as SyncType] = records || []; + (results as unknown as Record)[DBSyncTypeMap[table]] = records || []; }; if (!typeParam || typeParam === 'books') { @@ -145,6 +175,99 @@ export async function GET(req: NextRequest) { if (!typeParam || typeParam === 'notes') { await queryTables('book_notes', ['id']).catch((err) => (errors['book_notes'] = err)); } + if (!typeParam || typeParam === 'stats') { + // PostgREST caps responses at ~1000 rows; stat_pages grows one row per page + // event, so page through both tables (ordered by updated_at ascending for a + // stable cursor) and accumulate every row — otherwise a device pulling >1000 + // events only gets the first page and then advances its cursor past the rest. + const PAGE = 1000; + const fetchAll = async (table: 'stat_books' | 'stat_pages', filterBook: boolean) => { + const all: Record[] = []; + let offset = 0; + for (;;) { + let q = supabase + .from(table) + .select('*') + .eq('user_id', user.id) + .or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`) + .order('updated_at', { ascending: true }) + .range(offset, offset + PAGE - 1); + if (filterBook && bookParam) q = q.eq('book_hash', bookParam); + const { data, error } = await q; + if (error) return { error }; + const rows = (data ?? []) as Record[]; + all.push(...rows); + if (rows.length < PAGE) break; + offset += PAGE; + } + return { data: all }; + }; + // A single bounded page of stat_pages for the app's client-driven paged + // pull, completed to the trailing updated_at millisecond so the client can + // advance its cursor with a strict `> cursor` without skipping ties. + const fetchPagedPages = async () => { + let q = supabase + .from('stat_pages') + .select('*') + .eq('user_id', user.id) + .or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`) + .order('updated_at', { ascending: true }) + .range(0, statsLimit - 1); + if (bookParam) q = q.eq('book_hash', bookParam); + const { data, error } = await q; + if (error) return { error }; + const rows = (data ?? []) as Record[]; + if (rows.length === statsLimit) { + const lastUpdated = rows[rows.length - 1]!['updated_at'] as string; + let eq = supabase + .from('stat_pages') + .select('*') + .eq('user_id', user.id) + .eq('updated_at', lastUpdated); + if (bookParam) eq = eq.eq('book_hash', bookParam); + const { data: extra, error: extraErr } = await eq; + if (extraErr) return { error: extraErr }; + const keyOf = (r: Record) => + `${r['book_hash']}|${r['page']}|${r['start_time']}`; + const seen = new Set(rows.map(keyOf)); + for (const r of (extra ?? []) as Record[]) { + const k = keyOf(r); + if (!seen.has(k)) { + seen.add(k); + rows.push(r); + } + } + } + return { data: rows }; + }; + // stat_books is always returned in full (one row per book, small); only + // stat_pages pages when the client asks (the koplugin omits `limit`). + const sb = await fetchAll('stat_books', false); + const sp = statsLimit > 0 ? await fetchPagedPages() : await fetchAll('stat_pages', true); + if (sb.error) + return NextResponse.json( + { error: `stat_books: ${sb.error.message || 'Unknown error'}` }, + { status: 500 }, + ); + if (sp.error) + return NextResponse.json( + { error: `stat_pages: ${sp.error.message || 'Unknown error'}` }, + { status: 500 }, + ); + // Attach updated_at_ms (epoch ms) so non-JS clients (the Lua koplugin) can + // compute their pull cursor without parsing ISO-8601 timestamps. + const withMs = (rows: T[]) => + rows.map((r) => ({ + ...r, + updated_at_ms: r.updated_at ? new Date(r.updated_at).getTime() : 0, + })); + ( + results as unknown as { statBooks: StatBookRecord[]; statPages: StatPageRecord[] } + ).statBooks = withMs((sb.data ?? []) as unknown as StatBookRecord[]); + ( + results as unknown as { statBooks: StatBookRecord[]; statPages: StatPageRecord[] } + ).statPages = withMs((sp.data ?? []) as unknown as StatPageRecord[]); + } const dbErrors = Object.values(errors).filter((err) => err !== null); if (dbErrors.length > 0) { @@ -174,7 +297,7 @@ export async function POST(req: NextRequest) { } const supabase = createSupabaseClient(token); const body = await req.json(); - const { books = [], configs = [], notes = [] } = body as SyncData; + const { books = [], configs = [], notes = [], statBooks = [], statPages = [] } = body as SyncData; const BATCH_SIZE = 100; const upsertRecords = async ( @@ -364,6 +487,65 @@ export async function POST(req: NextRequest) { ); } + if (statBooks.length > 0) { + const rows = statBooks.map((b: StatBookRecord) => ({ + user_id: user.id, + book_hash: b.book_hash, + title: b.title, + authors: b.authors, + updated_at: new Date().toISOString(), + deleted_at: b.deleted_at ?? null, + })); + const { error } = await supabase + .from('stat_books') + .upsert(rows, { onConflict: 'user_id,book_hash' }); + if (error) return NextResponse.json({ error: error.message }, { status: 500 }); + } + + if (statPages.length > 0) { + // Process in batches so the "longer-duration-wins" merge stays correct at + // scale: the existing-row fetch is scoped to each batch's (book_hash, + // start_time) keys (not a book's whole history) and bounded under + // PostgREST's ~1000-row cap — otherwise existing rows beyond 1000 are + // invisible to pickWinningPages and a shorter duration could overwrite a + // longer one. + const BATCH = 500; + for (let off = 0; off < statPages.length; off += BATCH) { + const batch = statPages.slice(off, off + BATCH); + const bookHashes = [...new Set(batch.map((p) => p.book_hash))]; + const startTimes = [...new Set(batch.map((p) => p.start_time))]; + const { data: existing, error: exErr } = await supabase + .from('stat_pages') + .select('*') + .eq('user_id', user.id) + .in('book_hash', bookHashes) + .in('start_time', startTimes); + if (exErr) return NextResponse.json({ error: exErr.message }, { status: 500 }); + const serverMap = new Map(); + (existing ?? []).forEach((r) => + serverMap.set(pageKey(r as StatPageRecord), r as StatPageRecord), + ); + const { toUpsert } = pickWinningPages(batch, serverMap); + const rows = toUpsert.map((p) => ({ + user_id: user.id, + book_hash: p.book_hash, + page: p.page, + start_time: p.start_time, + duration: p.duration, + total_pages: p.total_pages, + ext: p.ext ?? null, + updated_at: new Date().toISOString(), + deleted_at: p.deleted_at ?? null, + })); + if (rows.length > 0) { + const { error } = await supabase + .from('stat_pages') + .upsert(rows, { onConflict: 'user_id,book_hash,page,start_time' }); + if (error) return NextResponse.json({ error: error.message }, { status: 500 }); + } + } + } + return NextResponse.json( { books: booksResult?.data || [], diff --git a/apps/readest-app/src/services/database/migrations/index.ts b/apps/readest-app/src/services/database/migrations/index.ts index dce4286b..82e79356 100644 --- a/apps/readest-app/src/services/database/migrations/index.ts +++ b/apps/readest-app/src/services/database/migrations/index.ts @@ -51,6 +51,65 @@ const migrations: Record = { // rows rather than UPDATE — Tantivy 0.25→0.26 has a known WASM-only // UPDATE regression (see fts-tests.ts:306 FIXME). MVP indexing is // write-once per book so this is naturally satisfied. + statistics: [ + { + name: '2026061501_statistics_koreader_schema', + sql: ` + CREATE TABLE IF NOT EXISTS book ( + id integer PRIMARY KEY autoincrement, + title text, authors text, notes integer, last_open integer, + highlights integer, pages integer, series text, language text, + md5 text, total_read_time integer, total_read_pages integer + ); + + CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); + + CREATE TABLE IF NOT EXISTS page_stat_data ( + id_book integer, + page integer NOT NULL DEFAULT 0, + start_time integer NOT NULL DEFAULT 0, + duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + UNIQUE (id_book, page, start_time) + ); + + CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time); + + CREATE TABLE IF NOT EXISTS numbers (number INTEGER PRIMARY KEY); + + INSERT OR IGNORE INTO numbers(number) + SELECT h.n * 100 + t.n * 10 + o.n + 1 + FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) o, + (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t, + (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) h; + + CREATE VIEW IF NOT EXISTS page_stat AS + SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration + FROM ( + SELECT id_book, page, total_pages, pages, start_time, duration, + ((page - 1) * pages) / total_pages + 1 AS first_page, + max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page, + idx + FROM page_stat_data + JOIN book ON book.id = id_book + JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1) + ); + + CREATE TABLE IF NOT EXISTS readest_page_ext ( + book_hash text NOT NULL, page integer NOT NULL, start_time integer NOT NULL, + ext text, PRIMARY KEY (book_hash, page, start_time) + ); + + CREATE TABLE IF NOT EXISTS readest_book_ext ( + book_hash text PRIMARY KEY, ext text + ); + + CREATE TABLE IF NOT EXISTS readest_stat_sync_state ( + key text PRIMARY KEY, value integer NOT NULL DEFAULT 0 + ); + `, + }, + ], reedy: [ { name: '2026052601_reedy_init', diff --git a/apps/readest-app/src/services/statistics/statisticsDb.ts b/apps/readest-app/src/services/statistics/statisticsDb.ts new file mode 100644 index 00000000..8455e50a --- /dev/null +++ b/apps/readest-app/src/services/statistics/statisticsDb.ts @@ -0,0 +1,231 @@ +import type { AppService } from '@/types/system'; +import type { DatabaseService, DatabaseRow } from '@/types/database'; +import type { PageStatEvent, StatBook } from '@/types/statistics'; + +interface BookRow extends DatabaseRow { + id: number; + title: string; + authors: string; + md5: string; + total_read_time: number; + total_read_pages: number; + last_open: number; + pages: number; +} + +type CursorKey = 'push' | 'pull'; + +/** + * Per-tab singleton open promise. OPFS permits only ONE access handle per file + * across the whole origin, so a second `connect()` to statistics.db throws + * `NoModificationAllowedError`. Every ReadingStatsTracker instance (and split- + * view books) must therefore share a single connection — we memoise the open + * and never thrash it. + */ +let sharedDb: Promise | null = null; +let lifecycleBound = false; + +function bindLifecycle(): void { + if (lifecycleBound || typeof document === 'undefined') return; + lifecycleBound = true; + // Fold the WAL into the main db when the tab is backgrounded/closed — the + // most reliable point for best-effort async OPFS work before teardown. + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden' && sharedDb) { + void sharedDb.then((s) => s.checkpoint()).catch(() => {}); + } + }); +} + +/** + * Typed wrapper over the KOReader-compatible `statistics.db`. All identity is + * keyed on `book_hash` (= Book.hash); the local autoincrement `id_book` never + * leaves this class. + */ +export class StatisticsDb { + private constructor(private readonly db: DatabaseService) {} + + /** Production entry point — opens + migrates statistics.db (per-tab singleton). */ + static async open(appService: AppService): Promise { + bindLifecycle(); + if (!sharedDb) { + sharedDb = (async () => { + const db = await appService.openDatabase('statistics', 'statistics.db', 'Data'); + return new StatisticsDb(db); + })(); + } + return sharedDb; + } + + /** Test/advanced entry point — wrap an already-migrated DatabaseService. */ + static from(db: DatabaseService): StatisticsDb { + return new StatisticsDb(db); + } + + /** + * Fold the WAL into the main db file and truncate it. The Turso engine does + * NOT implement `PRAGMA wal_autocheckpoint` (so there's no auto threshold to + * rely on), but `wal_checkpoint(TRUNCATE)` works — verified folding a 688 KB + * WAL back to 0 B. We call this when the tab is hidden so the WAL stays bounded. + */ + async checkpoint(): Promise { + await this.db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); + } + + /** Checkpoint, close the underlying connection, and reset the singleton. */ + async close(): Promise { + try { + await this.checkpoint(); + } catch { + // best-effort — a checkpoint failure must not block close + } + await this.db.close(); + sharedDb = null; + } + + async upsertBook(book: StatBook): Promise { + const existing = await this.db.select(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [ + book.bookMd5, + ]); + if (existing[0]) { + // md5 is the identity; keep the latest title/authors (LWW, matches server stat_books). + await this.db.execute(`UPDATE book SET title = ?, authors = ? WHERE id = ?`, [ + book.title, + book.authors, + existing[0].id, + ]); + return existing[0].id; + } + await this.db.execute( + `INSERT INTO book (title, authors, md5, notes, last_open, highlights, pages, total_read_time, total_read_pages) + VALUES (?, ?, ?, 0, 0, 0, 0, 0, 0) + ON CONFLICT(title, authors, md5) DO NOTHING`, + [book.title, book.authors, book.bookMd5], + ); + const rows = await this.db.select(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [ + book.bookMd5, + ]); + return rows[0]!.id; + } + + async insertPageEvent( + idBook: number, + e: { page: number; startTime: number; duration: number; totalPages: number }, + ): Promise { + await this.db.execute( + `INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id_book, page, start_time) + DO UPDATE SET duration = max(duration, excluded.duration), total_pages = excluded.total_pages`, + [idBook, e.page, e.startTime, e.duration, e.totalPages], + ); + } + + async recomputeBookTotals(idBook: number): Promise { + await this.db.execute( + `UPDATE book SET + total_read_time = COALESCE((SELECT SUM(duration) FROM page_stat_data WHERE id_book = ?), 0), + total_read_pages = COALESCE((SELECT COUNT(DISTINCT page) FROM page_stat_data WHERE id_book = ?), 0), + last_open = COALESCE((SELECT MAX(start_time + duration) FROM page_stat_data WHERE id_book = ?), last_open), + pages = COALESCE((SELECT total_pages FROM page_stat_data WHERE id_book = ? ORDER BY start_time DESC LIMIT 1), pages) + WHERE id = ?`, + [idBook, idBook, idBook, idBook, idBook], + ); + } + + async getBookByMd5(md5: string): Promise { + const rows = await this.db.select(`SELECT * FROM book WHERE md5 = ? LIMIT 1`, [md5]); + return rows[0] ?? null; + } + + /** + * Ensure a book row exists for an event whose metadata record isn't in the + * current batch (paged pull can deliver an event before its `stat_books` + * record). Unlike `upsertBook`, this NEVER overwrites an existing real title + * with the hash placeholder — the real record, arriving in any page, wins. + */ + private async ensureBookId(bookMd5: string): Promise { + const existing = await this.db.select(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [ + bookMd5, + ]); + if (existing[0]) return existing[0].id; + return this.upsertBook({ bookMd5, title: bookMd5, authors: '' }); + } + + /** Events with start_time > cursor, joined to their md5, for pushing. */ + async getEventsForPush( + sinceStartTime: number, + ): Promise<{ events: PageStatEvent[]; books: StatBook[] }> { + const rows = await this.db.select( + `SELECT b.md5 AS bookMd5, b.title AS title, b.authors AS authors, + p.page AS page, p.start_time AS startTime, p.duration AS duration, p.total_pages AS totalPages + FROM page_stat_data p JOIN book b ON b.id = p.id_book + WHERE p.start_time > ? + ORDER BY p.start_time ASC`, + [sinceStartTime], + ); + const events: PageStatEvent[] = rows.map((r) => ({ + bookMd5: String(r['bookMd5']), + page: Number(r['page']), + startTime: Number(r['startTime']), + duration: Number(r['duration']), + totalPages: Number(r['totalPages']), + })); + const bookMap = new Map(); + for (const r of rows) { + const md5 = String(r['bookMd5']); + if (!bookMap.has(md5)) { + bookMap.set(md5, { + bookMd5: md5, + title: String(r['title']), + authors: String(r['authors']), + }); + } + } + return { events, books: [...bookMap.values()] }; + } + + async applyRemoteEvents(books: StatBook[], events: PageStatEvent[]): Promise { + if (books.length === 0 && events.length === 0) return; + // One transaction for the whole pulled batch: a single commit instead of + // O(rows) fsyncs, and the apply is atomic (a failed pull leaves no partial + // state). Critical when a fresh device backfills tens of thousands of rows. + await this.db.execute('BEGIN'); + try { + const idByMd5 = new Map(); + for (const b of books) idByMd5.set(b.bookMd5, await this.upsertBook(b)); + // Books referenced only by events (no metadata record) get a placeholder row. + const touched = new Set(); + for (const e of events) { + let id = idByMd5.get(e.bookMd5); + if (id === undefined) { + id = await this.ensureBookId(e.bookMd5); + idByMd5.set(e.bookMd5, id); + } + await this.insertPageEvent(id, e); + touched.add(id); + } + for (const id of touched) await this.recomputeBookTotals(id); + await this.db.execute('COMMIT'); + } catch (err) { + await this.db.execute('ROLLBACK'); + throw err; + } + } + + async getCursor(key: CursorKey): Promise { + const rows = await this.db.select<{ value: number }>( + `SELECT value FROM readest_stat_sync_state WHERE key = ?`, + [key], + ); + return rows[0]?.value ?? 0; + } + + async setCursor(key: CursorKey, value: number): Promise { + await this.db.execute( + `INSERT INTO readest_stat_sync_state (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + [key, value], + ); + } +} diff --git a/apps/readest-app/src/services/statistics/statsSync.ts b/apps/readest-app/src/services/statistics/statsSync.ts new file mode 100644 index 00000000..6ac72965 --- /dev/null +++ b/apps/readest-app/src/services/statistics/statsSync.ts @@ -0,0 +1,93 @@ +import type { StatisticsDb } from './statisticsDb'; +import type { SyncClient, StatPageRecord, StatBookRecord } from '@/libs/sync'; +import type { PageStatEvent, StatBook } from '@/types/statistics'; + +type PushClient = Pick; +type PullClient = Pick; + +const toWirePage = (e: PageStatEvent): StatPageRecord => ({ + book_hash: e.bookMd5, + page: e.page, + start_time: e.startTime, + duration: e.duration, + total_pages: e.totalPages, +}); + +const toWireBook = (b: StatBook): StatBookRecord => ({ + book_hash: b.bookMd5, + title: b.title, + authors: b.authors, +}); + +/** Events per push request — bounds request size for a large offline backlog. */ +const PUSH_CHUNK = 500; +/** Page events per pull request — bounds the receiving device's memory. */ +const PULL_PAGE = 1000; + +/** + * Push local events newer than the push cursor, in bounded chunks. The cursor + * advances per successful chunk, so an interrupted push (e.g. a 1000-event + * backlog over flaky network) resumes from the last chunk rather than restarting. + */ +export async function pushStats(stats: StatisticsDb, client: PushClient): Promise { + const cursor = await stats.getCursor('push'); + const { events, books } = await stats.getEventsForPush(cursor); + if (events.length === 0) return; + const bookByHash = new Map(books.map((b) => [b.bookMd5, b])); + let i = 0; + while (i < events.length) { + let end = Math.min(i + PUSH_CHUNK, events.length); + // Never split a start_time across chunks — advancing the push cursor past it + // would drop the remaining same-second events (e.g. split-view) on resume. + const lastStart = events[end - 1]!.startTime; + while (end < events.length && events[end]!.startTime === lastStart) end++; + const chunk = events.slice(i, end); + const seen = new Set(); + const chunkBooks: StatBookRecord[] = []; + for (const e of chunk) { + if (seen.has(e.bookMd5)) continue; + seen.add(e.bookMd5); + const b = bookByHash.get(e.bookMd5); + if (b) chunkBooks.push(toWireBook(b)); + } + await client.pushChanges({ statBooks: chunkBooks, statPages: chunk.map(toWirePage) }); + await stats.setCursor('push', chunk[chunk.length - 1]!.startTime); + i = end; + } +} + +/** + * Pull events since the pull cursor in bounded pages, applying each before + * fetching the next so memory stays flat and a fresh-device backfill is + * resumable (the cursor persists between pages). The server completes the + * trailing millisecond of each page, so a strict `> cursor` next pull never + * skips rows that share an `updated_at`. + */ +export async function pullStats(stats: StatisticsDb, client: PullClient): Promise { + for (;;) { + const since = await stats.getCursor('pull'); + const res = await client.pullChanges(since, 'stats', undefined, undefined, PULL_PAGE); + const wireBooks = (res.statBooks ?? []) as StatBookRecord[]; + const wirePages = (res.statPages ?? []) as StatPageRecord[]; + if (wireBooks.length === 0 && wirePages.length === 0) break; + const books: StatBook[] = wireBooks.map((b) => ({ + bookMd5: b.book_hash, + title: b.title, + authors: b.authors, + })); + const events: PageStatEvent[] = wirePages.map((p) => ({ + bookMd5: p.book_hash, + page: p.page, + startTime: p.start_time, + duration: p.duration, + totalPages: p.total_pages, + })); + await stats.applyRemoteEvents(books, events); + // Advance the cursor to the newest page-event updated_at_ms. Stop when a + // page yields no further page-event progress (covers the empty-pages and + // the all-books-no-pages cases, and guards against a stalled cursor). + const newest = wirePages.reduce((m, p) => Math.max(m, p.updated_at_ms ?? 0), since); + if (newest <= since) break; + await stats.setCursor('pull', newest); + } +} diff --git a/apps/readest-app/src/services/statistics/trackerCore.ts b/apps/readest-app/src/services/statistics/trackerCore.ts new file mode 100644 index 00000000..4bbe4867 --- /dev/null +++ b/apps/readest-app/src/services/statistics/trackerCore.ts @@ -0,0 +1,60 @@ +import type { StatsTrackingConfig } from '@/types/statistics'; + +interface PendingEvent { + page: number; + startTime: number; // Unix seconds + totalPages: number; +} + +export interface FlushedEvent { + page: number; + startTime: number; + duration: number; + totalPages: number; +} + +/** + * Pure page-dwell tracker. The React layer feeds it `nowSeconds` and the + * current `(page, totalPages)`; it returns zero-or-one immutable events to + * persist. Events end on page-change / idle / hide / close. Each returned + * event is final — its start_time and duration never change afterwards, which + * lets sync use a simple start_time high-water cursor. + */ +export class TrackerCore { + private pending: PendingEvent | null = null; + + constructor(private readonly cfg: StatsTrackingConfig) {} + + /** Notify the current page at `now`. Returns events flushed by leaving the prior page. */ + onPage(page: number, totalPages: number, now: number): FlushedEvent[] { + if (this.pending && this.pending.page === page) { + // Same page (e.g. resume after idle, or a no-op relocate): keep dwelling. + return []; + } + const flushed = this.flush(now); + this.pending = { page, startTime: now, totalPages }; + return flushed; + } + + onIdle(now: number): FlushedEvent[] { + return this.flush(now); // flush + pause (pending cleared) + } + + onHide(now: number): FlushedEvent[] { + return this.flush(now); + } + + onClose(now: number): FlushedEvent[] { + return this.flush(now); + } + + private flush(now: number): FlushedEvent[] { + const p = this.pending; + this.pending = null; + if (!p) return []; + const raw = now - p.startTime; + const duration = Math.min(Math.max(raw, 0), this.cfg.maxEventSeconds); + if (duration < this.cfg.minEventSeconds) return []; + return [{ page: p.page, startTime: p.startTime, duration, totalPages: p.totalPages }]; + } +} diff --git a/apps/readest-app/src/types/settings.ts b/apps/readest-app/src/types/settings.ts index ccbc4845..0eb3275f 100644 --- a/apps/readest-app/src/types/settings.ts +++ b/apps/readest-app/src/types/settings.ts @@ -233,7 +233,8 @@ export type SyncCategory = | 'texture' | 'opds_catalog' | 'settings' - | 'credentials'; + | 'credentials' + | 'stats'; export const SYNC_CATEGORIES: readonly SyncCategory[] = [ 'book', @@ -244,6 +245,7 @@ export const SYNC_CATEGORIES: readonly SyncCategory[] = [ 'texture', 'opds_catalog', 'settings', + 'stats', 'credentials', ] as const; diff --git a/apps/readest-app/src/types/statistics.ts b/apps/readest-app/src/types/statistics.ts new file mode 100644 index 00000000..a14cb0f8 --- /dev/null +++ b/apps/readest-app/src/types/statistics.ts @@ -0,0 +1,37 @@ +// KOReader-compatible reading-statistics types. +// +// The canonical unit is the per-page read event (KOReader's page_stat_data +// row). Aggregates (sessions, streaks, totals) are DERIVED via SQL, never +// stored as separate records. Times are Unix seconds; pages are 1-based. + +/** One immutable page-read event — a KOReader `page_stat_data` row. */ +export interface PageStatEvent { + bookMd5: string; // = Book.hash = KOReader book.md5 + page: number; // 1-based + startTime: number; // Unix seconds + duration: number; // seconds + totalPages: number; +} + +/** KOReader book identity — the only book metadata that syncs. */ +export interface StatBook { + bookMd5: string; + title: string; + authors: string; // KOReader stores authors as a single text field +} + +/** Tunables for the tracker's flush/idle behavior. KOReader-aligned defaults. */ +export interface StatsTrackingConfig { + /** Seconds of inactivity before the current page event is flushed + paused. */ + idleTimeoutSeconds: number; + /** Hard per-event duration cap (safety net if a visibility event is missed). */ + maxEventSeconds: number; + /** Events shorter than this are dropped (ignore sub-second page flips). */ + minEventSeconds: number; +} + +export const DEFAULT_STATS_TRACKING_CONFIG: StatsTrackingConfig = { + idleTimeoutSeconds: 120, + maxEventSeconds: 120, + minEventSeconds: 3, +}; diff --git a/apps/readest.koplugin/locales/ar/translation.po b/apps/readest.koplugin/locales/ar/translation.po index 3066b067..4a2cdb64 100644 --- a/apps/readest.koplugin/locales/ar/translation.po +++ b/apps/readest.koplugin/locales/ar/translation.po @@ -436,3 +436,9 @@ msgstr "تمت مزامنة تقدم القراءة" msgid "No saved reading progress found for this book" msgstr "لم يتم العثور على تقدم قراءة محفوظ لهذا الكتاب" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/bn/translation.po b/apps/readest.koplugin/locales/bn/translation.po index 417981da..f4d65620 100644 --- a/apps/readest.koplugin/locales/bn/translation.po +++ b/apps/readest.koplugin/locales/bn/translation.po @@ -436,3 +436,9 @@ msgstr "পড়ার অগ্রগতি সিঙ্ক হয়েছে msgid "No saved reading progress found for this book" msgstr "এই বইয়ের জন্য সংরক্ষিত পড়ার অগ্রগতি পাওয়া যায়নি" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/bo/translation.po b/apps/readest.koplugin/locales/bo/translation.po index dc233845..49395e2e 100644 --- a/apps/readest.koplugin/locales/bo/translation.po +++ b/apps/readest.koplugin/locales/bo/translation.po @@ -436,3 +436,9 @@ msgstr "ཀློག་པའི་འཕེལ་རིམ་མཉམ་འག msgid "No saved reading progress found for this book" msgstr "དཔེ་ཆ་འདིའི་ཀློག་པའི་འཕེལ་རིམ་ཉར་ཚགས་མ་རྙེད།" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/de/translation.po b/apps/readest.koplugin/locales/de/translation.po index 46d20b9f..807931a6 100644 --- a/apps/readest.koplugin/locales/de/translation.po +++ b/apps/readest.koplugin/locales/de/translation.po @@ -436,3 +436,9 @@ msgstr "Lesefortschritt synchronisiert" msgid "No saved reading progress found for this book" msgstr "Kein gespeicherter Lesefortschritt für dieses Buch gefunden" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/el/translation.po b/apps/readest.koplugin/locales/el/translation.po index 2809a629..9a6f9eda 100644 --- a/apps/readest.koplugin/locales/el/translation.po +++ b/apps/readest.koplugin/locales/el/translation.po @@ -436,3 +436,9 @@ msgstr "Η πρόοδος ανάγνωσης συγχρονίστηκε" msgid "No saved reading progress found for this book" msgstr "Δεν βρέθηκε αποθηκευμένη πρόοδος ανάγνωσης για αυτό το βιβλίο" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/es/translation.po b/apps/readest.koplugin/locales/es/translation.po index 144dea07..616895b5 100644 --- a/apps/readest.koplugin/locales/es/translation.po +++ b/apps/readest.koplugin/locales/es/translation.po @@ -436,3 +436,9 @@ msgstr "Progreso de lectura sincronizado" msgid "No saved reading progress found for this book" msgstr "No se encontró progreso de lectura guardado para este libro" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/fa/translation.po b/apps/readest.koplugin/locales/fa/translation.po index 5a9f1efd..0c59b52f 100644 --- a/apps/readest.koplugin/locales/fa/translation.po +++ b/apps/readest.koplugin/locales/fa/translation.po @@ -436,3 +436,9 @@ msgstr "پیشرفت مطالعه همگام‌سازی شد" msgid "No saved reading progress found for this book" msgstr "پیشرفت مطالعه ذخیره‌شده‌ای برای این کتاب یافت نشد" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/fr/translation.po b/apps/readest.koplugin/locales/fr/translation.po index c6e361ea..f5b45e34 100644 --- a/apps/readest.koplugin/locales/fr/translation.po +++ b/apps/readest.koplugin/locales/fr/translation.po @@ -436,3 +436,9 @@ msgstr "Progression de lecture synchronisée" msgid "No saved reading progress found for this book" msgstr "Aucune progression de lecture enregistrée pour ce livre" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/he/translation.po b/apps/readest.koplugin/locales/he/translation.po index 6d3ad198..49ef223b 100644 --- a/apps/readest.koplugin/locales/he/translation.po +++ b/apps/readest.koplugin/locales/he/translation.po @@ -436,3 +436,9 @@ msgstr "התקדמות הקריאה סונכרנה" msgid "No saved reading progress found for this book" msgstr "לא נמצאה התקדמות קריאה שמורה לספר זה" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/hi/translation.po b/apps/readest.koplugin/locales/hi/translation.po index 7342fbcc..018c5c46 100644 --- a/apps/readest.koplugin/locales/hi/translation.po +++ b/apps/readest.koplugin/locales/hi/translation.po @@ -436,3 +436,9 @@ msgstr "पठन प्रगति सिंक हो गई" msgid "No saved reading progress found for this book" msgstr "इस पुस्तक के लिए सहेजी गई पठन प्रगति नहीं मिली" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/hu/translation.po b/apps/readest.koplugin/locales/hu/translation.po index ea3bf5bc..964cde89 100644 --- a/apps/readest.koplugin/locales/hu/translation.po +++ b/apps/readest.koplugin/locales/hu/translation.po @@ -436,3 +436,9 @@ msgstr "Olvasási haladás szinkronizálva" msgid "No saved reading progress found for this book" msgstr "Nem található mentett olvasási haladás ehhez a könyvhöz" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/id/translation.po b/apps/readest.koplugin/locales/id/translation.po index 7be2ba99..050331e5 100644 --- a/apps/readest.koplugin/locales/id/translation.po +++ b/apps/readest.koplugin/locales/id/translation.po @@ -436,3 +436,9 @@ msgstr "Progres baca disinkronkan" msgid "No saved reading progress found for this book" msgstr "Tidak ada progres baca tersimpan untuk buku ini" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/it/translation.po b/apps/readest.koplugin/locales/it/translation.po index a0e15ff4..3f3ecfe8 100644 --- a/apps/readest.koplugin/locales/it/translation.po +++ b/apps/readest.koplugin/locales/it/translation.po @@ -436,3 +436,9 @@ msgstr "Progresso di lettura sincronizzato" msgid "No saved reading progress found for this book" msgstr "Nessun progresso di lettura salvato per questo libro" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ja/translation.po b/apps/readest.koplugin/locales/ja/translation.po index 91405b04..fd28c072 100644 --- a/apps/readest.koplugin/locales/ja/translation.po +++ b/apps/readest.koplugin/locales/ja/translation.po @@ -436,3 +436,9 @@ msgstr "読書の進捗を同期しました" msgid "No saved reading progress found for this book" msgstr "この書籍の保存された読書進捗が見つかりません" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ko/translation.po b/apps/readest.koplugin/locales/ko/translation.po index 5247b1db..a9bd2b93 100644 --- a/apps/readest.koplugin/locales/ko/translation.po +++ b/apps/readest.koplugin/locales/ko/translation.po @@ -436,3 +436,9 @@ msgstr "읽기 진행 상황이 동기화되었습니다" msgid "No saved reading progress found for this book" msgstr "이 책의 저장된 읽기 진행 상황을 찾을 수 없습니다" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ms/translation.po b/apps/readest.koplugin/locales/ms/translation.po index b70de3df..59f3b31d 100644 --- a/apps/readest.koplugin/locales/ms/translation.po +++ b/apps/readest.koplugin/locales/ms/translation.po @@ -436,3 +436,9 @@ msgstr "Kemajuan bacaan disegerakkan" msgid "No saved reading progress found for this book" msgstr "Tiada kemajuan bacaan tersimpan ditemui untuk buku ini" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/nl/translation.po b/apps/readest.koplugin/locales/nl/translation.po index ef7811c3..85f45640 100644 --- a/apps/readest.koplugin/locales/nl/translation.po +++ b/apps/readest.koplugin/locales/nl/translation.po @@ -436,3 +436,9 @@ msgstr "Leesvoortgang gesynchroniseerd" msgid "No saved reading progress found for this book" msgstr "Geen opgeslagen leesvoortgang gevonden voor dit boek" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/pl/translation.po b/apps/readest.koplugin/locales/pl/translation.po index cc4172d7..9c979f1d 100644 --- a/apps/readest.koplugin/locales/pl/translation.po +++ b/apps/readest.koplugin/locales/pl/translation.po @@ -436,3 +436,9 @@ msgstr "Postęp czytania zsynchronizowany" msgid "No saved reading progress found for this book" msgstr "Nie znaleziono zapisanego postępu czytania dla tej książki" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/pt-BR/translation.po b/apps/readest.koplugin/locales/pt-BR/translation.po index 434b5f66..f0fc7c0c 100644 --- a/apps/readest.koplugin/locales/pt-BR/translation.po +++ b/apps/readest.koplugin/locales/pt-BR/translation.po @@ -436,3 +436,9 @@ msgstr "Progresso de leitura sincronizado" msgid "No saved reading progress found for this book" msgstr "Nenhum progresso de leitura salvo encontrado para este livro" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/pt/translation.po b/apps/readest.koplugin/locales/pt/translation.po index 785c4989..9518fa3e 100644 --- a/apps/readest.koplugin/locales/pt/translation.po +++ b/apps/readest.koplugin/locales/pt/translation.po @@ -436,3 +436,9 @@ msgstr "Progresso de leitura sincronizado" msgid "No saved reading progress found for this book" msgstr "Não foi encontrado progresso de leitura guardado para este livro" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ro/translation.po b/apps/readest.koplugin/locales/ro/translation.po index b06f4240..c589428c 100644 --- a/apps/readest.koplugin/locales/ro/translation.po +++ b/apps/readest.koplugin/locales/ro/translation.po @@ -436,3 +436,9 @@ msgstr "Progres de citire sincronizat" msgid "No saved reading progress found for this book" msgstr "Nu s-a găsit progres de citire salvat pentru această carte" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ru/translation.po b/apps/readest.koplugin/locales/ru/translation.po index 6780ba77..53de7f1c 100644 --- a/apps/readest.koplugin/locales/ru/translation.po +++ b/apps/readest.koplugin/locales/ru/translation.po @@ -436,3 +436,9 @@ msgstr "Прогресс чтения синхронизирован" msgid "No saved reading progress found for this book" msgstr "Сохранённый прогресс чтения для этой книги не найден" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/si/translation.po b/apps/readest.koplugin/locales/si/translation.po index 861321fa..8921c18e 100644 --- a/apps/readest.koplugin/locales/si/translation.po +++ b/apps/readest.koplugin/locales/si/translation.po @@ -436,3 +436,9 @@ msgstr "කියවීමේ ප්‍රගතිය සමමුහූර් msgid "No saved reading progress found for this book" msgstr "මෙම පොත සඳහා සුරකින ලද කියවීමේ ප්‍රගතියක් හමු නොවුණි" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/sl/translation.po b/apps/readest.koplugin/locales/sl/translation.po index 02886140..6f37fc29 100644 --- a/apps/readest.koplugin/locales/sl/translation.po +++ b/apps/readest.koplugin/locales/sl/translation.po @@ -436,3 +436,9 @@ msgstr "Napredek branja sinhroniziran" msgid "No saved reading progress found for this book" msgstr "Za to knjigo ni shranjenega napredka branja" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/sv/translation.po b/apps/readest.koplugin/locales/sv/translation.po index 51b37eb9..d8123292 100644 --- a/apps/readest.koplugin/locales/sv/translation.po +++ b/apps/readest.koplugin/locales/sv/translation.po @@ -436,3 +436,9 @@ msgstr "Läsförloppet synkroniserat" msgid "No saved reading progress found for this book" msgstr "Inget sparat läsförlopp hittades för boken" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/ta/translation.po b/apps/readest.koplugin/locales/ta/translation.po index 156abef0..df21e804 100644 --- a/apps/readest.koplugin/locales/ta/translation.po +++ b/apps/readest.koplugin/locales/ta/translation.po @@ -436,3 +436,9 @@ msgstr "வாசிப்பு முன்னேற்றம் ஒத்த msgid "No saved reading progress found for this book" msgstr "இந்த நூலுக்கான சேமிக்கப்பட்ட வாசிப்பு முன்னேற்றம் இல்லை" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/th/translation.po b/apps/readest.koplugin/locales/th/translation.po index 8bf575d2..0336bab5 100644 --- a/apps/readest.koplugin/locales/th/translation.po +++ b/apps/readest.koplugin/locales/th/translation.po @@ -436,3 +436,9 @@ msgstr "ซิงค์ความคืบหน้าการอ่านแ msgid "No saved reading progress found for this book" msgstr "ไม่พบความคืบหน้าการอ่านที่บันทึกไว้สำหรับหนังสือเล่มนี้" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/tr/translation.po b/apps/readest.koplugin/locales/tr/translation.po index 6a64b598..12f215c8 100644 --- a/apps/readest.koplugin/locales/tr/translation.po +++ b/apps/readest.koplugin/locales/tr/translation.po @@ -436,3 +436,9 @@ msgstr "Okuma ilerlemesi senkronize edildi" msgid "No saved reading progress found for this book" msgstr "Bu kitap için kayıtlı okuma ilerlemesi bulunamadı" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/uk/translation.po b/apps/readest.koplugin/locales/uk/translation.po index f76b2541..275a6442 100644 --- a/apps/readest.koplugin/locales/uk/translation.po +++ b/apps/readest.koplugin/locales/uk/translation.po @@ -436,3 +436,9 @@ msgstr "Прогрес читання синхронізовано" msgid "No saved reading progress found for this book" msgstr "Збережений прогрес читання для цієї книги не знайдено" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/uz/translation.po b/apps/readest.koplugin/locales/uz/translation.po index 7f3f4ac2..0cd874f0 100644 --- a/apps/readest.koplugin/locales/uz/translation.po +++ b/apps/readest.koplugin/locales/uz/translation.po @@ -436,3 +436,9 @@ msgstr "Oʻqish jarayoni sinxronlandi" msgid "No saved reading progress found for this book" msgstr "Ushbu kitob uchun saqlangan oʻqish jarayoni topilmadi" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/vi/translation.po b/apps/readest.koplugin/locales/vi/translation.po index 7505e15d..d1553fad 100644 --- a/apps/readest.koplugin/locales/vi/translation.po +++ b/apps/readest.koplugin/locales/vi/translation.po @@ -436,3 +436,9 @@ msgstr "Tiến trình đọc đã đồng bộ" msgid "No saved reading progress found for this book" msgstr "Không tìm thấy tiến trình đọc đã lưu cho sách này" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/zh-CN/translation.po b/apps/readest.koplugin/locales/zh-CN/translation.po index d7237cbb..cf73f4e4 100644 --- a/apps/readest.koplugin/locales/zh-CN/translation.po +++ b/apps/readest.koplugin/locales/zh-CN/translation.po @@ -436,3 +436,9 @@ msgstr "阅读进度已同步" msgid "No saved reading progress found for this book" msgstr "未找到此书籍的已保存阅读进度" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/locales/zh-TW/translation.po b/apps/readest.koplugin/locales/zh-TW/translation.po index 102ff4b5..1802d988 100644 --- a/apps/readest.koplugin/locales/zh-TW/translation.po +++ b/apps/readest.koplugin/locales/zh-TW/translation.po @@ -436,3 +436,9 @@ msgstr "閱讀進度已同步" msgid "No saved reading progress found for this book" msgstr "找不到此書籍的閱讀進度" + +msgid "Failed to push reading statistics" +msgstr "" + +msgid "Failed to pull reading statistics" +msgstr "" diff --git a/apps/readest.koplugin/main.lua b/apps/readest.koplugin/main.lua index 54badfaf..fef40c51 100644 --- a/apps/readest.koplugin/main.lua +++ b/apps/readest.koplugin/main.lua @@ -12,6 +12,7 @@ local _ = require("readest_i18n") local SyncAuth = require("readest_syncauth") local SyncConfig = require("readest_syncconfig") local SyncAnnotations = require("readest_syncannotations") +local SyncStats = require("readest_syncstats") local SelfUpdate = require("readest_selfupdate") local ReadestSync = WidgetContainer:new{ @@ -91,6 +92,7 @@ function ReadestSync:onReaderReady() UIManager:nextTick(function() self:pullBookConfig(false) self:pullBookNotes(false) + self:pullBookStats(false) end) end self:onDispatcherRegisterReaderActions() @@ -533,6 +535,27 @@ function ReadestSync:pullBookConfig(interactive) ) end +-- ── Reading statistics sync ──────────────────────────────────────── + +function ReadestSync:pushBookStats(interactive) + if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookStats(interactive) end) then + return + end + local client = self:ensureClient(interactive) + if not client then return end + SyncStats:push(self.settings, client, interactive) +end + +function ReadestSync:pullBookStats(interactive) + if NetworkMgr:willRerunWhenOnline(function() self:pullBookStats(interactive) end) then + return + end + local client = self:ensureClient(interactive) + if not client then return end + SyncStats:pull(self.settings, client, interactive, + function() SyncAuth:logout(self.settings, self.path) end) +end + -- ── Annotation sync ──────────────────────────────────────────────── function ReadestSync:pushBookNotes(interactive, full_sync) @@ -722,6 +745,7 @@ function ReadestSync:onCloseDocument() NetworkMgr:goOnlineToRun(function() self:pushBookConfig(false) self:pushBookNotes(false) + self:pushBookStats(false) self:syncBooksLibrary("both", false) end) end diff --git a/apps/readest.koplugin/readest-sync-api.json b/apps/readest.koplugin/readest-sync-api.json index 683ad7b6..757c6439 100644 --- a/apps/readest.koplugin/readest-sync-api.json +++ b/apps/readest.koplugin/readest-sync-api.json @@ -12,7 +12,7 @@ "path": "/sync", "method": "POST", "required_params": ["books", "notes", "configs"], - "payload": ["books", "notes", "configs"], + "payload": ["books", "notes", "configs", "statBooks", "statPages"], "expected_status": [200, 201, 301, 400, 401, 403] }, "pullBooks": { diff --git a/apps/readest.koplugin/readest_syncstats.lua b/apps/readest.koplugin/readest_syncstats.lua new file mode 100644 index 00000000..a55bb2bd --- /dev/null +++ b/apps/readest.koplugin/readest_syncstats.lua @@ -0,0 +1,129 @@ +local DataStorage = require("datastorage") +local InfoMessage = require("ui/widget/infomessage") +local UIManager = require("ui/uimanager") +local SQ3 = require("lua-ljsqlite3/init") +local _ = require("readest_i18n") + +local SyncStats = {} + +local function db_path() + return DataStorage:getSettingsDir() .. "/statistics.sqlite3" +end + +-- Read book md5/title/authors + page events with start_time > cursor. +function SyncStats:collectSince(cursor) + local conn = SQ3.open(db_path()) + local books, pages, seen = {}, {}, {} + local stmt = conn:prepare([[ + SELECT b.md5, b.title, b.authors, p.page, p.start_time, p.duration, p.total_pages + FROM page_stat_data p JOIN book b ON b.id = p.id_book + WHERE p.start_time > ? ORDER BY p.start_time ASC]]) + stmt:reset():bind(tonumber(cursor) or 0) + local row = stmt:step() + while row ~= nil do + local md5 = row[1] + if md5 and not seen[md5] then + seen[md5] = true + table.insert(books, { book_hash = md5, title = row[2] or "", authors = row[3] or "" }) + end + table.insert(pages, { + book_hash = md5, + page = tonumber(row[4]), + start_time = tonumber(row[5]), + duration = tonumber(row[6]), + total_pages = tonumber(row[7]), + }) + row = stmt:step() + end + stmt:close() + conn:close() + return books, pages +end + +-- Upsert pulled rows into the local statistics.sqlite3 (union / longer-duration). +function SyncStats:applyRemote(books, pages) + local conn = SQ3.open(db_path()) + conn:exec("BEGIN;") + local insert_book = conn:prepare("INSERT OR IGNORE INTO book (title, authors, md5) VALUES (?, ?, ?);") + for _, b in ipairs(books or {}) do + insert_book:reset():bind(b.title or "", b.authors or "", b.book_hash):step() + end + insert_book:close() + local find_id = conn:prepare("SELECT id FROM book WHERE md5 = ? LIMIT 1;") + local insert_page = conn:prepare([[ + INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id_book, page, start_time) + DO UPDATE SET duration = max(duration, excluded.duration), total_pages = excluded.total_pages;]]) + local id_cache = {} + local touched = {} + for _, p in ipairs(pages or {}) do + local id = id_cache[p.book_hash] + if not id then + local r = find_id:reset():bind(p.book_hash):step() + if r ~= nil then id = tonumber(r[1]); id_cache[p.book_hash] = id end + end + if id then + insert_page:reset():bind(id, p.page, p.start_time, p.duration, p.total_pages):step() + touched[id] = true + end + end + find_id:close() + insert_page:close() + -- Mirror the Readest app's recomputeBookTotals so a KOReader device shows + -- fresh totals right after a pull (id is a trusted integer from the DB). + for id in pairs(touched) do + conn:exec(string.format([[ + UPDATE book SET + total_read_time = COALESCE((SELECT SUM(duration) FROM page_stat_data WHERE id_book = %d), 0), + total_read_pages = COALESCE((SELECT COUNT(DISTINCT page) FROM page_stat_data WHERE id_book = %d), 0), + last_open = COALESCE((SELECT MAX(start_time + duration) FROM page_stat_data WHERE id_book = %d), last_open) + WHERE id = %d;]], id, id, id, id)) + end + conn:exec("COMMIT;") + conn:close() +end + +function SyncStats:push(settings, client, interactive) + local cursor = settings:readSetting("stats_push_cursor") or 0 + local books, pages = self:collectSince(cursor) + if #pages == 0 then return end + local max_start = cursor + for _, p in ipairs(pages) do if p.start_time > max_start then max_start = p.start_time end end + client:pushChanges( + { statBooks = books, statPages = pages }, + function(success) + if success then + settings:saveSetting("stats_push_cursor", max_start) + elseif interactive then + UIManager:show(InfoMessage:new{ text = _("Failed to push reading statistics"), timeout = 2 }) + end + end) +end + +function SyncStats:pull(settings, client, interactive, logout_fn) + local since = settings:readSetting("stats_pull_cursor") or 0 + -- pullChanges requires since/type/book/meta_hash params (readest-sync-api.json). + client:pullChanges( + { since = since, type = "stats", book = "", meta_hash = "" }, + function(success, response, status) + if not success then + if status == 401 or status == 403 then + if logout_fn then logout_fn() end + end + if interactive then + UIManager:show(InfoMessage:new{ text = _("Failed to pull reading statistics"), timeout = 2 }) + end + return + end + self:applyRemote(response.statBooks, response.statPages) + local newest = since + for _, p in ipairs(response.statPages or {}) do + local u = tonumber(p.updated_at_ms) or 0 + if u > newest then newest = u end + end + if newest > since then settings:saveSetting("stats_pull_cursor", newest) end + end) +end + +return SyncStats diff --git a/apps/readest.koplugin/spec/syncstats_spec.lua b/apps/readest.koplugin/spec/syncstats_spec.lua new file mode 100644 index 00000000..1dce910c --- /dev/null +++ b/apps/readest.koplugin/spec/syncstats_spec.lua @@ -0,0 +1,149 @@ +-- syncstats_spec.lua +-- Tests for readest_syncstats.lua: collectSince cursor filtering and +-- applyRemote upsert with max-duration conflict resolution. + +local spec_helper = require("spec_helper") + +-- Minimal KOReader stubs the module pulls at require-time. +package.preload["ui/widget/infomessage"] = function() + return { new = function() return {} end } +end +package.preload["ui/uimanager"] = function() + return { show = function() end } +end +package.preload["readest_i18n"] = function() + return function(s) return s end +end + +local SQ3 = require("lua-ljsqlite3/init") +local DataStorage = require("datastorage") + +local function statsDbPath() + return DataStorage:getSettingsDir() .. "/statistics.sqlite3" +end + +local function seedDb() + local conn = SQ3.open(statsDbPath()) + conn:exec([[ + CREATE TABLE IF NOT EXISTS book ( + id integer PRIMARY KEY autoincrement, title text, authors text, notes integer, + last_open integer, highlights integer, pages integer, series text, language text, + md5 text, total_read_time integer, total_read_pages integer); + CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5); + CREATE TABLE IF NOT EXISTS page_stat_data ( + id_book integer, page integer NOT NULL DEFAULT 0, + start_time integer NOT NULL DEFAULT 0, + duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + UNIQUE (id_book, page, start_time)); + ]]) + conn:close() +end + +describe("readest_syncstats", function() + local SyncStats + + before_each(function() + spec_helper.reset() + os.remove(statsDbPath()) + seedDb() + package.loaded["readest_syncstats"] = nil + SyncStats = require("readest_syncstats") + end) + + it("collects only events past the cursor, joined with md5", function() + local conn = SQ3.open(statsDbPath()) + conn:exec("INSERT INTO book (title, authors, md5) VALUES ('T', 'A', 'md5-1');") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 1, 100, 5, 9);") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 2, 200, 6, 9);") + conn:close() + + local books, pages = SyncStats:collectSince(150) + assert.are.equal(1, #pages) + assert.are.equal(200, pages[1].start_time) + assert.are.equal("md5-1", pages[1].book_hash) + assert.are.equal(1, #books) + assert.are.equal("md5-1", books[1].book_hash) + end) + + it("returns all events when cursor is 0", function() + local conn = SQ3.open(statsDbPath()) + conn:exec("INSERT INTO book (title, authors, md5) VALUES ('B', 'C', 'md5-3');") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 1, 10, 3, 5);") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 2, 20, 4, 5);") + conn:close() + + local books, pages = SyncStats:collectSince(0) + assert.are.equal(2, #pages) + assert.are.equal(1, #books) + end) + + it("returns empty tables when no events are past the cursor", function() + local conn = SQ3.open(statsDbPath()) + conn:exec("INSERT INTO book (title, authors, md5) VALUES ('T', 'A', 'md5-1');") + conn:exec("INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (1, 1, 100, 5, 9);") + conn:close() + + local books, pages = SyncStats:collectSince(100) + assert.are.equal(0, #pages) + assert.are.equal(0, #books) + end) + + it("keeps the longer duration when applying remote events", function() + SyncStats:applyRemote( + { { book_hash = "md5-2", title = "T2", authors = "A2" } }, + { + { book_hash = "md5-2", page = 1, start_time = 300, duration = 8, total_pages = 20 }, + { book_hash = "md5-2", page = 1, start_time = 300, duration = 20, total_pages = 20 }, + }) + + local conn = SQ3.open(statsDbPath()) + local count = conn:rowexec("SELECT COUNT(*) FROM page_stat_data;") + local dur = conn:rowexec("SELECT duration FROM page_stat_data WHERE start_time = 300;") + conn:close() + assert.are.equal(1, tonumber(count)) + assert.are.equal(20, tonumber(dur)) + end) + + it("inserts a new book row when applying remote data for an unknown md5", function() + SyncStats:applyRemote( + { { book_hash = "new-md5", title = "New Book", authors = "Auth" } }, + { { book_hash = "new-md5", page = 5, start_time = 500, duration = 10, total_pages = 100 } }) + + local conn = SQ3.open(statsDbPath()) + local count = conn:rowexec("SELECT COUNT(*) FROM book WHERE md5 = 'new-md5';") + local pcount = conn:rowexec("SELECT COUNT(*) FROM page_stat_data;") + conn:close() + assert.are.equal(1, tonumber(count)) + assert.are.equal(1, tonumber(pcount)) + end) + + it("skips page rows whose book md5 has no matching book", function() + SyncStats:applyRemote( + {}, + { { book_hash = "ghost-md5", page = 1, start_time = 999, duration = 5, total_pages = 10 } }) + + local conn = SQ3.open(statsDbPath()) + local pcount = conn:rowexec("SELECT COUNT(*) FROM page_stat_data;") + conn:close() + assert.are.equal(0, tonumber(pcount)) + end) + + it("recomputes book totals after applying remote events", function() + SyncStats:applyRemote( + { { book_hash = "md5-tot", title = "Tot", authors = "A" } }, + { + { book_hash = "md5-tot", page = 1, start_time = 100, duration = 8, total_pages = 50 }, + { book_hash = "md5-tot", page = 2, start_time = 200, duration = 12, total_pages = 50 }, + }) + + local conn = SQ3.open(statsDbPath()) + local total_time = conn:rowexec("SELECT total_read_time FROM book WHERE md5 = 'md5-tot';") + local total_pages = conn:rowexec("SELECT total_read_pages FROM book WHERE md5 = 'md5-tot';") + local last_open = conn:rowexec("SELECT last_open FROM book WHERE md5 = 'md5-tot';") + conn:close() + assert.are.equal(20, tonumber(total_time)) -- 8 + 12 + assert.are.equal(2, tonumber(total_pages)) -- distinct pages 1,2 + assert.are.equal(212, tonumber(last_open)) -- max(start_time + duration) = 200 + 12 + end) +end) diff --git a/docker/volumes/db/migrations/014_add_reading_stats.sql b/docker/volumes/db/migrations/014_add_reading_stats.sql new file mode 100644 index 00000000..0a3b9878 --- /dev/null +++ b/docker/volumes/db/migrations/014_add_reading_stats.sql @@ -0,0 +1,41 @@ +-- Migration 014: reading statistics sync (KOReader-compatible page events) + +CREATE TABLE IF NOT EXISTS public.stat_books ( + user_id uuid NOT NULL, + book_hash text NOT NULL, + title text NOT NULL DEFAULT '', + authors text NOT NULL DEFAULT '', + updated_at timestamp with time zone NOT NULL DEFAULT now(), + deleted_at timestamp with time zone NULL, + CONSTRAINT stat_books_pkey PRIMARY KEY (user_id, book_hash), + CONSTRAINT stat_books_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_stat_books_user_updated ON public.stat_books (user_id, updated_at); + +CREATE TABLE IF NOT EXISTS public.stat_pages ( + user_id uuid NOT NULL, + book_hash text NOT NULL, + page integer NOT NULL, + start_time bigint NOT NULL, + duration integer NOT NULL DEFAULT 0, + total_pages integer NOT NULL DEFAULT 0, + ext jsonb NULL, + updated_at timestamp with time zone NOT NULL DEFAULT now(), + deleted_at timestamp with time zone NULL, + CONSTRAINT stat_pages_pkey PRIMARY KEY (user_id, book_hash, page, start_time), + CONSTRAINT stat_pages_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_stat_pages_user_updated ON public.stat_pages (user_id, updated_at); + +ALTER TABLE public.stat_books ENABLE ROW LEVEL SECURITY; +ALTER TABLE public.stat_pages ENABLE ROW LEVEL SECURITY; + +CREATE POLICY stat_books_select ON public.stat_books FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_insert ON public.stat_books FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_update ON public.stat_books FOR UPDATE TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_books_delete ON public.stat_books FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id); + +CREATE POLICY stat_pages_select ON public.stat_pages FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_insert ON public.stat_pages FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_update ON public.stat_pages FOR UPDATE to authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY stat_pages_delete ON public.stat_pages FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id);