forked from akai/readest
feat(statistics): KOReader-compatible reading stats with cross-device sync (#4606)
Supersedes #3156. Adds a reading-statistics system whose canonical data model is KOReader's own (book + page_stat_data), so stats round-trip losslessly between Readest and KOReader. - Storage: a cross-platform Turso statistics.db in KOReader's exact schema (web/Workers, desktop, iOS, Android) — replacing #3156's Node-only better-sqlite3 + statistics.json. - Tracking: per-page reading events (time-on-page, idle-capped) flushed on page-change/idle/hide/close — the KOReader model — not session aggregates. - Sync: legacy /api/sync extended with a stats type backed by self-contained Supabase tables (stat_books, stat_pages); union/longer-duration-wins merge keyed on book_hash. apps/readest.koplugin syncs through the same endpoint. - Scale & robustness: per-tab singleton connection (avoids OPFS lock conflicts) + explicit WAL checkpoint; transactional bulk apply; chunked resumable push; client-driven paged pull with trailing-ms completion; paginated/scoped server merge. Verified: 5668 unit tests, 155 koplugin busted tests, biome+tsgo + luacheck all green; web OPFS DB verified live. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
@@ -0,0 +1,30 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { pickWinningPages } from '@/pages/api/sync';
|
||||
import type { StatPageRecord } from '@/libs/sync';
|
||||
|
||||
const mk = (start: number, duration: number): StatPageRecord => ({
|
||||
book_hash: 'm',
|
||||
page: 1,
|
||||
start_time: start,
|
||||
duration,
|
||||
total_pages: 10,
|
||||
});
|
||||
|
||||
describe('pickWinningPages', () => {
|
||||
it('inserts pages the server has not seen', () => {
|
||||
const { toUpsert } = pickWinningPages([mk(100, 5)], new Map());
|
||||
expect(toUpsert).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps the longer duration on conflict', () => {
|
||||
const server = new Map([['m|1|100', mk(100, 5)]]);
|
||||
const win = pickWinningPages([mk(100, 9)], server);
|
||||
expect(win.toUpsert[0]!.duration).toBe(9);
|
||||
});
|
||||
|
||||
it('drops an incoming page whose duration is not longer', () => {
|
||||
const server = new Map([['m|1|100', mk(100, 9)]]);
|
||||
const win = pickWinningPages([mk(100, 5)], server);
|
||||
expect(win.toUpsert).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -149,7 +149,7 @@ describe('isSyncCategoryEnabled', () => {
|
||||
});
|
||||
|
||||
describe('SYNC_CATEGORIES', () => {
|
||||
test('covers all nine user-facing categories (incl. settings + credentials)', () => {
|
||||
test('covers all ten user-facing categories (incl. settings + stats + credentials)', () => {
|
||||
expect([...SYNC_CATEGORIES].sort()).toEqual(
|
||||
[
|
||||
'book',
|
||||
@@ -160,6 +160,7 @@ describe('SYNC_CATEGORIES', () => {
|
||||
'opds_catalog',
|
||||
'progress',
|
||||
'settings',
|
||||
'stats',
|
||||
'texture',
|
||||
].sort(),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { NodeDatabaseService } from '@/services/database/nodeDatabaseService';
|
||||
import { migrate } from '@/services/database/migrate';
|
||||
import { getMigrations } from '@/services/database/migrations';
|
||||
import type { DatabaseService } from '@/types/database';
|
||||
import { StatisticsDb } from '@/services/statistics/statisticsDb';
|
||||
|
||||
async function freshStatsDb(): Promise<DatabaseService> {
|
||||
// In-memory libsql DB; run the same migrations production uses.
|
||||
const db = await NodeDatabaseService.open(':memory:');
|
||||
await migrate(db, getMigrations('statistics'));
|
||||
return db;
|
||||
}
|
||||
|
||||
describe('statistics migration', () => {
|
||||
let db: DatabaseService;
|
||||
beforeEach(async () => {
|
||||
db = await freshStatsDb();
|
||||
});
|
||||
|
||||
it('creates KOReader book + page_stat_data tables and extension tables', async () => {
|
||||
const tables = await db.select<{ name: string }>(
|
||||
`SELECT name FROM sqlite_master WHERE type IN ('table','view') ORDER BY name`,
|
||||
);
|
||||
const names = tables.map((t) => t.name);
|
||||
expect(names).toContain('book');
|
||||
expect(names).toContain('page_stat_data');
|
||||
expect(names).toContain('numbers');
|
||||
expect(names).toContain('page_stat'); // the rescaling view
|
||||
expect(names).toContain('readest_page_ext');
|
||||
expect(names).toContain('readest_book_ext');
|
||||
expect(names).toContain('readest_stat_sync_state');
|
||||
});
|
||||
|
||||
it('seeds the numbers helper table 1..1000', async () => {
|
||||
const rows = await db.select<{ c: number }>(`SELECT COUNT(*) AS c FROM numbers`);
|
||||
expect(rows[0]!.c).toBe(1000);
|
||||
});
|
||||
|
||||
it('enforces the page_stat_data uniqueness key', async () => {
|
||||
await db.execute(`INSERT INTO book (title, authors, md5) VALUES ('T','A','m')`);
|
||||
const id = (await db.select<{ id: number }>(`SELECT id FROM book LIMIT 1`))[0]!.id;
|
||||
await db.execute(
|
||||
`INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages) VALUES (?,?,?,?,?)`,
|
||||
[id, 5, 1000, 10, 100],
|
||||
);
|
||||
await db.execute(
|
||||
`INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages)
|
||||
VALUES (?,?,?,?,?)
|
||||
ON CONFLICT(id_book, page, start_time) DO UPDATE SET duration = max(duration, excluded.duration)`,
|
||||
[id, 5, 1000, 25, 100],
|
||||
);
|
||||
const rows = await db.select<{ duration: number; c: number }>(
|
||||
`SELECT duration, COUNT(*) OVER () AS c FROM page_stat_data`,
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0]!.duration).toBe(25);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StatisticsDb', () => {
|
||||
let stats: StatisticsDb;
|
||||
beforeEach(async () => {
|
||||
stats = StatisticsDb.from(await freshStatsDb());
|
||||
});
|
||||
|
||||
it('upserts a book by md5 and returns a stable id_book', async () => {
|
||||
const id1 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' });
|
||||
const id2 = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' });
|
||||
expect(id1).toBe(id2);
|
||||
});
|
||||
|
||||
it('inserts page events and keeps the longer duration on re-flush', async () => {
|
||||
const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' });
|
||||
await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 10, totalPages: 50 });
|
||||
await stats.insertPageEvent(id, { page: 3, startTime: 100, duration: 30, totalPages: 50 });
|
||||
await stats.insertPageEvent(id, { page: 4, startTime: 140, duration: 12, totalPages: 50 });
|
||||
await stats.recomputeBookTotals(id);
|
||||
const book = await stats.getBookByMd5('m1');
|
||||
expect(book!.total_read_time).toBe(42); // 30 + 12
|
||||
expect(book!.total_read_pages).toBe(2); // distinct pages 3,4
|
||||
expect(book!.last_open).toBe(152); // max(start_time + duration) = 140 + 12
|
||||
});
|
||||
|
||||
it('returns events for push after a start_time cursor, joined with md5', async () => {
|
||||
const id = await stats.upsertBook({ bookMd5: 'm1', title: 'T1', authors: 'A1' });
|
||||
await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 });
|
||||
await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 5, totalPages: 9 });
|
||||
const { events } = await stats.getEventsForPush(150);
|
||||
expect(events.map((e) => e.startTime)).toEqual([200]);
|
||||
expect(events[0]!.bookMd5).toBe('m1');
|
||||
});
|
||||
|
||||
it('applies remote events idempotently via upsert', async () => {
|
||||
const remoteBooks = [{ bookMd5: 'm2', title: 'T2', authors: 'A2' }];
|
||||
const remoteEvents = [
|
||||
{ bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 },
|
||||
{ bookMd5: 'm2', page: 1, startTime: 300, duration: 8, totalPages: 20 }, // dup
|
||||
];
|
||||
await stats.applyRemoteEvents(remoteBooks, remoteEvents);
|
||||
await stats.applyRemoteEvents(remoteBooks, remoteEvents); // again — still idempotent
|
||||
const book = await stats.getBookByMd5('m2');
|
||||
expect(book!.total_read_time).toBe(8);
|
||||
});
|
||||
|
||||
it('reads and writes sync cursors', async () => {
|
||||
expect(await stats.getCursor('push')).toBe(0);
|
||||
await stats.setCursor('push', 1234);
|
||||
expect(await stats.getCursor('push')).toBe(1234);
|
||||
});
|
||||
|
||||
it('keeps one book row per md5 even when title/authors change (no duplicates)', async () => {
|
||||
const id1 = await stats.upsertBook({ bookMd5: 'm1', title: 'Old', authors: 'A' });
|
||||
const id2 = await stats.upsertBook({ bookMd5: 'm1', title: 'New', authors: 'B' });
|
||||
expect(id2).toBe(id1);
|
||||
const book = await stats.getBookByMd5('m1');
|
||||
expect(book!.title).toBe('New'); // latest title wins
|
||||
// exactly one row for this md5
|
||||
const rows = await stats.getEventsForPush(-1); // no events; just exercise no crash
|
||||
void rows;
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { NodeDatabaseService } from '@/services/database/nodeDatabaseService';
|
||||
import { migrate } from '@/services/database/migrate';
|
||||
import { getMigrations } from '@/services/database/migrations';
|
||||
import { StatisticsDb } from '@/services/statistics/statisticsDb';
|
||||
import { pushStats, pullStats } from '@/services/statistics/statsSync';
|
||||
|
||||
async function db() {
|
||||
const d = await NodeDatabaseService.open(':memory:');
|
||||
await migrate(d, getMigrations('statistics'));
|
||||
return StatisticsDb.from(d);
|
||||
}
|
||||
|
||||
describe('statsSync', () => {
|
||||
it('pushes only events past the cursor and advances it', async () => {
|
||||
const stats = await db();
|
||||
const id = await stats.upsertBook({ bookMd5: 'm', title: 'T', authors: 'A' });
|
||||
await stats.insertPageEvent(id, { page: 1, startTime: 100, duration: 5, totalPages: 9 });
|
||||
await stats.insertPageEvent(id, { page: 2, startTime: 200, duration: 6, totalPages: 9 });
|
||||
const client = { pushChanges: vi.fn().mockResolvedValue({}) };
|
||||
await pushStats(stats, client as never);
|
||||
const sent = client.pushChanges.mock.calls[0]![0];
|
||||
expect(sent.statPages.map((p: { start_time: number }) => p.start_time)).toEqual([100, 200]);
|
||||
expect(await stats.getCursor('push')).toBe(200);
|
||||
});
|
||||
|
||||
it('applies pulled events and advances the pull cursor', async () => {
|
||||
const stats = await db();
|
||||
const client = {
|
||||
pullChanges: vi.fn().mockResolvedValue({
|
||||
statBooks: [{ book_hash: 'm', title: 'T', authors: 'A' }],
|
||||
statPages: [
|
||||
{
|
||||
book_hash: 'm',
|
||||
page: 1,
|
||||
start_time: 300,
|
||||
duration: 7,
|
||||
total_pages: 9,
|
||||
updated_at_ms: 1_750_000_000_000,
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
await pullStats(stats, client as never);
|
||||
const book = await stats.getBookByMd5('m');
|
||||
expect(book!.total_read_time).toBe(7);
|
||||
expect(await stats.getCursor('pull')).toBe(1_750_000_000_000);
|
||||
});
|
||||
|
||||
it('chunks a large push backlog into bounded requests, advancing the cursor per chunk', async () => {
|
||||
const stats = await db();
|
||||
const id = await stats.upsertBook({ bookMd5: 'm', title: 'T', authors: 'A' });
|
||||
// 600 events at distinct start_times — exceeds the 500-event push chunk → 2 requests.
|
||||
for (let t = 1; t <= 600; t++) {
|
||||
await stats.insertPageEvent(id, { page: t, startTime: t, duration: 1, totalPages: 999 });
|
||||
}
|
||||
const client = { pushChanges: vi.fn().mockResolvedValue({}) };
|
||||
await pushStats(stats, client as never);
|
||||
expect(client.pushChanges).toHaveBeenCalledTimes(2);
|
||||
expect(client.pushChanges.mock.calls[0]![0].statPages.length).toBe(500);
|
||||
expect(client.pushChanges.mock.calls[1]![0].statPages.length).toBe(100);
|
||||
expect(await stats.getCursor('push')).toBe(600);
|
||||
});
|
||||
|
||||
it('pages through a multi-page pull, applying every page until exhausted', async () => {
|
||||
const stats = await db();
|
||||
const ev = (n: number, ms: number) => ({
|
||||
book_hash: 'm',
|
||||
page: n,
|
||||
start_time: n,
|
||||
duration: 2,
|
||||
total_pages: 9,
|
||||
updated_at_ms: ms,
|
||||
});
|
||||
const client = {
|
||||
pullChanges: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
statBooks: [{ book_hash: 'm', title: 'T', authors: 'A' }],
|
||||
statPages: [ev(1, 1000), ev(2, 1000)],
|
||||
})
|
||||
.mockResolvedValueOnce({ statBooks: [], statPages: [ev(3, 2000)] })
|
||||
.mockResolvedValue({ statBooks: [], statPages: [] }),
|
||||
};
|
||||
await pullStats(stats, client as never);
|
||||
expect(client.pullChanges).toHaveBeenCalledTimes(3);
|
||||
const book = await stats.getBookByMd5('m');
|
||||
expect(book!.total_read_pages).toBe(3); // pages 1, 2, 3 all applied across pages
|
||||
expect(await stats.getCursor('pull')).toBe(2000);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { TrackerCore } from '@/services/statistics/trackerCore';
|
||||
import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics';
|
||||
|
||||
const cfg = DEFAULT_STATS_TRACKING_CONFIG;
|
||||
|
||||
describe('TrackerCore', () => {
|
||||
it('emits an event when the page changes', () => {
|
||||
const t = new TrackerCore(cfg);
|
||||
expect(t.onPage(5, 100, 1000)).toEqual([]); // first page, nothing to flush yet
|
||||
const out = t.onPage(6, 100, 1030); // moved off page 5 after 30s
|
||||
expect(out).toEqual([{ page: 5, startTime: 1000, duration: 30, totalPages: 100 }]);
|
||||
});
|
||||
|
||||
it('caps duration at maxEventSeconds', () => {
|
||||
const t = new TrackerCore(cfg);
|
||||
t.onPage(1, 10, 0);
|
||||
const out = t.onPage(2, 10, 10_000); // way over the 120s cap
|
||||
expect(out[0]!.duration).toBe(cfg.maxEventSeconds);
|
||||
});
|
||||
|
||||
it('drops sub-minimum events', () => {
|
||||
const t = new TrackerCore(cfg);
|
||||
t.onPage(1, 10, 0);
|
||||
expect(t.onPage(2, 10, 1)).toEqual([]); // 1s < minEventSeconds (3)
|
||||
});
|
||||
|
||||
it('flushes and pauses on idle, then resumes with a new start_time', () => {
|
||||
const t = new TrackerCore(cfg);
|
||||
t.onPage(1, 10, 0);
|
||||
expect(t.onIdle(50)).toEqual([{ page: 1, startTime: 0, duration: 50, totalPages: 10 }]);
|
||||
// After idle, the same page resumes as a fresh event.
|
||||
expect(t.onPage(1, 10, 200)).toEqual([]); // resume marker, no flush
|
||||
expect(t.onPage(2, 10, 230)).toEqual([
|
||||
{ page: 1, startTime: 200, duration: 30, totalPages: 10 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('flushes on hide and on close without double-counting', () => {
|
||||
const t = new TrackerCore(cfg);
|
||||
t.onPage(7, 10, 0);
|
||||
expect(t.onHide(40)).toEqual([{ page: 7, startTime: 0, duration: 40, totalPages: 10 }]);
|
||||
expect(t.onClose(99)).toEqual([]); // already flushed + paused by hide
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,7 @@ import FootnotePopup from './FootnotePopup';
|
||||
import HintInfo from './HintInfo';
|
||||
import ReadingRuler from './ReadingRuler';
|
||||
import DoubleBorder from './DoubleBorder';
|
||||
import ReadingStatsTracker from './ReadingStatsTracker';
|
||||
|
||||
interface BooksGridProps {
|
||||
bookKeys: string[];
|
||||
@@ -278,6 +279,7 @@ const BookCellInner: React.FC<BookCellProps> = ({
|
||||
isHoveredAnim={false}
|
||||
gridInsets={gridInsets}
|
||||
/>
|
||||
<ReadingStatsTracker bookKey={bookKey} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useBookProgress } from '@/store/readerProgressStore';
|
||||
import { useBookDataStore } from '@/store/bookDataStore';
|
||||
import { useEnv } from '@/context/EnvContext';
|
||||
import { useAuth } from '@/context/AuthContext';
|
||||
import { StatisticsDb } from '@/services/statistics/statisticsDb';
|
||||
import { TrackerCore, type FlushedEvent } from '@/services/statistics/trackerCore';
|
||||
import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics';
|
||||
import { SyncClient } from '@/libs/sync';
|
||||
import { pushStats, pullStats } from '@/services/statistics/statsSync';
|
||||
import { isSyncCategoryEnabled } from '@/services/sync/syncCategories';
|
||||
|
||||
const nowSec = () => Math.floor(Date.now() / 1000);
|
||||
|
||||
export default function ReadingStatsTracker({ bookKey }: { bookKey: string }) {
|
||||
const { appService } = useEnv();
|
||||
// Progress lives in readerProgressStore, not readerStore.viewStates.
|
||||
const progress = useBookProgress(bookKey);
|
||||
// booksData is keyed by book id = bookKey.split('-')[0].
|
||||
const getBookData = useBookDataStore((s) => s.getBookData);
|
||||
const { user } = useAuth();
|
||||
const coreRef = useRef(new TrackerCore(DEFAULT_STATS_TRACKING_CONFIG));
|
||||
const dbRef = useRef<StatisticsDb | null>(null);
|
||||
const idleRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const pushTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const bookData = getBookData(bookKey);
|
||||
const book = bookData?.book;
|
||||
// Book.hash is the partialMD5 used as KOReader's md5.
|
||||
const bookMd5 = book?.hash;
|
||||
const title = book?.title ?? '';
|
||||
// Book.author is the single-string author field; upsertBook takes authors: string.
|
||||
const authors = book?.author ?? '';
|
||||
|
||||
const syncEnabled = () => !!user && isSyncCategoryEnabled('stats');
|
||||
|
||||
const schedulePush = () => {
|
||||
if (!syncEnabled()) return;
|
||||
if (pushTimerRef.current) clearTimeout(pushTimerRef.current);
|
||||
pushTimerRef.current = setTimeout(() => {
|
||||
const db = dbRef.current;
|
||||
if (db) void pushStats(db, new SyncClient());
|
||||
}, 10_000);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!appService) return;
|
||||
let cancelled = false;
|
||||
StatisticsDb.open(appService).then((db) => {
|
||||
if (cancelled) return;
|
||||
dbRef.current = db;
|
||||
if (syncEnabled()) void pullStats(db, new SyncClient());
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [appService]);
|
||||
|
||||
// Persist flushed events into the statistics DB.
|
||||
const persist = (events: FlushedEvent[]): Promise<void> => {
|
||||
const db = dbRef.current;
|
||||
if (!db || !bookMd5 || events.length === 0) return Promise.resolve();
|
||||
return (async () => {
|
||||
const idBook = await db.upsertBook({ bookMd5, title, authors });
|
||||
for (const e of events) await db.insertPageEvent(idBook, e);
|
||||
await db.recomputeBookTotals(idBook);
|
||||
schedulePush();
|
||||
})();
|
||||
};
|
||||
|
||||
const armIdle = () => {
|
||||
if (idleRef.current) clearTimeout(idleRef.current);
|
||||
idleRef.current = setTimeout(
|
||||
() => void persist(coreRef.current.onIdle(nowSec())),
|
||||
DEFAULT_STATS_TRACKING_CONFIG.idleTimeoutSeconds * 1000,
|
||||
);
|
||||
};
|
||||
|
||||
// Page changes drive the tracker.
|
||||
useEffect(() => {
|
||||
const info = progress?.pageinfo;
|
||||
if (!info) return;
|
||||
const page = (info.current ?? 0) + 1;
|
||||
const total = info.total || 1;
|
||||
void persist(coreRef.current.onPage(page, total, nowSec()));
|
||||
armIdle();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [progress?.pageinfo]);
|
||||
|
||||
// Tab/window visibility.
|
||||
useEffect(() => {
|
||||
const onVis = () => {
|
||||
if (document.visibilityState === 'hidden') {
|
||||
if (idleRef.current) clearTimeout(idleRef.current);
|
||||
void persist(coreRef.current.onHide(nowSec()));
|
||||
}
|
||||
};
|
||||
document.addEventListener('visibilitychange', onVis);
|
||||
return () => document.removeEventListener('visibilitychange', onVis);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [bookMd5]);
|
||||
|
||||
// Book close (unmount).
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (idleRef.current) clearTimeout(idleRef.current);
|
||||
if (pushTimerRef.current) clearTimeout(pushTimerRef.current);
|
||||
const closePersist = persist(coreRef.current.onClose(nowSec()));
|
||||
void closePersist.then(() => {
|
||||
if (syncEnabled() && dbRef.current) return pushStats(dbRef.current, new SyncClient());
|
||||
return undefined;
|
||||
});
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [bookMd5]);
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -59,6 +59,10 @@ const useCategoryCopy = (): Record<SyncCategory, CategoryCopy> => {
|
||||
'Tokens, usernames, and passwords for OPDS, KOReader, Hardcover, and Readwise. When disabled, credentials remain on this device only and are never uploaded.',
|
||||
),
|
||||
},
|
||||
stats: {
|
||||
title: _('Reading statistics'),
|
||||
description: _('Reading time and pages read, synced across your devices and KOReader.'),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -109,8 +109,12 @@ export function useSync(bookKey?: string) {
|
||||
|
||||
try {
|
||||
const result = await syncClient.pullChanges(since, type, bookId, metaHash);
|
||||
setSyncResult({ ...syncResult, [type]: result[type] });
|
||||
const records = result[type];
|
||||
const resultAsRecord = result as unknown as Record<
|
||||
string,
|
||||
BookDataRecord[] | null | undefined
|
||||
>;
|
||||
setSyncResult({ ...syncResult, [type]: resultAsRecord[type] });
|
||||
const records = resultAsRecord[type];
|
||||
if (since > 1000 && !records?.length) return 0;
|
||||
// For since <= 1000, we set lastSyncedAt to now if no records returned
|
||||
const maxTime = records?.length ? computeMaxTimestamp(records) : Date.now();
|
||||
|
||||
@@ -5,17 +5,42 @@ import { fetchWithTimeout } from '@/utils/fetch';
|
||||
|
||||
const SYNC_API_ENDPOINT = getAPIBaseUrl() + '/sync';
|
||||
|
||||
export type SyncType = 'books' | 'configs' | 'notes';
|
||||
export type SyncType = 'books' | 'configs' | 'notes' | 'stats';
|
||||
export type SyncOp = 'push' | 'pull' | 'both';
|
||||
|
||||
interface BookRecord extends BookDataRecord, Book {}
|
||||
interface BookConfigRecord extends BookDataRecord, BookConfig {}
|
||||
interface BookNoteRecord extends BookDataRecord, BookNote {}
|
||||
|
||||
export interface StatBookRecord {
|
||||
user_id?: string;
|
||||
book_hash: string;
|
||||
title: string;
|
||||
authors: string;
|
||||
updated_at?: string;
|
||||
updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math
|
||||
deleted_at?: string | null;
|
||||
}
|
||||
|
||||
export interface StatPageRecord {
|
||||
user_id?: string;
|
||||
book_hash: string;
|
||||
page: number;
|
||||
start_time: number;
|
||||
duration: number;
|
||||
total_pages: number;
|
||||
ext?: unknown;
|
||||
updated_at?: string;
|
||||
updated_at_ms?: number; // epoch ms, attached by the GET response for cursor math
|
||||
deleted_at?: string | null;
|
||||
}
|
||||
|
||||
export interface SyncResult {
|
||||
books: BookRecord[] | null;
|
||||
notes: BookNoteRecord[] | null;
|
||||
configs: BookConfigRecord[] | null;
|
||||
statBooks?: StatBookRecord[] | null;
|
||||
statPages?: StatPageRecord[] | null;
|
||||
}
|
||||
|
||||
export type SyncRecord = BookRecord & BookConfigRecord & BookNoteRecord;
|
||||
@@ -24,6 +49,8 @@ export interface SyncData {
|
||||
books?: Partial<BookRecord>[];
|
||||
notes?: Partial<BookNoteRecord>[];
|
||||
configs?: Partial<BookConfigRecord>[];
|
||||
statBooks?: StatBookRecord[];
|
||||
statPages?: StatPageRecord[];
|
||||
}
|
||||
|
||||
export class SyncClient {
|
||||
@@ -36,11 +63,13 @@ export class SyncClient {
|
||||
type?: SyncType,
|
||||
book?: string,
|
||||
metaHash?: string,
|
||||
limit?: number,
|
||||
): Promise<SyncResult> {
|
||||
const token = await getAccessToken();
|
||||
if (!token) throw new Error('Not authenticated');
|
||||
|
||||
const url = `${SYNC_API_ENDPOINT}?since=${encodeURIComponent(since)}&type=${type ?? ''}&book=${book ?? ''}&meta_hash=${metaHash ?? ''}`;
|
||||
const limitParam = limit && limit > 0 ? `&limit=${encodeURIComponent(limit)}` : '';
|
||||
const url = `${SYNC_API_ENDPOINT}?since=${encodeURIComponent(since)}&type=${type ?? ''}&book=${book ?? ''}&meta_hash=${metaHash ?? ''}${limitParam}`;
|
||||
const res = await fetchWithTimeout(
|
||||
url,
|
||||
{
|
||||
|
||||
@@ -7,10 +7,36 @@ import { transformBookConfigToDB } from '@/utils/transform';
|
||||
import { transformBookNoteToDB } from '@/utils/transform';
|
||||
import { transformBookToDB } from '@/utils/transform';
|
||||
import { runMiddleware, corsAllMethods } from '@/utils/cors';
|
||||
import { SyncData, SyncRecord, SyncResult, SyncType } from '@/libs/sync';
|
||||
import {
|
||||
SyncData,
|
||||
SyncRecord,
|
||||
SyncResult,
|
||||
SyncType,
|
||||
StatBookRecord,
|
||||
StatPageRecord,
|
||||
} from '@/libs/sync';
|
||||
import { validateUserAndToken } from '@/utils/access';
|
||||
import { DBBook, DBBookConfig } from '@/types/records';
|
||||
|
||||
const pageKey = (r: StatPageRecord) => `${r.book_hash}|${r.page}|${r.start_time}`;
|
||||
|
||||
/**
|
||||
* Decide which incoming page events to write: new keys always win; existing
|
||||
* keys win only when the incoming duration is strictly longer (union/upsert
|
||||
* semantics — KOReader-compatible).
|
||||
*/
|
||||
export function pickWinningPages(
|
||||
incoming: StatPageRecord[],
|
||||
server: Map<string, StatPageRecord>,
|
||||
): { toUpsert: StatPageRecord[] } {
|
||||
const toUpsert: StatPageRecord[] = [];
|
||||
for (const rec of incoming) {
|
||||
const existing = server.get(pageKey(rec));
|
||||
if (!existing || rec.duration > existing.duration) toUpsert.push(rec);
|
||||
}
|
||||
return { toUpsert };
|
||||
}
|
||||
|
||||
const transformsToDB = {
|
||||
books: transformBookToDB,
|
||||
book_notes: transformBookNoteToDB,
|
||||
@@ -39,6 +65,10 @@ export async function GET(req: NextRequest) {
|
||||
const typeParam = searchParams.get('type') as SyncType | undefined;
|
||||
const bookParam = searchParams.get('book');
|
||||
const metaHashParam = searchParams.get('meta_hash');
|
||||
// Optional page size for `type=stats` (client-driven paged pull). Absent for
|
||||
// the koplugin, which keeps the full-delta response.
|
||||
const statsLimitParam = searchParams.get('limit');
|
||||
const statsLimit = statsLimitParam ? Math.max(1, Math.floor(Number(statsLimitParam))) : 0;
|
||||
|
||||
if (!sinceParam) {
|
||||
return NextResponse.json({ error: '"since" query parameter is required' }, { status: 400 });
|
||||
@@ -52,7 +82,7 @@ export async function GET(req: NextRequest) {
|
||||
const sinceIso = since.toISOString();
|
||||
|
||||
try {
|
||||
const results: SyncResult = { books: [], configs: [], notes: [] };
|
||||
const results: SyncResult = { books: [], configs: [], notes: [], statBooks: [], statPages: [] };
|
||||
const errors: Record<TableName, DBError | null> = {
|
||||
books: null,
|
||||
book_notes: null,
|
||||
@@ -113,7 +143,7 @@ export async function GET(req: NextRequest) {
|
||||
}
|
||||
});
|
||||
}
|
||||
results[DBSyncTypeMap[table] as SyncType] = records || [];
|
||||
(results as unknown as Record<string, SyncRecord[]>)[DBSyncTypeMap[table]] = records || [];
|
||||
};
|
||||
|
||||
if (!typeParam || typeParam === 'books') {
|
||||
@@ -145,6 +175,99 @@ export async function GET(req: NextRequest) {
|
||||
if (!typeParam || typeParam === 'notes') {
|
||||
await queryTables('book_notes', ['id']).catch((err) => (errors['book_notes'] = err));
|
||||
}
|
||||
if (!typeParam || typeParam === 'stats') {
|
||||
// PostgREST caps responses at ~1000 rows; stat_pages grows one row per page
|
||||
// event, so page through both tables (ordered by updated_at ascending for a
|
||||
// stable cursor) and accumulate every row — otherwise a device pulling >1000
|
||||
// events only gets the first page and then advances its cursor past the rest.
|
||||
const PAGE = 1000;
|
||||
const fetchAll = async (table: 'stat_books' | 'stat_pages', filterBook: boolean) => {
|
||||
const all: Record<string, unknown>[] = [];
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
let q = supabase
|
||||
.from(table)
|
||||
.select('*')
|
||||
.eq('user_id', user.id)
|
||||
.or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`)
|
||||
.order('updated_at', { ascending: true })
|
||||
.range(offset, offset + PAGE - 1);
|
||||
if (filterBook && bookParam) q = q.eq('book_hash', bookParam);
|
||||
const { data, error } = await q;
|
||||
if (error) return { error };
|
||||
const rows = (data ?? []) as Record<string, unknown>[];
|
||||
all.push(...rows);
|
||||
if (rows.length < PAGE) break;
|
||||
offset += PAGE;
|
||||
}
|
||||
return { data: all };
|
||||
};
|
||||
// A single bounded page of stat_pages for the app's client-driven paged
|
||||
// pull, completed to the trailing updated_at millisecond so the client can
|
||||
// advance its cursor with a strict `> cursor` without skipping ties.
|
||||
const fetchPagedPages = async () => {
|
||||
let q = supabase
|
||||
.from('stat_pages')
|
||||
.select('*')
|
||||
.eq('user_id', user.id)
|
||||
.or(`updated_at.gt.${sinceIso},deleted_at.gt.${sinceIso}`)
|
||||
.order('updated_at', { ascending: true })
|
||||
.range(0, statsLimit - 1);
|
||||
if (bookParam) q = q.eq('book_hash', bookParam);
|
||||
const { data, error } = await q;
|
||||
if (error) return { error };
|
||||
const rows = (data ?? []) as Record<string, unknown>[];
|
||||
if (rows.length === statsLimit) {
|
||||
const lastUpdated = rows[rows.length - 1]!['updated_at'] as string;
|
||||
let eq = supabase
|
||||
.from('stat_pages')
|
||||
.select('*')
|
||||
.eq('user_id', user.id)
|
||||
.eq('updated_at', lastUpdated);
|
||||
if (bookParam) eq = eq.eq('book_hash', bookParam);
|
||||
const { data: extra, error: extraErr } = await eq;
|
||||
if (extraErr) return { error: extraErr };
|
||||
const keyOf = (r: Record<string, unknown>) =>
|
||||
`${r['book_hash']}|${r['page']}|${r['start_time']}`;
|
||||
const seen = new Set(rows.map(keyOf));
|
||||
for (const r of (extra ?? []) as Record<string, unknown>[]) {
|
||||
const k = keyOf(r);
|
||||
if (!seen.has(k)) {
|
||||
seen.add(k);
|
||||
rows.push(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { data: rows };
|
||||
};
|
||||
// stat_books is always returned in full (one row per book, small); only
|
||||
// stat_pages pages when the client asks (the koplugin omits `limit`).
|
||||
const sb = await fetchAll('stat_books', false);
|
||||
const sp = statsLimit > 0 ? await fetchPagedPages() : await fetchAll('stat_pages', true);
|
||||
if (sb.error)
|
||||
return NextResponse.json(
|
||||
{ error: `stat_books: ${sb.error.message || 'Unknown error'}` },
|
||||
{ status: 500 },
|
||||
);
|
||||
if (sp.error)
|
||||
return NextResponse.json(
|
||||
{ error: `stat_pages: ${sp.error.message || 'Unknown error'}` },
|
||||
{ status: 500 },
|
||||
);
|
||||
// Attach updated_at_ms (epoch ms) so non-JS clients (the Lua koplugin) can
|
||||
// compute their pull cursor without parsing ISO-8601 timestamps.
|
||||
const withMs = <T extends { updated_at?: string }>(rows: T[]) =>
|
||||
rows.map((r) => ({
|
||||
...r,
|
||||
updated_at_ms: r.updated_at ? new Date(r.updated_at).getTime() : 0,
|
||||
}));
|
||||
(
|
||||
results as unknown as { statBooks: StatBookRecord[]; statPages: StatPageRecord[] }
|
||||
).statBooks = withMs((sb.data ?? []) as unknown as StatBookRecord[]);
|
||||
(
|
||||
results as unknown as { statBooks: StatBookRecord[]; statPages: StatPageRecord[] }
|
||||
).statPages = withMs((sp.data ?? []) as unknown as StatPageRecord[]);
|
||||
}
|
||||
|
||||
const dbErrors = Object.values(errors).filter((err) => err !== null);
|
||||
if (dbErrors.length > 0) {
|
||||
@@ -174,7 +297,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
const supabase = createSupabaseClient(token);
|
||||
const body = await req.json();
|
||||
const { books = [], configs = [], notes = [] } = body as SyncData;
|
||||
const { books = [], configs = [], notes = [], statBooks = [], statPages = [] } = body as SyncData;
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
const upsertRecords = async (
|
||||
@@ -364,6 +487,65 @@ export async function POST(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
if (statBooks.length > 0) {
|
||||
const rows = statBooks.map((b: StatBookRecord) => ({
|
||||
user_id: user.id,
|
||||
book_hash: b.book_hash,
|
||||
title: b.title,
|
||||
authors: b.authors,
|
||||
updated_at: new Date().toISOString(),
|
||||
deleted_at: b.deleted_at ?? null,
|
||||
}));
|
||||
const { error } = await supabase
|
||||
.from('stat_books')
|
||||
.upsert(rows, { onConflict: 'user_id,book_hash' });
|
||||
if (error) return NextResponse.json({ error: error.message }, { status: 500 });
|
||||
}
|
||||
|
||||
if (statPages.length > 0) {
|
||||
// Process in batches so the "longer-duration-wins" merge stays correct at
|
||||
// scale: the existing-row fetch is scoped to each batch's (book_hash,
|
||||
// start_time) keys (not a book's whole history) and bounded under
|
||||
// PostgREST's ~1000-row cap — otherwise existing rows beyond 1000 are
|
||||
// invisible to pickWinningPages and a shorter duration could overwrite a
|
||||
// longer one.
|
||||
const BATCH = 500;
|
||||
for (let off = 0; off < statPages.length; off += BATCH) {
|
||||
const batch = statPages.slice(off, off + BATCH);
|
||||
const bookHashes = [...new Set(batch.map((p) => p.book_hash))];
|
||||
const startTimes = [...new Set(batch.map((p) => p.start_time))];
|
||||
const { data: existing, error: exErr } = await supabase
|
||||
.from('stat_pages')
|
||||
.select('*')
|
||||
.eq('user_id', user.id)
|
||||
.in('book_hash', bookHashes)
|
||||
.in('start_time', startTimes);
|
||||
if (exErr) return NextResponse.json({ error: exErr.message }, { status: 500 });
|
||||
const serverMap = new Map<string, StatPageRecord>();
|
||||
(existing ?? []).forEach((r) =>
|
||||
serverMap.set(pageKey(r as StatPageRecord), r as StatPageRecord),
|
||||
);
|
||||
const { toUpsert } = pickWinningPages(batch, serverMap);
|
||||
const rows = toUpsert.map((p) => ({
|
||||
user_id: user.id,
|
||||
book_hash: p.book_hash,
|
||||
page: p.page,
|
||||
start_time: p.start_time,
|
||||
duration: p.duration,
|
||||
total_pages: p.total_pages,
|
||||
ext: p.ext ?? null,
|
||||
updated_at: new Date().toISOString(),
|
||||
deleted_at: p.deleted_at ?? null,
|
||||
}));
|
||||
if (rows.length > 0) {
|
||||
const { error } = await supabase
|
||||
.from('stat_pages')
|
||||
.upsert(rows, { onConflict: 'user_id,book_hash,page,start_time' });
|
||||
if (error) return NextResponse.json({ error: error.message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(
|
||||
{
|
||||
books: booksResult?.data || [],
|
||||
|
||||
@@ -51,6 +51,65 @@ const migrations: Record<SchemaType, MigrationEntry[]> = {
|
||||
// rows rather than UPDATE — Tantivy 0.25→0.26 has a known WASM-only
|
||||
// UPDATE regression (see fts-tests.ts:306 FIXME). MVP indexing is
|
||||
// write-once per book so this is naturally satisfied.
|
||||
statistics: [
|
||||
{
|
||||
name: '2026061501_statistics_koreader_schema',
|
||||
sql: `
|
||||
CREATE TABLE IF NOT EXISTS book (
|
||||
id integer PRIMARY KEY autoincrement,
|
||||
title text, authors text, notes integer, last_open integer,
|
||||
highlights integer, pages integer, series text, language text,
|
||||
md5 text, total_read_time integer, total_read_pages integer
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS book_title_authors_md5 ON book(title, authors, md5);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS page_stat_data (
|
||||
id_book integer,
|
||||
page integer NOT NULL DEFAULT 0,
|
||||
start_time integer NOT NULL DEFAULT 0,
|
||||
duration integer NOT NULL DEFAULT 0,
|
||||
total_pages integer NOT NULL DEFAULT 0,
|
||||
UNIQUE (id_book, page, start_time)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS numbers (number INTEGER PRIMARY KEY);
|
||||
|
||||
INSERT OR IGNORE INTO numbers(number)
|
||||
SELECT h.n * 100 + t.n * 10 + o.n + 1
|
||||
FROM (SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) o,
|
||||
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) t,
|
||||
(SELECT 0 AS n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) h;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS page_stat AS
|
||||
SELECT id_book, first_page + idx - 1 AS page, start_time, duration / (last_page - first_page + 1) AS duration
|
||||
FROM (
|
||||
SELECT id_book, page, total_pages, pages, start_time, duration,
|
||||
((page - 1) * pages) / total_pages + 1 AS first_page,
|
||||
max(((page - 1) * pages) / total_pages + 1, (page * pages) / total_pages) AS last_page,
|
||||
idx
|
||||
FROM page_stat_data
|
||||
JOIN book ON book.id = id_book
|
||||
JOIN (SELECT number as idx FROM numbers) AS N ON idx <= (last_page - first_page + 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS readest_page_ext (
|
||||
book_hash text NOT NULL, page integer NOT NULL, start_time integer NOT NULL,
|
||||
ext text, PRIMARY KEY (book_hash, page, start_time)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS readest_book_ext (
|
||||
book_hash text PRIMARY KEY, ext text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS readest_stat_sync_state (
|
||||
key text PRIMARY KEY, value integer NOT NULL DEFAULT 0
|
||||
);
|
||||
`,
|
||||
},
|
||||
],
|
||||
reedy: [
|
||||
{
|
||||
name: '2026052601_reedy_init',
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import type { AppService } from '@/types/system';
|
||||
import type { DatabaseService, DatabaseRow } from '@/types/database';
|
||||
import type { PageStatEvent, StatBook } from '@/types/statistics';
|
||||
|
||||
interface BookRow extends DatabaseRow {
|
||||
id: number;
|
||||
title: string;
|
||||
authors: string;
|
||||
md5: string;
|
||||
total_read_time: number;
|
||||
total_read_pages: number;
|
||||
last_open: number;
|
||||
pages: number;
|
||||
}
|
||||
|
||||
type CursorKey = 'push' | 'pull';
|
||||
|
||||
/**
|
||||
* Per-tab singleton open promise. OPFS permits only ONE access handle per file
|
||||
* across the whole origin, so a second `connect()` to statistics.db throws
|
||||
* `NoModificationAllowedError`. Every ReadingStatsTracker instance (and split-
|
||||
* view books) must therefore share a single connection — we memoise the open
|
||||
* and never thrash it.
|
||||
*/
|
||||
let sharedDb: Promise<StatisticsDb> | null = null;
|
||||
let lifecycleBound = false;
|
||||
|
||||
function bindLifecycle(): void {
|
||||
if (lifecycleBound || typeof document === 'undefined') return;
|
||||
lifecycleBound = true;
|
||||
// Fold the WAL into the main db when the tab is backgrounded/closed — the
|
||||
// most reliable point for best-effort async OPFS work before teardown.
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'hidden' && sharedDb) {
|
||||
void sharedDb.then((s) => s.checkpoint()).catch(() => {});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Typed wrapper over the KOReader-compatible `statistics.db`. All identity is
|
||||
* keyed on `book_hash` (= Book.hash); the local autoincrement `id_book` never
|
||||
* leaves this class.
|
||||
*/
|
||||
export class StatisticsDb {
|
||||
private constructor(private readonly db: DatabaseService) {}
|
||||
|
||||
/** Production entry point — opens + migrates statistics.db (per-tab singleton). */
|
||||
static async open(appService: AppService): Promise<StatisticsDb> {
|
||||
bindLifecycle();
|
||||
if (!sharedDb) {
|
||||
sharedDb = (async () => {
|
||||
const db = await appService.openDatabase('statistics', 'statistics.db', 'Data');
|
||||
return new StatisticsDb(db);
|
||||
})();
|
||||
}
|
||||
return sharedDb;
|
||||
}
|
||||
|
||||
/** Test/advanced entry point — wrap an already-migrated DatabaseService. */
|
||||
static from(db: DatabaseService): StatisticsDb {
|
||||
return new StatisticsDb(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold the WAL into the main db file and truncate it. The Turso engine does
|
||||
* NOT implement `PRAGMA wal_autocheckpoint` (so there's no auto threshold to
|
||||
* rely on), but `wal_checkpoint(TRUNCATE)` works — verified folding a 688 KB
|
||||
* WAL back to 0 B. We call this when the tab is hidden so the WAL stays bounded.
|
||||
*/
|
||||
async checkpoint(): Promise<void> {
|
||||
await this.db.execute('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/** Checkpoint, close the underlying connection, and reset the singleton. */
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
await this.checkpoint();
|
||||
} catch {
|
||||
// best-effort — a checkpoint failure must not block close
|
||||
}
|
||||
await this.db.close();
|
||||
sharedDb = null;
|
||||
}
|
||||
|
||||
async upsertBook(book: StatBook): Promise<number> {
|
||||
const existing = await this.db.select<BookRow>(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [
|
||||
book.bookMd5,
|
||||
]);
|
||||
if (existing[0]) {
|
||||
// md5 is the identity; keep the latest title/authors (LWW, matches server stat_books).
|
||||
await this.db.execute(`UPDATE book SET title = ?, authors = ? WHERE id = ?`, [
|
||||
book.title,
|
||||
book.authors,
|
||||
existing[0].id,
|
||||
]);
|
||||
return existing[0].id;
|
||||
}
|
||||
await this.db.execute(
|
||||
`INSERT INTO book (title, authors, md5, notes, last_open, highlights, pages, total_read_time, total_read_pages)
|
||||
VALUES (?, ?, ?, 0, 0, 0, 0, 0, 0)
|
||||
ON CONFLICT(title, authors, md5) DO NOTHING`,
|
||||
[book.title, book.authors, book.bookMd5],
|
||||
);
|
||||
const rows = await this.db.select<BookRow>(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [
|
||||
book.bookMd5,
|
||||
]);
|
||||
return rows[0]!.id;
|
||||
}
|
||||
|
||||
async insertPageEvent(
|
||||
idBook: number,
|
||||
e: { page: number; startTime: number; duration: number; totalPages: number },
|
||||
): Promise<void> {
|
||||
await this.db.execute(
|
||||
`INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id_book, page, start_time)
|
||||
DO UPDATE SET duration = max(duration, excluded.duration), total_pages = excluded.total_pages`,
|
||||
[idBook, e.page, e.startTime, e.duration, e.totalPages],
|
||||
);
|
||||
}
|
||||
|
||||
async recomputeBookTotals(idBook: number): Promise<void> {
|
||||
await this.db.execute(
|
||||
`UPDATE book SET
|
||||
total_read_time = COALESCE((SELECT SUM(duration) FROM page_stat_data WHERE id_book = ?), 0),
|
||||
total_read_pages = COALESCE((SELECT COUNT(DISTINCT page) FROM page_stat_data WHERE id_book = ?), 0),
|
||||
last_open = COALESCE((SELECT MAX(start_time + duration) FROM page_stat_data WHERE id_book = ?), last_open),
|
||||
pages = COALESCE((SELECT total_pages FROM page_stat_data WHERE id_book = ? ORDER BY start_time DESC LIMIT 1), pages)
|
||||
WHERE id = ?`,
|
||||
[idBook, idBook, idBook, idBook, idBook],
|
||||
);
|
||||
}
|
||||
|
||||
async getBookByMd5(md5: string): Promise<BookRow | null> {
|
||||
const rows = await this.db.select<BookRow>(`SELECT * FROM book WHERE md5 = ? LIMIT 1`, [md5]);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a book row exists for an event whose metadata record isn't in the
|
||||
* current batch (paged pull can deliver an event before its `stat_books`
|
||||
* record). Unlike `upsertBook`, this NEVER overwrites an existing real title
|
||||
* with the hash placeholder — the real record, arriving in any page, wins.
|
||||
*/
|
||||
private async ensureBookId(bookMd5: string): Promise<number> {
|
||||
const existing = await this.db.select<BookRow>(`SELECT id FROM book WHERE md5 = ? LIMIT 1`, [
|
||||
bookMd5,
|
||||
]);
|
||||
if (existing[0]) return existing[0].id;
|
||||
return this.upsertBook({ bookMd5, title: bookMd5, authors: '' });
|
||||
}
|
||||
|
||||
/** Events with start_time > cursor, joined to their md5, for pushing. */
|
||||
async getEventsForPush(
|
||||
sinceStartTime: number,
|
||||
): Promise<{ events: PageStatEvent[]; books: StatBook[] }> {
|
||||
const rows = await this.db.select<DatabaseRow>(
|
||||
`SELECT b.md5 AS bookMd5, b.title AS title, b.authors AS authors,
|
||||
p.page AS page, p.start_time AS startTime, p.duration AS duration, p.total_pages AS totalPages
|
||||
FROM page_stat_data p JOIN book b ON b.id = p.id_book
|
||||
WHERE p.start_time > ?
|
||||
ORDER BY p.start_time ASC`,
|
||||
[sinceStartTime],
|
||||
);
|
||||
const events: PageStatEvent[] = rows.map((r) => ({
|
||||
bookMd5: String(r['bookMd5']),
|
||||
page: Number(r['page']),
|
||||
startTime: Number(r['startTime']),
|
||||
duration: Number(r['duration']),
|
||||
totalPages: Number(r['totalPages']),
|
||||
}));
|
||||
const bookMap = new Map<string, StatBook>();
|
||||
for (const r of rows) {
|
||||
const md5 = String(r['bookMd5']);
|
||||
if (!bookMap.has(md5)) {
|
||||
bookMap.set(md5, {
|
||||
bookMd5: md5,
|
||||
title: String(r['title']),
|
||||
authors: String(r['authors']),
|
||||
});
|
||||
}
|
||||
}
|
||||
return { events, books: [...bookMap.values()] };
|
||||
}
|
||||
|
||||
async applyRemoteEvents(books: StatBook[], events: PageStatEvent[]): Promise<void> {
|
||||
if (books.length === 0 && events.length === 0) return;
|
||||
// One transaction for the whole pulled batch: a single commit instead of
|
||||
// O(rows) fsyncs, and the apply is atomic (a failed pull leaves no partial
|
||||
// state). Critical when a fresh device backfills tens of thousands of rows.
|
||||
await this.db.execute('BEGIN');
|
||||
try {
|
||||
const idByMd5 = new Map<string, number>();
|
||||
for (const b of books) idByMd5.set(b.bookMd5, await this.upsertBook(b));
|
||||
// Books referenced only by events (no metadata record) get a placeholder row.
|
||||
const touched = new Set<number>();
|
||||
for (const e of events) {
|
||||
let id = idByMd5.get(e.bookMd5);
|
||||
if (id === undefined) {
|
||||
id = await this.ensureBookId(e.bookMd5);
|
||||
idByMd5.set(e.bookMd5, id);
|
||||
}
|
||||
await this.insertPageEvent(id, e);
|
||||
touched.add(id);
|
||||
}
|
||||
for (const id of touched) await this.recomputeBookTotals(id);
|
||||
await this.db.execute('COMMIT');
|
||||
} catch (err) {
|
||||
await this.db.execute('ROLLBACK');
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async getCursor(key: CursorKey): Promise<number> {
|
||||
const rows = await this.db.select<{ value: number }>(
|
||||
`SELECT value FROM readest_stat_sync_state WHERE key = ?`,
|
||||
[key],
|
||||
);
|
||||
return rows[0]?.value ?? 0;
|
||||
}
|
||||
|
||||
async setCursor(key: CursorKey, value: number): Promise<void> {
|
||||
await this.db.execute(
|
||||
`INSERT INTO readest_stat_sync_state (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
||||
[key, value],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import type { StatisticsDb } from './statisticsDb';
|
||||
import type { SyncClient, StatPageRecord, StatBookRecord } from '@/libs/sync';
|
||||
import type { PageStatEvent, StatBook } from '@/types/statistics';
|
||||
|
||||
type PushClient = Pick<SyncClient, 'pushChanges'>;
|
||||
type PullClient = Pick<SyncClient, 'pullChanges'>;
|
||||
|
||||
const toWirePage = (e: PageStatEvent): StatPageRecord => ({
|
||||
book_hash: e.bookMd5,
|
||||
page: e.page,
|
||||
start_time: e.startTime,
|
||||
duration: e.duration,
|
||||
total_pages: e.totalPages,
|
||||
});
|
||||
|
||||
const toWireBook = (b: StatBook): StatBookRecord => ({
|
||||
book_hash: b.bookMd5,
|
||||
title: b.title,
|
||||
authors: b.authors,
|
||||
});
|
||||
|
||||
/** Events per push request — bounds request size for a large offline backlog. */
|
||||
const PUSH_CHUNK = 500;
|
||||
/** Page events per pull request — bounds the receiving device's memory. */
|
||||
const PULL_PAGE = 1000;
|
||||
|
||||
/**
|
||||
* Push local events newer than the push cursor, in bounded chunks. The cursor
|
||||
* advances per successful chunk, so an interrupted push (e.g. a 1000-event
|
||||
* backlog over flaky network) resumes from the last chunk rather than restarting.
|
||||
*/
|
||||
export async function pushStats(stats: StatisticsDb, client: PushClient): Promise<void> {
|
||||
const cursor = await stats.getCursor('push');
|
||||
const { events, books } = await stats.getEventsForPush(cursor);
|
||||
if (events.length === 0) return;
|
||||
const bookByHash = new Map(books.map((b) => [b.bookMd5, b]));
|
||||
let i = 0;
|
||||
while (i < events.length) {
|
||||
let end = Math.min(i + PUSH_CHUNK, events.length);
|
||||
// Never split a start_time across chunks — advancing the push cursor past it
|
||||
// would drop the remaining same-second events (e.g. split-view) on resume.
|
||||
const lastStart = events[end - 1]!.startTime;
|
||||
while (end < events.length && events[end]!.startTime === lastStart) end++;
|
||||
const chunk = events.slice(i, end);
|
||||
const seen = new Set<string>();
|
||||
const chunkBooks: StatBookRecord[] = [];
|
||||
for (const e of chunk) {
|
||||
if (seen.has(e.bookMd5)) continue;
|
||||
seen.add(e.bookMd5);
|
||||
const b = bookByHash.get(e.bookMd5);
|
||||
if (b) chunkBooks.push(toWireBook(b));
|
||||
}
|
||||
await client.pushChanges({ statBooks: chunkBooks, statPages: chunk.map(toWirePage) });
|
||||
await stats.setCursor('push', chunk[chunk.length - 1]!.startTime);
|
||||
i = end;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull events since the pull cursor in bounded pages, applying each before
|
||||
* fetching the next so memory stays flat and a fresh-device backfill is
|
||||
* resumable (the cursor persists between pages). The server completes the
|
||||
* trailing millisecond of each page, so a strict `> cursor` next pull never
|
||||
* skips rows that share an `updated_at`.
|
||||
*/
|
||||
export async function pullStats(stats: StatisticsDb, client: PullClient): Promise<void> {
|
||||
for (;;) {
|
||||
const since = await stats.getCursor('pull');
|
||||
const res = await client.pullChanges(since, 'stats', undefined, undefined, PULL_PAGE);
|
||||
const wireBooks = (res.statBooks ?? []) as StatBookRecord[];
|
||||
const wirePages = (res.statPages ?? []) as StatPageRecord[];
|
||||
if (wireBooks.length === 0 && wirePages.length === 0) break;
|
||||
const books: StatBook[] = wireBooks.map((b) => ({
|
||||
bookMd5: b.book_hash,
|
||||
title: b.title,
|
||||
authors: b.authors,
|
||||
}));
|
||||
const events: PageStatEvent[] = wirePages.map((p) => ({
|
||||
bookMd5: p.book_hash,
|
||||
page: p.page,
|
||||
startTime: p.start_time,
|
||||
duration: p.duration,
|
||||
totalPages: p.total_pages,
|
||||
}));
|
||||
await stats.applyRemoteEvents(books, events);
|
||||
// Advance the cursor to the newest page-event updated_at_ms. Stop when a
|
||||
// page yields no further page-event progress (covers the empty-pages and
|
||||
// the all-books-no-pages cases, and guards against a stalled cursor).
|
||||
const newest = wirePages.reduce((m, p) => Math.max(m, p.updated_at_ms ?? 0), since);
|
||||
if (newest <= since) break;
|
||||
await stats.setCursor('pull', newest);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { StatsTrackingConfig } from '@/types/statistics';
|
||||
|
||||
interface PendingEvent {
|
||||
page: number;
|
||||
startTime: number; // Unix seconds
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
export interface FlushedEvent {
|
||||
page: number;
|
||||
startTime: number;
|
||||
duration: number;
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure page-dwell tracker. The React layer feeds it `nowSeconds` and the
|
||||
* current `(page, totalPages)`; it returns zero-or-one immutable events to
|
||||
* persist. Events end on page-change / idle / hide / close. Each returned
|
||||
* event is final — its start_time and duration never change afterwards, which
|
||||
* lets sync use a simple start_time high-water cursor.
|
||||
*/
|
||||
export class TrackerCore {
|
||||
private pending: PendingEvent | null = null;
|
||||
|
||||
constructor(private readonly cfg: StatsTrackingConfig) {}
|
||||
|
||||
/** Notify the current page at `now`. Returns events flushed by leaving the prior page. */
|
||||
onPage(page: number, totalPages: number, now: number): FlushedEvent[] {
|
||||
if (this.pending && this.pending.page === page) {
|
||||
// Same page (e.g. resume after idle, or a no-op relocate): keep dwelling.
|
||||
return [];
|
||||
}
|
||||
const flushed = this.flush(now);
|
||||
this.pending = { page, startTime: now, totalPages };
|
||||
return flushed;
|
||||
}
|
||||
|
||||
onIdle(now: number): FlushedEvent[] {
|
||||
return this.flush(now); // flush + pause (pending cleared)
|
||||
}
|
||||
|
||||
onHide(now: number): FlushedEvent[] {
|
||||
return this.flush(now);
|
||||
}
|
||||
|
||||
onClose(now: number): FlushedEvent[] {
|
||||
return this.flush(now);
|
||||
}
|
||||
|
||||
private flush(now: number): FlushedEvent[] {
|
||||
const p = this.pending;
|
||||
this.pending = null;
|
||||
if (!p) return [];
|
||||
const raw = now - p.startTime;
|
||||
const duration = Math.min(Math.max(raw, 0), this.cfg.maxEventSeconds);
|
||||
if (duration < this.cfg.minEventSeconds) return [];
|
||||
return [{ page: p.page, startTime: p.startTime, duration, totalPages: p.totalPages }];
|
||||
}
|
||||
}
|
||||
@@ -233,7 +233,8 @@ export type SyncCategory =
|
||||
| 'texture'
|
||||
| 'opds_catalog'
|
||||
| 'settings'
|
||||
| 'credentials';
|
||||
| 'credentials'
|
||||
| 'stats';
|
||||
|
||||
export const SYNC_CATEGORIES: readonly SyncCategory[] = [
|
||||
'book',
|
||||
@@ -244,6 +245,7 @@ export const SYNC_CATEGORIES: readonly SyncCategory[] = [
|
||||
'texture',
|
||||
'opds_catalog',
|
||||
'settings',
|
||||
'stats',
|
||||
'credentials',
|
||||
] as const;
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
// KOReader-compatible reading-statistics types.
|
||||
//
|
||||
// The canonical unit is the per-page read event (KOReader's page_stat_data
|
||||
// row). Aggregates (sessions, streaks, totals) are DERIVED via SQL, never
|
||||
// stored as separate records. Times are Unix seconds; pages are 1-based.
|
||||
|
||||
/** One immutable page-read event — a KOReader `page_stat_data` row. */
|
||||
export interface PageStatEvent {
|
||||
bookMd5: string; // = Book.hash = KOReader book.md5
|
||||
page: number; // 1-based
|
||||
startTime: number; // Unix seconds
|
||||
duration: number; // seconds
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
/** KOReader book identity — the only book metadata that syncs. */
|
||||
export interface StatBook {
|
||||
bookMd5: string;
|
||||
title: string;
|
||||
authors: string; // KOReader stores authors as a single text field
|
||||
}
|
||||
|
||||
/** Tunables for the tracker's flush/idle behavior. KOReader-aligned defaults. */
|
||||
export interface StatsTrackingConfig {
|
||||
/** Seconds of inactivity before the current page event is flushed + paused. */
|
||||
idleTimeoutSeconds: number;
|
||||
/** Hard per-event duration cap (safety net if a visibility event is missed). */
|
||||
maxEventSeconds: number;
|
||||
/** Events shorter than this are dropped (ignore sub-second page flips). */
|
||||
minEventSeconds: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_STATS_TRACKING_CONFIG: StatsTrackingConfig = {
|
||||
idleTimeoutSeconds: 120,
|
||||
maxEventSeconds: 120,
|
||||
minEventSeconds: 3,
|
||||
};
|
||||
Reference in New Issue
Block a user