feat(statistics): KOReader-compatible reading stats with cross-device sync (#4606)

Supersedes #3156. Adds a reading-statistics system whose canonical data model
is KOReader's own (book + page_stat_data), so stats round-trip losslessly
between Readest and KOReader.

- Storage: a cross-platform Turso statistics.db in KOReader's exact schema
  (web/Workers, desktop, iOS, Android) — replacing #3156's Node-only
  better-sqlite3 + statistics.json.
- Tracking: per-page reading events (time-on-page, idle-capped) flushed on
  page-change/idle/hide/close — the KOReader model — not session aggregates.
- Sync: legacy /api/sync extended with a stats type backed by self-contained
  Supabase tables (stat_books, stat_pages); union/longer-duration-wins merge
  keyed on book_hash. apps/readest.koplugin syncs through the same endpoint.
- Scale & robustness: per-tab singleton connection (avoids OPFS lock
  conflicts) + explicit WAL checkpoint; transactional bulk apply; chunked
  resumable push; client-driven paged pull with trailing-ms completion;
  paginated/scoped server merge.

Verified: 5668 unit tests, 155 koplugin busted tests, biome+tsgo + luacheck
all green; web OPFS DB verified live.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
2026-06-16 16:42:39 +08:00
committed by GitHub
parent 359e406e51
commit 35b02c4efc
57 changed files with 3437 additions and 11 deletions
@@ -25,6 +25,7 @@ import FootnotePopup from './FootnotePopup';
import HintInfo from './HintInfo';
import ReadingRuler from './ReadingRuler';
import DoubleBorder from './DoubleBorder';
import ReadingStatsTracker from './ReadingStatsTracker';
interface BooksGridProps {
bookKeys: string[];
@@ -278,6 +279,7 @@ const BookCellInner: React.FC<BookCellProps> = ({
isHoveredAnim={false}
gridInsets={gridInsets}
/>
<ReadingStatsTracker bookKey={bookKey} />
</div>
);
};
@@ -0,0 +1,121 @@
'use client';
import { useEffect, useRef } from 'react';
import { useBookProgress } from '@/store/readerProgressStore';
import { useBookDataStore } from '@/store/bookDataStore';
import { useEnv } from '@/context/EnvContext';
import { useAuth } from '@/context/AuthContext';
import { StatisticsDb } from '@/services/statistics/statisticsDb';
import { TrackerCore, type FlushedEvent } from '@/services/statistics/trackerCore';
import { DEFAULT_STATS_TRACKING_CONFIG } from '@/types/statistics';
import { SyncClient } from '@/libs/sync';
import { pushStats, pullStats } from '@/services/statistics/statsSync';
import { isSyncCategoryEnabled } from '@/services/sync/syncCategories';
const nowSec = () => Math.floor(Date.now() / 1000);
export default function ReadingStatsTracker({ bookKey }: { bookKey: string }) {
const { appService } = useEnv();
// Progress lives in readerProgressStore, not readerStore.viewStates.
const progress = useBookProgress(bookKey);
// booksData is keyed by book id = bookKey.split('-')[0].
const getBookData = useBookDataStore((s) => s.getBookData);
const { user } = useAuth();
const coreRef = useRef(new TrackerCore(DEFAULT_STATS_TRACKING_CONFIG));
const dbRef = useRef<StatisticsDb | null>(null);
const idleRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const pushTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const bookData = getBookData(bookKey);
const book = bookData?.book;
// Book.hash is the partialMD5 used as KOReader's md5.
const bookMd5 = book?.hash;
const title = book?.title ?? '';
// Book.author is the single-string author field; upsertBook takes authors: string.
const authors = book?.author ?? '';
const syncEnabled = () => !!user && isSyncCategoryEnabled('stats');
const schedulePush = () => {
if (!syncEnabled()) return;
if (pushTimerRef.current) clearTimeout(pushTimerRef.current);
pushTimerRef.current = setTimeout(() => {
const db = dbRef.current;
if (db) void pushStats(db, new SyncClient());
}, 10_000);
};
useEffect(() => {
if (!appService) return;
let cancelled = false;
StatisticsDb.open(appService).then((db) => {
if (cancelled) return;
dbRef.current = db;
if (syncEnabled()) void pullStats(db, new SyncClient());
});
return () => {
cancelled = true;
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [appService]);
// Persist flushed events into the statistics DB.
const persist = (events: FlushedEvent[]): Promise<void> => {
const db = dbRef.current;
if (!db || !bookMd5 || events.length === 0) return Promise.resolve();
return (async () => {
const idBook = await db.upsertBook({ bookMd5, title, authors });
for (const e of events) await db.insertPageEvent(idBook, e);
await db.recomputeBookTotals(idBook);
schedulePush();
})();
};
const armIdle = () => {
if (idleRef.current) clearTimeout(idleRef.current);
idleRef.current = setTimeout(
() => void persist(coreRef.current.onIdle(nowSec())),
DEFAULT_STATS_TRACKING_CONFIG.idleTimeoutSeconds * 1000,
);
};
// Page changes drive the tracker.
useEffect(() => {
const info = progress?.pageinfo;
if (!info) return;
const page = (info.current ?? 0) + 1;
const total = info.total || 1;
void persist(coreRef.current.onPage(page, total, nowSec()));
armIdle();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [progress?.pageinfo]);
// Tab/window visibility.
useEffect(() => {
const onVis = () => {
if (document.visibilityState === 'hidden') {
if (idleRef.current) clearTimeout(idleRef.current);
void persist(coreRef.current.onHide(nowSec()));
}
};
document.addEventListener('visibilitychange', onVis);
return () => document.removeEventListener('visibilitychange', onVis);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [bookMd5]);
// Book close (unmount).
useEffect(() => {
return () => {
if (idleRef.current) clearTimeout(idleRef.current);
if (pushTimerRef.current) clearTimeout(pushTimerRef.current);
const closePersist = persist(coreRef.current.onClose(nowSec()));
void closePersist.then(() => {
if (syncEnabled() && dbRef.current) return pushStats(dbRef.current, new SyncClient());
return undefined;
});
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [bookMd5]);
return null;
}