feat(koplugin): add support for annotation sync (#3579)

Add bidirectional annotation/highlight sync between KOReader and Readest:

- Add xpointer0/xpointer1 fields to BookNote and DBBookNote types for
  KOReader XPointer positions alongside Readest's CFI format
- Extend transform layer to pass through xpointer fields to/from DB
- Convert CFI→XPointer on push and XPointer→CFI on pull in useNotesSync,
  discarding notes that fail conversion
- Support KOReader's text()[K].N indexed text node format in xcfi.ts for
  paragraphs with inline elements (e.g. <a> page anchors)
- Generate KOReader-compatible XPointers: text().N for single text nodes,
  text()[K].N only when multiple direct text nodes exist
- Skip cfi-inert elements (injected by Readest at runtime) in XPointer
  path building and resolution
- Map highlight colors between KOReader and Readest color systems
- Implement KOReader plugin annotation push/pull with deterministic IDs,
  auto-sync on document open/close, and UIManager refresh on pull
- Refactor koplugin into focused modules: syncauth, syncconfig,
  syncannotations, selfupdate

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
2026-03-21 23:43:51 +08:00
committed by GitHub
parent 2f430bf2ff
commit 76b239f382
15 changed files with 1507 additions and 686 deletions
@@ -0,0 +1,114 @@
import { describe, it, expect } from 'vitest';
import { transformBookNoteToDB, transformBookNoteFromDB } from '@/utils/transform';
import { BookNote } from '@/types/book';
import { DBBookNote } from '@/types/records';
describe('transformBookNoteToDB with xpointer fields', () => {
it('passes through xpointer0 and xpointer1', () => {
const note: BookNote = {
bookHash: 'abc123',
metaHash: 'meta456',
id: 'note1',
type: 'annotation',
cfi: 'epubcfi(/6/4!/4/2/1:0)',
xpointer0: '/body/DocFragment[2]/body/div[1]/p[1]/text().0',
xpointer1: '/body/DocFragment[2]/body/div[1]/p[1]/text().50',
text: 'highlighted text',
note: 'my note',
style: 'highlight',
color: 'yellow',
createdAt: 1700000000000,
updatedAt: 1700000001000,
};
const dbNote = transformBookNoteToDB(note, 'user1');
expect(dbNote.xpointer0).toBe('/body/DocFragment[2]/body/div[1]/p[1]/text().0');
expect(dbNote.xpointer1).toBe('/body/DocFragment[2]/body/div[1]/p[1]/text().50');
expect(dbNote.cfi).toBe('epubcfi(/6/4!/4/2/1:0)');
});
it('handles missing xpointer fields (Readest-only note)', () => {
const note: BookNote = {
bookHash: 'abc123',
id: 'note2',
type: 'annotation',
cfi: 'epubcfi(/6/4!/4/2/1:0)',
text: 'text',
note: '',
createdAt: 1700000000000,
updatedAt: 1700000001000,
};
const dbNote = transformBookNoteToDB(note, 'user1');
expect(dbNote.xpointer0).toBeUndefined();
expect(dbNote.xpointer1).toBeUndefined();
expect(dbNote.cfi).toBe('epubcfi(/6/4!/4/2/1:0)');
});
});
describe('transformBookNoteFromDB with xpointer fields', () => {
it('reads xpointer0 and xpointer1 from DB', () => {
const dbNote: DBBookNote = {
user_id: 'user1',
book_hash: 'abc123',
meta_hash: 'meta456',
id: 'note1',
type: 'annotation',
cfi: 'epubcfi(/6/4!/4/2/1:0)',
xpointer0: '/body/DocFragment[2]/body/div[1]/p[1]/text().0',
xpointer1: '/body/DocFragment[2]/body/div[1]/p[1]/text().50',
text: 'highlighted text',
note: 'my note',
style: 'highlight',
color: 'yellow',
created_at: '2023-11-14T22:13:20.000Z',
updated_at: '2023-11-14T22:13:21.000Z',
};
const note = transformBookNoteFromDB(dbNote);
expect(note.xpointer0).toBe('/body/DocFragment[2]/body/div[1]/p[1]/text().0');
expect(note.xpointer1).toBe('/body/DocFragment[2]/body/div[1]/p[1]/text().50');
expect(note.cfi).toBe('epubcfi(/6/4!/4/2/1:0)');
});
it('handles missing xpointer fields from DB', () => {
const dbNote: DBBookNote = {
user_id: 'user1',
book_hash: 'abc123',
id: 'note2',
type: 'annotation',
cfi: 'epubcfi(/6/4!/4/2/1:0)',
note: '',
created_at: '2023-11-14T22:13:20.000Z',
updated_at: '2023-11-14T22:13:21.000Z',
};
const note = transformBookNoteFromDB(dbNote);
expect(note.xpointer0).toBeUndefined();
expect(note.xpointer1).toBeUndefined();
});
it('defaults cfi to empty string when missing from DB (KOReader note)', () => {
const dbNote: DBBookNote = {
user_id: 'user1',
book_hash: 'abc123',
id: 'note3',
type: 'annotation',
xpointer0: '/body/DocFragment[1]/body/p[1]/text().0',
xpointer1: '/body/DocFragment[1]/body/p[1]/text().20',
note: '',
created_at: '2023-11-14T22:13:20.000Z',
updated_at: '2023-11-14T22:13:21.000Z',
};
const note = transformBookNoteFromDB(dbNote);
expect(note.cfi).toBe('');
expect(note.xpointer0).toBe('/body/DocFragment[1]/body/p[1]/text().0');
expect(note.xpointer1).toBe('/body/DocFragment[1]/body/p[1]/text().20');
});
});
@@ -241,8 +241,12 @@ describe('CFIToXPointerConverter', () => {
const xpointer = converter.cfiToXPointer(cfi);
expect(cfi).toMatch(/^epubcfi\([^,]+,[^,]+,[^,]+\)$/);
expect(xpointer.pos0).toBe(pos0);
expect(xpointer.pos1).toBe(pos1);
// cfiToXPointer now produces text()[K].N format
expect(xpointer.pos0).toBe('/body/DocFragment[2]/body/div/p[1]/text().6');
expect(xpointer.pos1).toBe('/body/DocFragment[2]/body/div/p[2]/text().16');
// Round-trip: the new format should convert back to the same CFI
const backCfi = converter.xPointerToCFI(xpointer.pos0!, xpointer.pos1!);
expect(backCfi).toBe(cfi);
});
});
@@ -382,4 +386,148 @@ describe('CFIToXPointerConverter', () => {
expect(result.xpointer).toBe('/body/DocFragment[3]/body/div/section/article/p[1]');
});
});
describe('indexed text node XPointers (text()[N].offset)', () => {
let inlineDoc: Document;
beforeEach(() => {
// Simulates: <p>...text...<a id="page96"></a> Megan likes the sea, too...</p>
inlineDoc = new DOMParser().parseFromString(
`<html><body>
<p>She spent a year of her Ph.D. at my old college at Cambridge. <a id="page96" tabindex="-1"></a>A woman, at Caius! Megan likes the sea, too. She's finishing her radioastronomy research.</p>
</body></html>`,
'text/html',
);
});
it('should convert text()[N].offset range XPointer to valid CFI', () => {
const converter = new XCFI(inlineDoc, 10);
// text()[2] = the 2nd direct text node child of <p>, i.e. the text after the <a>
const pos0 = '/body/DocFragment[11]/body/p/text()[2].44';
const pos1 = '/body/DocFragment[11]/body/p/text()[2].69';
const cfi = converter.xPointerToCFI(pos0, pos1);
// Should produce a valid range CFI pointing into the 3rd child node (text after <a>)
expect(cfi).toMatch(/^epubcfi\(/);
expect(cfi).toMatch(/,.*,/); // Range CFI has two commas
// /3 = 3rd child of <p> (1:text, 2:<a>, 3:text), offsets 44 and 69
expect(cfi).toContain('/3:44');
expect(cfi).toContain('/3:69');
});
it('should convert text()[1].offset XPointer to valid CFI', () => {
const converter = new XCFI(inlineDoc, 10);
// text()[1] = the 1st direct text node child of <p>, i.e. text before the <a>
const xp = '/body/DocFragment[11]/body/p/text()[1].5';
const cfi = converter.xPointerToCFI(xp);
expect(cfi).toMatch(/^epubcfi\(/);
// /1 = 1st child of <p> (text node), offset 5
expect(cfi).toContain('/1:5');
});
it('should handle text()[N].offset with multiple inline elements', () => {
const multiInlineDoc = new DOMParser().parseFromString(
`<html><body>
<p>Start text <em>emphasis</em> middle text <a id="link1">link</a> end text here.</p>
</body></html>`,
'text/html',
);
const converter = new XCFI(multiInlineDoc, 5);
// Direct children of <p>: text, <em>, text, <a>, text
// text()[3] = " end text here." (3rd direct text node of <p>)
const xp = '/body/DocFragment[6]/body/p/text()[3].4';
const cfi = converter.xPointerToCFI(xp);
expect(cfi).toMatch(/^epubcfi\(/);
// /5 = 5th child of <p> (1:text, 2:<em>, 3:text, 4:<a>, 5:text), offset 4
expect(cfi).toContain('/5:4');
});
it('should produce correct CFI for text()[1] with no inline siblings', () => {
const simpleDoc = new DOMParser().parseFromString(
`<html><body><p>Hello world</p></body></html>`,
'text/html',
);
const converter = new XCFI(simpleDoc, 0);
const xp = '/body/DocFragment[1]/body/p/text()[1].5';
const cfi = converter.xPointerToCFI(xp);
expect(cfi).toMatch(/^epubcfi\(/);
expect(cfi).toContain('/1:5');
});
});
describe('cfi-inert elements should be invisible to XPointer', () => {
it('should skip cfi-inert div when resolving KOReader XPointer', () => {
const doc = new DOMParser().parseFromString(
`<html><body>
<div cfi-inert="">skip link</div>
<div class="body">
<div class="chapter">
<div class="text">
<p>Alice thought this a very curious thing.</p>
</div>
</div>
</div>
</body></html>`,
'text/html',
);
const converter = new XCFI(doc, 10);
// KOReader XPointer: div (no index) means the only "real" div
const xp = '/body/DocFragment[11]/body/div/div/div/p/text().10';
const cfi = converter.xPointerToCFI(xp);
expect(cfi).toMatch(/^epubcfi\(/);
});
it('should skip cfi-inert div when building XPointer path from element', () => {
const doc = new DOMParser().parseFromString(
`<html><body>
<div cfi-inert="">skip link</div>
<div class="body">
<div class="chapter">
<div class="text">
<p>Alice thought this a very curious thing.</p>
</div>
</div>
</div>
</body></html>`,
'text/html',
);
const converter = new XCFI(doc, 10);
// Navigate to the <p> via DOM, then build XPointer from it
const p = doc.querySelector('p')!;
// Use xPointerToCFI and verify the KOReader XPointer resolves correctly
const koXp = '/body/DocFragment[11]/body/div/div/div/p/text().10';
const cfi = converter.xPointerToCFI(koXp);
// Verify the same CFI is produced when starting from a Readest-generated range
const range = doc.createRange();
const textNode = p.firstChild!;
range.setStart(textNode, 10);
range.setEnd(textNode, 10);
// The xPointerToCFI should resolve through the content div, not the cfi-inert div
expect(cfi).toMatch(/^epubcfi\(/);
});
it('should handle cfi-inert with multiple real siblings', () => {
const doc = new DOMParser().parseFromString(
`<html><body>
<div cfi-inert="">skip</div>
<div class="first">First</div>
<div class="second"><p>Content</p></div>
</body></html>`,
'text/html',
);
const converter = new XCFI(doc, 0);
// Two real divs: div[1]=first, div[2]=second — cfi-inert is invisible
const xp = '/body/DocFragment[1]/body/div[2]/p';
const cfi = converter.xPointerToCFI(xp);
expect(cfi).toMatch(/^epubcfi\(/);
});
});
});
@@ -678,6 +678,7 @@ const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {
const color = settings.globalReadSettings.highlightStyles[style];
setSelectedStyle(style);
setSelectedColor(color);
console.log('Adding annotation:', cfi, 'style:', style, 'color:', color);
const annotation: BookNote = {
id: uniqueId(),
type: 'annotation',
@@ -1,18 +1,104 @@
import { useCallback, useEffect } from 'react';
import { useAuth } from '@/context/AuthContext';
import { useSync } from '@/hooks/useSync';
import { BookNote } from '@/types/book';
import { BookNote, FIXED_LAYOUT_FORMATS } from '@/types/book';
import { useBookDataStore } from '@/store/bookDataStore';
import { useReaderStore } from '@/store/readerStore';
import { SYNC_NOTES_INTERVAL_SEC } from '@/services/constants';
import { throttle } from '@/utils/throttle';
import { getXPointerFromCFI, getCFIFromXPointer, XCFI } from '@/utils/xcfi';
import { getIndexFromCfi } from '@/utils/cfi';
export const useNotesSync = (bookKey: string) => {
const { user } = useAuth();
const { syncedNotes, syncNotes, lastSyncedAtNotes } = useSync(bookKey);
const { getConfig, setConfig, getBookData } = useBookDataStore();
const { getView } = useReaderStore();
const config = getConfig(bookKey);
const populateXPointersForPush = async (notes: BookNote[]): Promise<BookNote[]> => {
const bookData = getBookData(bookKey);
const book = bookData?.book;
if (!book || FIXED_LAYOUT_FORMATS.has(book.format)) return notes;
const view = getView(bookKey);
if (!view) return notes;
const enriched: BookNote[] = [];
for (const note of notes) {
if (note.cfi && !note.xpointer0) {
try {
const contents = view.renderer.getContents();
const primaryIndex = view.renderer.primaryIndex;
const content = contents.find((x) => x.index === primaryIndex) ?? contents[0];
if (content) {
const xpResult = await getXPointerFromCFI(
note.cfi,
content.doc,
content.index || 0,
bookData.bookDoc ?? undefined,
);
enriched.push({
...note,
xpointer0: xpResult.pos0 || xpResult.xpointer,
xpointer1: xpResult.pos1,
updatedAt: Date.now(),
});
continue;
}
} catch {
// Conversion failed — push without xpointers
}
}
enriched.push(note);
}
return enriched;
};
const convertXPointersOnPull = async (notes: BookNote[]): Promise<BookNote[]> => {
const bookData = getBookData(bookKey);
const book = bookData?.book;
if (!book || FIXED_LAYOUT_FORMATS.has(book.format)) return notes.filter((n) => n.cfi);
const view = getView(bookKey);
const converted: BookNote[] = [];
for (const note of notes) {
if (note.xpointer0 && !note.cfi) {
try {
let cfi: string | undefined;
if (note.xpointer1) {
const spineIndex = XCFI.extractSpineIndex(note.xpointer0);
const doc = await bookData.bookDoc?.sections?.[spineIndex]?.createDocument();
if (doc) {
const converter = new XCFI(doc, spineIndex);
cfi = converter.xPointerToCFI(note.xpointer0, note.xpointer1);
}
} else {
const contents = view?.renderer.getContents() ?? [];
const primaryIndex = view?.renderer.primaryIndex;
const content = contents.find((x) => x.index === primaryIndex) ?? contents[0];
cfi = await getCFIFromXPointer(
note.xpointer0,
content?.doc,
content?.index,
bookData.bookDoc ?? undefined,
);
}
if (cfi) {
converted.push({ ...note, cfi, updatedAt: Date.now() });
}
} catch {
// Conversion failed — discard note
}
} else if (note.cfi) {
converted.push(note);
}
// Discard notes with neither cfi nor xpointer
}
return converted;
};
const getNewNotes = useCallback(() => {
const config = getConfig(bookKey);
const book = getBookData(bookKey)?.book;
@@ -20,7 +106,10 @@ export const useNotesSync = (bookKey: string) => {
const bookNotes = config.booknotes ?? [];
const newNotes = bookNotes.filter(
(note) => lastSyncedAtNotes < note.updatedAt || lastSyncedAtNotes < (note.deletedAt ?? 0),
(note) =>
!note.xpointer0 ||
lastSyncedAtNotes < note.updatedAt ||
lastSyncedAtNotes < (note.deletedAt ?? 0),
);
newNotes.forEach((note) => {
note.bookHash = book.hash;
@@ -38,7 +127,13 @@ export const useNotesSync = (bookKey: string) => {
() => {
const book = getBookData(bookKey)?.book;
const newNotes = getNewNotes();
syncNotes(newNotes.notes, book?.hash, book?.metaHash, 'both');
if (newNotes.notes?.length) {
populateXPointersForPush(newNotes.notes).then((enriched) => {
syncNotes(enriched, book?.hash, book?.metaHash, 'both');
});
} else {
syncNotes(newNotes.notes, book?.hash, book?.metaHash, 'both');
}
},
SYNC_NOTES_INTERVAL_SEC * 1000,
{ emitLast: false },
@@ -69,19 +164,32 @@ export const useNotesSync = (bookKey: string) => {
}
return note;
};
if (syncedNotes?.length && config) {
const processSyncedNotes = async () => {
if (!syncedNotes?.length || !config) return;
const view = getView(bookKey);
const book = getBookData(bookKey)?.book;
const newNotes = syncedNotes.filter(
(note) => note.bookHash === book?.hash || note.metaHash === book?.metaHash,
);
if (!newNotes.length) return;
// Convert xpointer-only notes (from KOReader) to CFI
const convertedNotes = await convertXPointersOnPull(newNotes);
convertedNotes.forEach((note) => {
if (note.cfi) {
const index = getIndexFromCfi(note.cfi);
if (index === view?.renderer.primaryIndex) {
view.addAnnotation(note);
}
}
});
const oldNotes = config.booknotes ?? [];
const mergedNotes = [
...oldNotes.filter((oldNote) => !newNotes.some((newNote) => newNote.id === oldNote.id)),
...newNotes.map(processNewNote),
...oldNotes.filter((oldNote) => !convertedNotes.some((n) => n.id === oldNote.id)),
...convertedNotes.map(processNewNote),
];
setConfig(bookKey, { booknotes: mergedNotes });
}
};
processSyncedNotes();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [syncedNotes]);
};
+3 -1
View File
@@ -84,7 +84,9 @@ export interface BookNote {
metaHash?: string;
id: string;
type: BookNoteType;
cfi: string;
cfi: string; // Canonicalized CFI for the note location
xpointer0?: string; // Start XPointer for the note location
xpointer1?: string; // End XPointer for the note location
page?: number;
text?: string;
style?: HighlightStyle;
+3 -1
View File
@@ -40,7 +40,9 @@ export interface DBBookNote {
meta_hash?: string;
id: string;
type: string;
cfi: string;
cfi?: string;
xpointer0?: string;
xpointer1?: string;
page?: number;
text?: string;
style?: string;
+9 -1
View File
@@ -147,6 +147,8 @@ export const transformBookNoteToDB = (bookNote: unknown, userId: string): DBBook
id,
type,
cfi,
xpointer0,
xpointer1,
page,
text,
style,
@@ -164,6 +166,8 @@ export const transformBookNoteToDB = (bookNote: unknown, userId: string): DBBook
id,
type,
cfi,
xpointer0,
xpointer1,
page,
text: sanitizeString(text),
style,
@@ -183,6 +187,8 @@ export const transformBookNoteFromDB = (dbBookNote: DBBookNote): BookNote => {
id,
type,
cfi,
xpointer0,
xpointer1,
page,
text,
style,
@@ -198,7 +204,9 @@ export const transformBookNoteFromDB = (dbBookNote: DBBookNote): BookNote => {
metaHash: meta_hash,
id,
type: type as BookNoteType,
cfi,
cfi: cfi ?? '',
xpointer0,
xpointer1,
page,
text,
style: style as HighlightStyle,
+92 -38
View File
@@ -187,8 +187,30 @@ export class XCFI {
/**
* Parse XPointer string to extract element and text offset
*
* Supports two KOReader text reference formats:
* - `/text().N` — cumulative character offset across all text in the element
* - `/text()[K].N` — Kth direct text node child (1-based), offset N within that node
*/
private parseXPointer(xpointer: string): { element: Element; textOffset?: number } {
// Format: /text()[K].N — indexed text node with offset
const indexedTextMatch = xpointer.match(/\/text\(\)\[(\d+)\]\.(\d+)$/);
if (indexedTextMatch) {
const textNodeIndex = parseInt(indexedTextMatch[1]!, 10); // 1-based
const offsetInNode = parseInt(indexedTextMatch[2]!, 10);
const elementPath = xpointer.replace(/\/text\(\)\[\d+\]\.\d+$/, '');
const element = this.resolveXPointerPath(elementPath);
if (!element) {
throw new Error(`Cannot resolve XPointer path: ${elementPath}`);
}
// Find the Kth direct text node child and compute cumulative offset
const textOffset = this.resolveIndexedTextNode(element, textNodeIndex, offsetInNode);
return { element, textOffset };
}
// Format: /text().N — cumulative character offset
const textOffsetMatch = xpointer.match(/\/text\(\)\.(\d+)$/);
const textOffset = textOffsetMatch ? parseInt(textOffsetMatch[1]!, 10) : undefined;
@@ -203,6 +225,38 @@ export class XCFI {
return { element, textOffset };
}
/**
* Resolve text()[K].N to a cumulative character offset within the element.
* K is the 1-based index of direct text node children of the element
* (counting only Text nodes that are immediate children, skipping element children).
* N is the character offset within that specific text node.
*/
private resolveIndexedTextNode(
element: Element,
textNodeIndex: number,
offsetInNode: number,
): number {
let directTextCount = 0;
let cumulativeOffset = 0;
for (const child of Array.from(element.childNodes)) {
if (child.nodeType === Node.TEXT_NODE) {
directTextCount++;
if (directTextCount === textNodeIndex) {
return cumulativeOffset + offsetInNode;
}
cumulativeOffset += (child.textContent || '').length;
} else if (child.nodeType === Node.ELEMENT_NODE) {
// Count text length inside child elements for cumulative offset
cumulativeOffset += (child.textContent || '').length;
}
}
throw new Error(
`Text node index ${textNodeIndex} out of bounds (found ${directTextCount} direct text nodes)`,
);
}
private resolveXPointerPath(path: string): Element | null {
const pathMatch = path.match(/^\/body\/DocFragment\[\d+\]\/body(.*)$/);
if (!pathMatch) {
@@ -239,9 +293,10 @@ export class XCFI {
throw new Error(`Invalid XPointer segment: ${segment}`);
}
// Find child elements with matching tag name
// Find child elements with matching tag name, skipping cfi-inert elements
const children = Array.from(current.children).filter(
(child) => child.tagName.toLowerCase() === tagName?.toLowerCase(),
(child) =>
!XCFI.isCfiInert(child) && child.tagName.toLowerCase() === tagName?.toLowerCase(),
);
if (index >= children.length) {
@@ -351,6 +406,14 @@ export class XCFI {
}
}
/**
* Check if an element is injected by Readest at runtime and should be
* invisible to XPointer path building / resolution (e.g. skip-link div).
*/
private static isCfiInert(element: Element): boolean {
return element.hasAttribute('cfi-inert');
}
/**
* Build XPointer path from DOM element
*/
@@ -364,10 +427,11 @@ export class XCFI {
if (!parent) break;
const tagName = current.tagName.toLowerCase();
// Count preceding siblings with same tag name (0-based for CREngine)
// Count preceding siblings with same tag name, skipping cfi-inert elements
let siblingIndex = 0;
let totalSameTagSiblings = 0;
for (const sibling of Array.from(parent.children)) {
if (XCFI.isCfiInert(sibling)) continue;
if (sibling.tagName.toLowerCase() === tagName) {
if (sibling === current) {
siblingIndex = totalSameTagSiblings;
@@ -400,7 +464,9 @@ export class XCFI {
}
/**
* Handle text offset within an element by finding character position
* Handle text offset within an element by finding character position.
* Produces KOReader-compatible text()[K].N format where K is the 1-based
* index of the direct text node child within its parent element.
*/
private handleTextOffset(element: Element, cfiOffset: number): string {
const textNodes: Text[] = [];
@@ -428,18 +494,29 @@ export class XCFI {
return this.buildXPointerPath(element);
}
// Find the containing element for this text node
let textParent = targetTextNode.parentElement;
while (textParent && !this.isSignificantElement(textParent)) {
textParent = textParent.parentElement;
}
if (!textParent) {
textParent = element as HTMLElement;
}
// Use the text node's direct parent for both path and indexing.
// This produces correct XPointers even when inline elements (like <a>)
// split text into multiple direct text node children.
const textParent = targetTextNode.parentElement || element;
const basePath = this.buildXPointerPath(textParent);
return `${basePath}/text().${offsetInNode}`;
// Count direct text node children and find which one the target is
let directTextCount = 0;
let directTextIndex = 0;
for (const child of Array.from(textParent.childNodes)) {
if (child.nodeType === Node.TEXT_NODE && (child.textContent || '').length > 0) {
directTextCount++;
if (child === targetTextNode) {
directTextIndex = directTextCount;
}
}
}
// Omit [1] when there is only one direct text node (matches KOReader format)
if (directTextCount <= 1) {
return `${basePath}/text().${offsetInNode}`;
}
return `${basePath}/text()[${directTextIndex || 1}].${offsetInNode}`;
}
/**
@@ -477,29 +554,6 @@ export class XCFI {
}
}
}
/**
* Check if an element is significant for XPointer path building
*/
private isSignificantElement(element: Element): boolean {
const tagName = element.tagName.toLowerCase();
// Skip inline formatting elements that don't affect structure
const inlineElements = new Set([
'span',
'em',
'strong',
'i',
'b',
'u',
'small',
'mark',
'sup',
'sub',
]);
return !inlineElements.has(tagName);
}
}
export const getCFIFromXPointer = async (
+112 -635
View File
@@ -1,22 +1,20 @@
local Device = require("device")
local Event = require("ui/event")
local Dispatcher = require("dispatcher")
local InfoMessage = require("ui/widget/infomessage")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local MultiInputDialog = require("ui/widget/multiinputdialog")
local NetworkMgr = require("ui/network/manager")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local time = require("ui/time")
local util = require("util")
local sha2 = require("ffi/sha2")
local T = require("ffi/util").template
local _ = require("gettext")
local SyncAuth = require("syncauth")
local SyncConfig = require("syncconfig")
local SyncAnnotations = require("syncannotations")
local SelfUpdate = require("selfupdate")
local ReadestSync = WidgetContainer:new{
name = "readest",
title = _("Readest Sync"),
settings = nil,
}
@@ -37,14 +35,7 @@ ReadestSync.default_settings = {
last_sync_at = nil,
}
local UPDATE_URLS = {
"https://download.readest.com/releases/latest.json",
"https://github.com/readest/readest/releases/latest/download/latest.json",
}
local DOWNLOAD_URLS = {
"https://download.readest.com/releases/%s/Readest-%s-1.koplugin.zip",
"https://github.com/readest/readest/releases/download/%s/Readest-%s-1.koplugin.zip",
}
-- ── Lifecycle ──────────────────────────────────────────────────────
function ReadestSync:init()
self.last_sync_timestamp = 0
@@ -63,17 +54,22 @@ function ReadestSync:onDispatcherRegisterActions()
Dispatcher:registerAction("readest_sync_toggle_autosync", { category="none", event="ReadestSyncToggleAutoSync", title=_("Toggle auto readest sync"), reader=true,})
Dispatcher:registerAction("readest_sync_push_progress", { category="none", event="ReadestSyncPushProgress", title=_("Push readest progress from this device"), reader=true,})
Dispatcher:registerAction("readest_sync_pull_progress", { category="none", event="ReadestSyncPullProgress", title=_("Pull readest progress from other devices"), reader=true, separator=true,})
Dispatcher:registerAction("readest_sync_push_annotations", { category="none", event="ReadestSyncPushAnnotations", title=_("Push readest annotations from this device"), reader=true,})
Dispatcher:registerAction("readest_sync_pull_annotations", { category="none", event="ReadestSyncPullAnnotations", title=_("Pull readest annotations from other devices"), reader=true, separator=true,})
end
function ReadestSync:onReaderReady()
if self.settings.auto_sync and self.settings.access_token then
UIManager:nextTick(function()
self:pullBookConfig(false)
self:pullBookNotes(false)
end)
end
self:onDispatcherRegisterActions()
end
-- ── Menu ───────────────────────────────────────────────────────────
function ReadestSync:addToMainMenu(menu_items)
menu_items.readest_sync = {
sorting_hint = "tools",
@@ -81,24 +77,24 @@ function ReadestSync:addToMainMenu(menu_items)
sub_item_table = {
{
text_func = function()
return self:needsLogin() and _("Log in Readest Account")
return SyncAuth:needsLogin(self.settings) and _("Log in Readest Account")
or _("Log out as ") .. (self.settings.user_name or "")
end,
callback_func = function()
if self:needsLogin() then
if SyncAuth:needsLogin(self.settings) then
return function(menu)
self:login(menu)
SyncAuth:login(self.settings, self.path, self.title, menu)
end
else
return function(menu)
self:logout(menu)
SyncAuth:logout(self.settings, menu)
end
end
end,
separator = true,
},
{
text = _("Auto sync book configs"),
text = _("Auto sync progress and annotations"),
checked_func = function() return self.settings.auto_sync end,
callback = function()
self:onReadestSyncToggleAutoSync()
@@ -124,6 +120,25 @@ function ReadestSync:addToMainMenu(menu_items)
end,
separator = true,
},
{
text = _("Push annotations now"),
enabled_func = function()
return self.settings.access_token ~= nil and self.ui.document ~= nil
end,
callback = function()
self:pushBookNotes(true)
end,
},
{
text = _("Pull annotations now"),
enabled_func = function()
return self.settings.access_token ~= nil and self.ui.document ~= nil
end,
callback = function()
self:pullBookNotes(true)
end,
separator = true,
},
{
text_func = function()
if self.installed_version then
@@ -132,340 +147,16 @@ function ReadestSync:addToMainMenu(menu_items)
return _("Check for update")
end,
callback = function()
self:checkForUpdate()
SelfUpdate:checkForUpdate(self.path, self.installed_version)
end,
},
}
}
end
function ReadestSync:needsLogin()
return not self.settings.access_token or not self.settings.expires_at
or self.settings.expires_at < os.time() + 60
end
-- ── Sync helpers (thin wrappers around modules) ────────────────────
function ReadestSync:tryRefreshToken()
if self.settings.refresh_token and self.settings.expires_at
and self.settings.expires_at < os.time() + self.settings.expires_in / 2 then
local client = self:getSupabaseAuthClient()
client:refresh_token(self.settings.refresh_token, function(success, response)
if success then
self.settings.access_token = response.access_token
self.settings.refresh_token = response.refresh_token
self.settings.expires_at = response.expires_at
self.settings.expires_in = response.expires_in
G_reader_settings:saveSetting("readest_sync", self.settings)
else
logger.err("ReadestSync: Token refresh failed:", response or "Unknown error")
end
end)
end
end
function ReadestSync:getSupabaseAuthClient()
if not self.settings.supabase_url or not self.settings.supabase_anon_key then
return nil
end
local SupabaseAuthClient = require("supabaseauth")
return SupabaseAuthClient:new{
service_spec = self.path .. "/supabase-auth-api.json",
custom_url = self.settings.supabase_url .. "/auth/v1/",
api_key = self.settings.supabase_anon_key,
}
end
function ReadestSync:getReadestSyncClient()
if not self.settings.access_token or not self.settings.expires_at or self.settings.expires_at < os.time() then
return nil
end
local ReadestSyncClient = require("readestsync")
return ReadestSyncClient:new{
service_spec = self.path .. "/readest-sync-api.json",
access_token = self.settings.access_token,
}
end
function ReadestSync:login(menu)
if NetworkMgr:willRerunWhenOnline(function() self:login(menu) end) then
return
end
local dialog
dialog = MultiInputDialog:new{
title = self.title,
fields = {
{
text = self.settings.user_email,
hint = "email@example.com",
},
{
hint = "password",
text_type = "password",
},
},
buttons = {
{
{
text = _("Cancel"),
id = "close",
callback = function()
UIManager:close(dialog)
end,
},
{
text = _("Login"),
callback = function()
local email, password = unpack(dialog:getFields())
email = util.trim(email)
if email == "" or password == "" then
UIManager:show(InfoMessage:new{
text = _("Please enter both email and password"),
timeout = 2,
})
return
end
UIManager:close(dialog)
self:doLogin(email, password, menu)
end,
},
},
},
}
UIManager:show(dialog)
dialog:onShowKeyboard()
end
function ReadestSync:doLogin(email, password, menu)
local client = self:getSupabaseAuthClient()
if not client then
UIManager:show(InfoMessage:new{
text = _("Please configure Supabase URL and API key first"),
timeout = 3,
})
return
end
UIManager:show(InfoMessage:new{
text = _("Logging in..."),
timeout = 1,
})
Device:setIgnoreInput(true)
local success, response = client:sign_in_password(email, password)
Device:setIgnoreInput(false)
if success then
self.settings.user_email = email
self.settings.user_id = response.user.id
self.settings.user_name = response.user.user_metadata.user_name or email
self.settings.access_token = response.access_token
self.settings.refresh_token = response.refresh_token
self.settings.expires_at = response.expires_at
self.settings.expires_in = response.expires_in
G_reader_settings:saveSetting("readest_sync", self.settings)
if menu then
menu:updateItems()
end
UIManager:show(InfoMessage:new{
text = _("Successfully logged in to Readest"),
timeout = 3,
})
else
UIManager:show(InfoMessage:new{
text = _("Login failed: ") .. (response.msg or "Unknown error"),
timeout = 3,
})
end
end
function ReadestSync:logout(menu)
if self.access_token then
local client = self:getSupabaseAuthClient()
if client then
client:sign_out(self.settings.access_token, function(success, response)
logger.dbg("ReadestSync: Sign out result:", success)
end)
end
end
self.settings.access_token = nil
self.settings.refresh_token = nil
self.settings.expires_at = nil
self.settings.expires_in = nil
G_reader_settings:saveSetting("readest_sync", self.settings)
if menu then
menu:updateItems()
end
UIManager:show(InfoMessage:new{
text = _("Logged out from Readest Sync"),
timeout = 2,
})
end
function normalizeIdentifier(identifier)
if identifier:match("urn:") then
-- Slice after the last ':'
return identifier:match("([^:]+)$")
elseif identifier:match(":") then
-- Slice after the first ':'
return identifier:match("^[^:]+:(.+)$")
end
return identifier
end
function normalizeAuthor(author)
-- Trim leading and trailing whitespace
author = author:gsub("^%s*(.-)%s*$", "%1")
return author
end
function ReadestSync:generateMetadataHash()
local doc_props = self.ui.doc_settings:readSetting("doc_props") or {}
local title = doc_props.title or ''
if title == '' then
local doc_path, filename = util.splitFilePathName(self.ui.doc_settings:readSetting("doc_path") or '')
local basename, suffix = util.splitFileNameSuffix(filename)
title = basename or ''
end
local authors = doc_props.authors or ''
if authors:find("\n") then
authors = util.splitToArray(authors, "\n")
for i, author in ipairs(authors) do
authors[i] = normalizeAuthor(author)
end
authors = table.concat(authors, ",")
else
authors = normalizeAuthor(authors)
end
local identifiers = doc_props.identifiers or ''
if identifiers:find("\n") then
local list = util.splitToArray(identifiers, "\n")
local normalized = {}
local priorities = { "uuid", "calibre", "isbn" }
local preferred = nil
for i, id in ipairs(list) do
normalized[i] = normalizeIdentifier(id)
local candidate = id:lower()
for _, p in ipairs(priorities) do
if candidate:find(p, 1, true) then
preferred = normalized[i]
break
end
end
end
if preferred then
identifiers = preferred
else
identifiers = table.concat(normalized, ",")
end
else
identifiers = normalizeIdentifier(identifiers)
end
local doc_meta = title .. "|" .. authors .. "|" .. identifiers
local meta_hash = sha2.md5(doc_meta)
return meta_hash
end
function ReadestSync:getMetaHash()
local doc_readest_sync = self.ui.doc_settings:readSetting("readest_sync") or {}
local meta_hash = doc_readest_sync.meta_hash_v1
if not meta_hash then
meta_hash = self:generateMetadataHash()
doc_readest_sync.meta_hash_v1 = meta_hash
self.ui.doc_settings:saveSetting("readest_sync", doc_readest_sync)
end
return meta_hash
end
function ReadestSync:getDocumentIdentifier()
return self.ui.doc_settings:readSetting("partial_md5_checksum")
end
function ReadestSync:showSyncedMessage()
UIManager:show(InfoMessage:new{
text = _("Progress has been synchronized."),
timeout = 3,
})
end
function ReadestSync:applyBookConfig(config)
logger.dbg("ReadestSync: Applying book config:", config)
local xpointer = config.xpointer
local progress = config.progress
local has_pages = self.ui.document.info.has_pages
-- Check if it's the bracket format: [page,total_pages]
local progress_pattern = "^%[(%d+),(%d+)%]$"
if has_pages and progress then
local page, total_pages = progress:match(progress_pattern)
local current_page = self.ui:getCurrentPage()
local new_page = tonumber(page)
if new_page > current_page then
self.ui.link:addCurrentLocationToStack()
self.ui:handleEvent(Event:new("GotoPage", new_page))
self:showSyncedMessage()
end
end
if not has_pages and xpointer then
local last_xpointer = self.ui.rolling:getLastProgress()
local working_xpointer = xpointer
local cmp_result = self.document:compareXPointers(last_xpointer, working_xpointer)
-- FIXME: Crengine is not very good at comparing XPointers, so we need to reduce the path
while cmp_result == nil and working_xpointer do
local last_slash_pos = working_xpointer:match("^.*()/")
if last_slash_pos and last_slash_pos > 1 then
working_xpointer = working_xpointer:sub(1, last_slash_pos - 1)
cmp_result = self.document:compareXPointers(last_xpointer, working_xpointer)
else
break
end
end
if cmp_result > 0 then
self.ui.link:addCurrentLocationToStack()
self.ui:handleEvent(Event:new("GotoXPointer", working_xpointer))
self:showSyncedMessage()
end
end
end
function ReadestSync:getCurrentBookConfig()
local book_hash = self:getDocumentIdentifier()
local meta_hash = self:getMetaHash()
if not book_hash or not meta_hash then
UIManager:show(InfoMessage:new{
text = _("Cannot identify the current book"),
timeout = 2,
})
return nil
end
local config = {
bookHash = book_hash,
metaHash = meta_hash,
progress = "",
xpointer = "",
updatedAt = os.time() * 1000,
}
local current_page = self.ui:getCurrentPage()
local page_count = self.ui.document:getPageCount()
config.progress = {current_page, page_count}
if not self.ui.document.info.has_pages then
config.xpointer = self.ui.rolling:getLastProgress()
end
return config
end
function ReadestSync:pushBookConfig(interactive)
function ReadestSync:ensureClient(interactive)
if not self.settings.access_token or not self.settings.user_id then
if interactive then
UIManager:show(InfoMessage:new{
@@ -473,23 +164,12 @@ function ReadestSync:pushBookConfig(interactive)
timeout = 2,
})
end
return
return nil
end
local now = os.time()
if not interactive and now - self.last_sync_timestamp <= API_CALL_DEBOUNCE_DELAY then
logger.dbg("ReadestSync: Debouncing push request")
return
end
SyncAuth:tryRefreshToken(self.settings, self.path)
local config = self:getCurrentBookConfig()
if not config then return end
if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookConfig(interactive) end) then
return
end
local client = self:getReadestSyncClient()
local client = SyncAuth:getReadestSyncClient(self.settings, self.path)
if not client then
if interactive then
UIManager:show(InfoMessage:new{
@@ -497,140 +177,85 @@ function ReadestSync:pushBookConfig(interactive)
timeout = 3,
})
end
return nil
end
return client
end
function ReadestSync:getBookIdentifiers()
local book_hash = SyncConfig:getDocumentIdentifier(self.ui)
local meta_hash = SyncConfig:getMetaHash(self.ui)
return book_hash, meta_hash
end
-- ── Config sync ────────────────────────────────────────────────────
function ReadestSync:pushBookConfig(interactive)
local now = os.time()
if not interactive and now - self.last_sync_timestamp <= API_CALL_DEBOUNCE_DELAY then
return
end
self:tryRefreshToken()
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pushing book config..."),
timeout = 1,
})
if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookConfig(interactive) end) then
return
end
local payload = {
books = {},
notes = {},
configs = { config }
}
local client = self:ensureClient(interactive)
if not client then return end
client:pushChanges(
payload,
function(success, response)
if interactive then
if success then
UIManager:show(InfoMessage:new{
text = _("Book config pushed successfully"),
timeout = 2,
})
else
UIManager:show(InfoMessage:new{
text = _("Failed to push book config"),
timeout = 2,
})
end
end
if success then
self.last_sync_timestamp = os.time()
end
end
self.last_sync_timestamp = SyncConfig:push(
self.ui, self.settings, client, interactive, self.last_sync_timestamp
)
end
function ReadestSync:pullBookConfig(interactive)
if not self.settings.access_token or not self.settings.user_id then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Please login first"),
timeout = 2,
})
end
return
end
local book_hash = self:getDocumentIdentifier()
local meta_hash = self:getMetaHash()
local book_hash, meta_hash = self:getBookIdentifiers()
if not book_hash or not meta_hash then return end
if NetworkMgr:willRerunWhenOnline(function() self:pullBookConfig(interactive) end) then
return
end
local client = self:getReadestSyncClient()
if not client then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Please configure Readest settings first"),
timeout = 3,
})
end
local client = self:ensureClient(interactive)
if not client then return end
SyncConfig:pull(
self.ui, self.settings, client, book_hash, meta_hash, interactive,
function() SyncAuth:logout(self.settings) end
)
end
-- ── Annotation sync ────────────────────────────────────────────────
function ReadestSync:pushBookNotes(interactive)
if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookNotes(interactive) end) then
return
end
self:tryRefreshToken()
local client = self:ensureClient(interactive)
if not client then return end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pulling book config..."),
timeout = 1,
})
SyncAnnotations:push(self.ui, self.settings, client, interactive)
end
function ReadestSync:pullBookNotes(interactive)
local book_hash, meta_hash = self:getBookIdentifiers()
if not book_hash or not meta_hash then return end
if NetworkMgr:willRerunWhenOnline(function() self:pullBookNotes(interactive) end) then
return
end
client:pullChanges(
{
since = 0,
type = "configs",
book = book_hash,
meta_hash = meta_hash,
},
function(success, response)
if not success then
if response and response.error == "Not authenticated" then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Authentication failed, please login again"),
timeout = 2,
})
end
self:logout()
return
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Failed to pull book config"),
timeout = 2,
})
end
return
end
local client = self:ensureClient(interactive)
if not client then return end
local data = response.configs
if data and #data > 0 then
local config = data[1]
if config then
self:applyBookConfig(config)
if interactive then
UIManager:show(InfoMessage:new{
text = _("Book config synchronized"),
timeout = 2,
})
end
return
end
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("No saved config found for this book"),
timeout = 2,
})
end
end
SyncAnnotations:pull(
self.ui, self.settings, client, book_hash, meta_hash, self.dialog, interactive
)
end
-- ── Event handlers ─────────────────────────────────────────────────
function ReadestSync:onReadestSyncToggleAutoSync(toggle)
if toggle == self.settings.auto_sync then
return true
@@ -650,10 +275,19 @@ function ReadestSync:onReadestSyncPullProgress()
self:pullBookConfig(true)
end
function ReadestSync:onReadestSyncPushAnnotations()
self:pushBookNotes(true)
end
function ReadestSync:onReadestSyncPullAnnotations()
self:pullBookNotes(true)
end
function ReadestSync:onCloseDocument()
if self.settings.auto_sync and self.settings.access_token then
NetworkMgr:goOnlineToRun(function()
self:pushBookConfig(false)
self:pushBookNotes(false)
end)
end
end
@@ -670,6 +304,14 @@ function ReadestSync:onPageUpdate(page)
end
end
function ReadestSync:onAnnotationsModified()
if self.settings.auto_sync and self.settings.access_token then
UIManager:nextTick(function()
self:pushBookNotes(false)
end)
end
end
function ReadestSync:onCloseWidget()
if self.delayed_push_task then
UIManager:unschedule(self.delayed_push_task)
@@ -677,169 +319,4 @@ function ReadestSync:onCloseWidget()
end
end
function ReadestSync:compareVersions(v1, v2)
local function parse(v)
local parts = {}
for num in tostring(v):gmatch("(%d+)") do
table.insert(parts, tonumber(num))
end
return parts
end
local p1, p2 = parse(v1), parse(v2)
local len = math.max(#p1, #p2)
for i = 1, len do
local a, b = p1[i] or 0, p2[i] or 0
if a < b then return -1 end
if a > b then return 1 end
end
return 0
end
function ReadestSync:fetchLatestVersion()
local http = require("socket.http")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
local json = require("json")
for _, url in ipairs(UPDATE_URLS) do
local sink = {}
socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
local code = socket.skip(1, http.request{
url = url,
sink = ltn12.sink.table(sink),
})
socketutil:reset_timeout()
if code == 200 then
local ok, data = pcall(json.decode, table.concat(sink))
if ok and data and data.version then
return data.version
end
end
logger.dbg("ReadestSync: failed to fetch latest.json from", url, "code:", code)
end
return nil
end
function ReadestSync:checkForUpdate()
local ConfirmBox = require("ui/widget/confirmbox")
if NetworkMgr:willRerunWhenOnline(function() self:checkForUpdate() end) then
return
end
UIManager:show(InfoMessage:new{
text = _("Checking for update…"),
timeout = 1,
})
local current_version = self.installed_version
Device:setIgnoreInput(true)
local latest_version = self:fetchLatestVersion()
Device:setIgnoreInput(false)
if not latest_version then
UIManager:show(InfoMessage:new{
text = _("Failed to check for update. Please try again later."),
timeout = 3,
})
return
end
if not current_version or self:compareVersions(current_version, latest_version) < 0 then
UIManager:show(ConfirmBox:new{
text = current_version
and T(_("A new version is available: v%1 (current: v%2).\n\nDo you want to update now?"), latest_version, current_version)
or T(_("A new version is available: v%1.\n\nDo you want to update now?"), latest_version),
ok_text = _("Update"),
ok_callback = function()
self:downloadAndInstall(latest_version)
end,
})
else
UIManager:show(InfoMessage:new{
text = T(_("You are up to date (v%1)."), current_version),
timeout = 3,
})
end
end
function ReadestSync:downloadAndInstall(version)
local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local http = require("socket.http")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
if NetworkMgr:willRerunWhenOnline(function() self:downloadAndInstall(version) end) then
return
end
local tag = "v" .. version
local zip_name = "Readest-" .. version .. "-1.koplugin.zip"
local tmp_path = DataStorage:getDataDir() .. "/" .. zip_name
UIManager:show(InfoMessage:new{
text = _("Downloading update…"),
timeout = 1,
})
Device:setIgnoreInput(true)
local download_ok = false
for _, url_template in ipairs(DOWNLOAD_URLS) do
local url = string.format(url_template, tag, version)
logger.dbg("ReadestSync: downloading from", url)
socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
local code = socket.skip(1, http.request{
url = url,
sink = ltn12.sink.file(io.open(tmp_path, "w")),
})
socketutil:reset_timeout()
if code == 200 then
download_ok = true
break
end
logger.dbg("ReadestSync: download failed from", url, "code:", code)
end
Device:setIgnoreInput(false)
if not download_ok then
os.remove(tmp_path)
UIManager:show(InfoMessage:new{
text = _("Failed to download update. Please try again later."),
timeout = 3,
})
return
end
local plugin_dir = self.path
local parent_dir = plugin_dir:match("(.*/)")
local ok, err = Device:unpackArchive(tmp_path, parent_dir)
os.remove(tmp_path)
if ok then
UIManager:show(ConfirmBox:new{
text = T(_("Readest plugin updated to v%1.\n\nPlease restart KOReader to apply the update."), version),
ok_text = _("Restart now"),
ok_callback = function()
UIManager:restartKOReader()
end,
cancel_text = _("Later"),
})
else
UIManager:show(InfoMessage:new{
text = T(_("Failed to install update: %1"), err or _("unknown error")),
timeout = 5,
})
end
end
return ReadestSync
return ReadestSync
+182
View File
@@ -0,0 +1,182 @@
local Device = require("device")
local InfoMessage = require("ui/widget/infomessage")
local NetworkMgr = require("ui/network/manager")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local T = require("ffi/util").template
local _ = require("gettext")
local SelfUpdate = {}
local UPDATE_URLS = {
"https://download.readest.com/releases/latest.json",
"https://github.com/readest/readest/releases/latest/download/latest.json",
}
local DOWNLOAD_URLS = {
"https://download.readest.com/releases/%s/Readest-%s-1.koplugin.zip",
"https://github.com/readest/readest/releases/download/%s/Readest-%s-1.koplugin.zip",
}
function SelfUpdate:compareVersions(v1, v2)
local function parse(v)
local parts = {}
for num in tostring(v):gmatch("(%d+)") do
table.insert(parts, tonumber(num))
end
return parts
end
local p1, p2 = parse(v1), parse(v2)
local len = math.max(#p1, #p2)
for i = 1, len do
local a, b = p1[i] or 0, p2[i] or 0
if a < b then return -1 end
if a > b then return 1 end
end
return 0
end
function SelfUpdate:fetchLatestVersion()
local http = require("socket.http")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
local json = require("json")
for _, url in ipairs(UPDATE_URLS) do
local sink = {}
socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT)
local code = socket.skip(1, http.request{
url = url,
sink = ltn12.sink.table(sink),
})
socketutil:reset_timeout()
if code == 200 then
local ok, data = pcall(json.decode, table.concat(sink))
if ok and data and data.version then
return data.version
end
end
logger.dbg("ReadestSync: failed to fetch latest.json from", url, "code:", code)
end
return nil
end
function SelfUpdate:checkForUpdate(plugin_path, installed_version)
local ConfirmBox = require("ui/widget/confirmbox")
if NetworkMgr:willRerunWhenOnline(function() self:checkForUpdate(plugin_path, installed_version) end) then
return
end
UIManager:show(InfoMessage:new{
text = _("Checking for update…"),
timeout = 1,
})
Device:setIgnoreInput(true)
local latest_version = self:fetchLatestVersion()
Device:setIgnoreInput(false)
if not latest_version then
UIManager:show(InfoMessage:new{
text = _("Failed to check for update. Please try again later."),
timeout = 3,
})
return
end
if not installed_version or self:compareVersions(installed_version, latest_version) < 0 then
UIManager:show(ConfirmBox:new{
text = installed_version
and T(_("A new version is available: v%1 (current: v%2).\n\nDo you want to update now?"), latest_version, installed_version)
or T(_("A new version is available: v%1.\n\nDo you want to update now?"), latest_version),
ok_text = _("Update"),
ok_callback = function()
self:downloadAndInstall(plugin_path, latest_version)
end,
})
else
UIManager:show(InfoMessage:new{
text = T(_("You are up to date (v%1)."), installed_version),
timeout = 3,
})
end
end
function SelfUpdate:downloadAndInstall(plugin_path, version)
local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local http = require("socket.http")
local ltn12 = require("ltn12")
local socket = require("socket")
local socketutil = require("socketutil")
if NetworkMgr:willRerunWhenOnline(function() self:downloadAndInstall(plugin_path, version) end) then
return
end
local tag = "v" .. version
local zip_name = "Readest-" .. version .. "-1.koplugin.zip"
local tmp_path = DataStorage:getDataDir() .. "/" .. zip_name
UIManager:show(InfoMessage:new{
text = _("Downloading update…"),
timeout = 1,
})
Device:setIgnoreInput(true)
local download_ok = false
for _, url_template in ipairs(DOWNLOAD_URLS) do
local url = string.format(url_template, tag, version)
logger.dbg("ReadestSync: downloading from", url)
socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT)
local code = socket.skip(1, http.request{
url = url,
sink = ltn12.sink.file(io.open(tmp_path, "w")),
})
socketutil:reset_timeout()
if code == 200 then
download_ok = true
break
end
logger.dbg("ReadestSync: download failed from", url, "code:", code)
end
Device:setIgnoreInput(false)
if not download_ok then
os.remove(tmp_path)
UIManager:show(InfoMessage:new{
text = _("Failed to download update. Please try again later."),
timeout = 3,
})
return
end
local parent_dir = plugin_path:match("(.*/)")
local ok, err = Device:unpackArchive(tmp_path, parent_dir)
os.remove(tmp_path)
if ok then
UIManager:show(ConfirmBox:new{
text = T(_("Readest plugin updated to v%1.\n\nPlease restart KOReader to apply the update."), version),
ok_text = _("Restart now"),
ok_callback = function()
UIManager:restartKOReader()
end,
cancel_text = _("Later"),
})
else
UIManager:show(InfoMessage:new{
text = T(_("Failed to install update: %1"), err or _("unknown error")),
timeout = 5,
})
end
end
return SelfUpdate
+277
View File
@@ -0,0 +1,277 @@
local Event = require("ui/event")
local InfoMessage = require("ui/widget/infomessage")
local NetworkMgr = require("ui/network/manager")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local sha2 = require("ffi/sha2")
local T = require("ffi/util").template
local _ = require("gettext")
local SyncAnnotations = {}
-- KOReader color name → Readest color value
local KO_TO_READEST_COLOR = {
yellow = "yellow",
red = "red",
green = "green",
blue = "blue",
purple = "violet",
orange = "#ff8800",
cyan = "#00bcd4",
olive = "#808000",
gray = "#9e9e9e",
}
-- Readest color value → KOReader color name
local READEST_TO_KO_COLOR = {
yellow = "yellow",
red = "red",
green = "green",
blue = "blue",
violet = "purple",
["#ff8800"] = "orange",
["#00bcd4"] = "cyan",
["#808000"] = "olive",
["#9e9e9e"] = "gray",
}
function SyncAnnotations:parseDatetimeToMs(dt)
if not dt then return os.time() * 1000 end
local y, m, d, h, min, s = dt:match("(%d+)-(%d+)-(%d+) (%d+):(%d+):(%d+)")
if y then
return os.time({
year = tonumber(y), month = tonumber(m), day = tonumber(d),
hour = tonumber(h), min = tonumber(min), sec = tonumber(s),
}) * 1000
end
return os.time() * 1000
end
function SyncAnnotations:generateNoteId(book_hash, note_type, pos0, pos1)
local raw = "ko:" .. book_hash .. ":" .. note_type .. ":" .. (pos0 or "") .. ":" .. (pos1 or "")
return sha2.md5(raw):sub(1, 7)
end
function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash)
local annotations = ui.annotation and ui.annotation.annotations
if not annotations then return {} end
local last_sync = settings.last_notes_sync_at or 0
local notes = {}
for _, item in ipairs(annotations) do
local pos0 = item.pos0
local pos1 = item.pos1
if type(pos0) == "table" then pos0 = nil end
if type(pos1) == "table" then pos1 = nil end
if pos0 then
local updated_at = self:parseDatetimeToMs(item.datetime_updated or item.datetime)
if updated_at <= last_sync then
goto skip
end
local note_type = item.drawer and "annotation" or "bookmark"
local id = self:generateNoteId(book_hash, note_type, tostring(pos0), pos1 and tostring(pos1))
local style = "highlight"
if item.drawer == "underscore" then
style = "underline"
elseif item.drawer == "strikeout" then
style = "squiggly"
end
local note = {
bookHash = book_hash,
metaHash = meta_hash,
id = id,
type = note_type,
xpointer0 = tostring(pos0),
xpointer1 = pos1 and tostring(pos1) or nil,
text = item.text or "",
note = item.note or "",
style = note_type == "annotation" and style or nil,
color = note_type == "annotation" and KO_TO_READEST_COLOR[item.color or "yellow"] or nil,
page = item.pageno,
createdAt = self:parseDatetimeToMs(item.datetime),
updatedAt = updated_at,
}
table.insert(notes, note)
end
::skip::
end
return notes
end
function SyncAnnotations:push(ui, settings, client, interactive)
local book_hash = ui.doc_settings:readSetting("partial_md5_checksum")
local meta_hash = ui.doc_settings:readSetting("readest_sync") or {}
meta_hash = meta_hash.meta_hash_v1
if not book_hash or not meta_hash then return end
local annotations = self:getAnnotations(ui, settings, book_hash, meta_hash)
if #annotations == 0 then
if interactive then
UIManager:show(InfoMessage:new{
text = _("No annotations to push"),
timeout = 2,
})
end
return
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pushing annotations..."),
timeout = 1,
})
end
local payload = {
books = {},
notes = annotations,
configs = {},
}
logger.dbg("ReadestSync: Pushing annotations, payload:", payload)
client:pushChanges(
payload,
function(success, _response)
if interactive then
if success then
UIManager:show(InfoMessage:new{
text = T(_("%1 annotations pushed successfully"), #annotations),
timeout = 2,
})
else
UIManager:show(InfoMessage:new{
text = _("Failed to push annotations"),
timeout = 2,
})
end
end
if success then
settings.last_notes_sync_at = os.time() * 1000
G_reader_settings:saveSetting("readest_sync", settings)
end
end
)
end
function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog, interactive)
if ui.document.info.has_pages then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Annotation sync is not supported for PDF documents"),
timeout = 3,
})
end
return
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pulling annotations..."),
timeout = 1,
})
end
client:pullChanges(
{
since = settings.last_notes_sync_at or 0,
type = "notes",
book = book_hash,
meta_hash = meta_hash,
},
function(success, response)
if not success then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Failed to pull annotations"),
timeout = 2,
})
end
return
end
local data = response.notes
if not data or #data == 0 then
if interactive then
UIManager:show(InfoMessage:new{
text = _("No new annotations found"),
timeout = 2,
})
end
return
end
logger.dbg("ReadestSync: Pulled annotations from sync:", data)
local annotation_mgr = ui.annotation
if not annotation_mgr then return end
local existing_positions = {}
for _, item in ipairs(annotation_mgr.annotations) do
local key = tostring(item.pos0) .. "|" .. tostring(item.pos1 or "")
existing_positions[key] = true
end
local added = 0
for _, note in ipairs(data) do
if note.deleted_at then
goto continue
end
local xp0 = note.xpointer0
local xp1 = note.xpointer1
if not xp0 then goto continue end
local key = xp0 .. "|" .. (xp1 or "")
if existing_positions[key] then goto continue end
local drawer = "lighten"
local note_type = note.type
if note_type == "bookmark" then
drawer = nil
elseif note.style == "underline" then
drawer = "underscore"
elseif note.style == "squiggly" then
drawer = "strikeout"
end
local item = {
pos0 = xp0,
pos1 = xp1 or xp0,
page = xp0,
text = note.text or "",
note = note.note or "",
drawer = drawer,
color = READEST_TO_KO_COLOR[note.color] or "yellow",
pageno = note.page,
datetime = os.date("%Y-%m-%d %H:%M:%S"),
}
local index = annotation_mgr:addItem(item)
ui:handleEvent(Event:new("AnnotationsModified", { item, index_modified = index }))
logger.dbg("ReadestSync: Added annotation from sync:", item)
existing_positions[key] = true
added = added + 1
::continue::
end
settings.last_notes_sync_at = os.time() * 1000
G_reader_settings:saveSetting("readest_sync", settings)
if interactive then
UIManager:show(InfoMessage:new{
text = T(_("%1 annotations pulled"), added),
timeout = 2,
})
end
if added > 0 then
UIManager:setDirty(dialog, "ui")
end
end
)
end
return SyncAnnotations
+181
View File
@@ -0,0 +1,181 @@
local Device = require("device")
local InfoMessage = require("ui/widget/infomessage")
local MultiInputDialog = require("ui/widget/multiinputdialog")
local NetworkMgr = require("ui/network/manager")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local util = require("util")
local _ = require("gettext")
local SyncAuth = {}
function SyncAuth:needsLogin(settings)
return not settings.access_token or not settings.expires_at
or settings.expires_at < os.time() + 60
end
function SyncAuth:tryRefreshToken(settings, path)
if settings.refresh_token and settings.expires_at
and settings.expires_at < os.time() + settings.expires_in / 2 then
local client = self:getSupabaseAuthClient(settings, path)
client:refresh_token(settings.refresh_token, function(success, response)
if success then
settings.access_token = response.access_token
settings.refresh_token = response.refresh_token
settings.expires_at = response.expires_at
settings.expires_in = response.expires_in
G_reader_settings:saveSetting("readest_sync", settings)
else
logger.err("ReadestSync: Token refresh failed:", response or "Unknown error")
end
end)
end
end
function SyncAuth:getSupabaseAuthClient(settings, path)
if not settings.supabase_url or not settings.supabase_anon_key then
return nil
end
local SupabaseAuthClient = require("supabaseauth")
return SupabaseAuthClient:new{
service_spec = path .. "/supabase-auth-api.json",
custom_url = settings.supabase_url .. "/auth/v1/",
api_key = settings.supabase_anon_key,
}
end
function SyncAuth:getReadestSyncClient(settings, path)
if not settings.access_token or not settings.expires_at or settings.expires_at < os.time() then
return nil
end
local ReadestSyncClient = require("readestsync")
return ReadestSyncClient:new{
service_spec = path .. "/readest-sync-api.json",
access_token = settings.access_token,
}
end
function SyncAuth:login(settings, path, title, menu)
if NetworkMgr:willRerunWhenOnline(function() self:login(settings, path, title, menu) end) then
return
end
local dialog
dialog = MultiInputDialog:new{
title = title,
fields = {
{
text = settings.user_email,
hint = "email@example.com",
},
{
hint = "password",
text_type = "password",
},
},
buttons = {
{
{
text = _("Cancel"),
id = "close",
callback = function()
UIManager:close(dialog)
end,
},
{
text = _("Login"),
callback = function()
local email, password = unpack(dialog:getFields())
email = util.trim(email)
if email == "" or password == "" then
UIManager:show(InfoMessage:new{
text = _("Please enter both email and password"),
timeout = 2,
})
return
end
UIManager:close(dialog)
self:doLogin(settings, path, email, password, menu)
end,
},
},
},
}
UIManager:show(dialog)
dialog:onShowKeyboard()
end
function SyncAuth:doLogin(settings, path, email, password, menu)
local client = self:getSupabaseAuthClient(settings, path)
if not client then
UIManager:show(InfoMessage:new{
text = _("Please configure Supabase URL and API key first"),
timeout = 3,
})
return
end
UIManager:show(InfoMessage:new{
text = _("Logging in..."),
timeout = 1,
})
Device:setIgnoreInput(true)
local success, response = client:sign_in_password(email, password)
Device:setIgnoreInput(false)
if success then
settings.user_email = email
settings.user_id = response.user.id
settings.user_name = response.user.user_metadata.user_name or email
settings.access_token = response.access_token
settings.refresh_token = response.refresh_token
settings.expires_at = response.expires_at
settings.expires_in = response.expires_in
G_reader_settings:saveSetting("readest_sync", settings)
if menu then
menu:updateItems()
end
UIManager:show(InfoMessage:new{
text = _("Successfully logged in to Readest"),
timeout = 3,
})
else
UIManager:show(InfoMessage:new{
text = _("Login failed: ") .. (response.msg or "Unknown error"),
timeout = 3,
})
end
end
function SyncAuth:logout(settings, menu)
if settings.access_token then
local client = self:getSupabaseAuthClient(settings, "")
if client then
client:sign_out(settings.access_token, function(success, _response)
logger.dbg("ReadestSync: Sign out result:", success)
end)
end
end
settings.access_token = nil
settings.refresh_token = nil
settings.expires_at = nil
settings.expires_in = nil
G_reader_settings:saveSetting("readest_sync", settings)
if menu then
menu:updateItems()
end
UIManager:show(InfoMessage:new{
text = _("Logged out from Readest Sync"),
timeout = 2,
})
end
return SyncAuth
+265
View File
@@ -0,0 +1,265 @@
local Event = require("ui/event")
local InfoMessage = require("ui/widget/infomessage")
local UIManager = require("ui/uimanager")
local logger = require("logger")
local util = require("util")
local sha2 = require("ffi/sha2")
local _ = require("gettext")
local SyncConfig = {}
local function normalizeIdentifier(identifier)
if identifier:match("urn:") then
return identifier:match("([^:]+)$")
elseif identifier:match(":") then
return identifier:match("^[^:]+:(.+)$")
end
return identifier
end
local function normalizeAuthor(author)
author = author:gsub("^%s*(.-)%s*$", "%1")
return author
end
function SyncConfig:generateMetadataHash(ui)
local doc_props = ui.doc_settings:readSetting("doc_props") or {}
local title = doc_props.title or ''
if title == '' then
local _doc_path, filename = util.splitFilePathName(ui.doc_settings:readSetting("doc_path") or '')
local basename, _suffix = util.splitFileNameSuffix(filename)
title = basename or ''
end
local authors = doc_props.authors or ''
if authors:find("\n") then
authors = util.splitToArray(authors, "\n")
for i, author in ipairs(authors) do
authors[i] = normalizeAuthor(author)
end
authors = table.concat(authors, ",")
else
authors = normalizeAuthor(authors)
end
local identifiers = doc_props.identifiers or ''
if identifiers:find("\n") then
local list = util.splitToArray(identifiers, "\n")
local normalized = {}
local priorities = { "uuid", "calibre", "isbn" }
local preferred = nil
for i, id in ipairs(list) do
normalized[i] = normalizeIdentifier(id)
local candidate = id:lower()
for _, p in ipairs(priorities) do
if candidate:find(p, 1, true) then
preferred = normalized[i]
break
end
end
end
if preferred then
identifiers = preferred
else
identifiers = table.concat(normalized, ",")
end
else
identifiers = normalizeIdentifier(identifiers)
end
local doc_meta = title .. "|" .. authors .. "|" .. identifiers
return sha2.md5(doc_meta)
end
function SyncConfig:getMetaHash(ui)
local doc_readest_sync = ui.doc_settings:readSetting("readest_sync") or {}
local meta_hash = doc_readest_sync.meta_hash_v1
if not meta_hash then
meta_hash = self:generateMetadataHash(ui)
doc_readest_sync.meta_hash_v1 = meta_hash
ui.doc_settings:saveSetting("readest_sync", doc_readest_sync)
end
return meta_hash
end
function SyncConfig:getDocumentIdentifier(ui)
return ui.doc_settings:readSetting("partial_md5_checksum")
end
function SyncConfig:getCurrentBookConfig(ui)
local book_hash = self:getDocumentIdentifier(ui)
local meta_hash = self:getMetaHash(ui)
if not book_hash or not meta_hash then
UIManager:show(InfoMessage:new{
text = _("Cannot identify the current book"),
timeout = 2,
})
return nil
end
local config = {
bookHash = book_hash,
metaHash = meta_hash,
progress = "",
xpointer = "",
updatedAt = os.time() * 1000,
}
local current_page = ui:getCurrentPage()
local page_count = ui.document:getPageCount()
config.progress = {current_page, page_count}
if not ui.document.info.has_pages then
config.xpointer = ui.rolling:getLastProgress()
end
return config
end
function SyncConfig:applyBookConfig(ui, config)
logger.dbg("ReadestSync: Applying book config:", config)
local xpointer = config.xpointer
local progress = config.progress
local has_pages = ui.document.info.has_pages
local progress_pattern = "^%[(%d+),(%d+)%]$"
if has_pages and progress then
local page, _total_pages = progress:match(progress_pattern)
local current_page = ui:getCurrentPage()
local new_page = tonumber(page)
if new_page > current_page then
ui.link:addCurrentLocationToStack()
ui:handleEvent(Event:new("GotoPage", new_page))
self:showSyncedMessage()
end
end
if not has_pages and xpointer then
local last_xpointer = ui.rolling:getLastProgress()
local working_xpointer = xpointer
local cmp_result = ui.document:compareXPointers(last_xpointer, working_xpointer)
while cmp_result == nil and working_xpointer do
local last_slash_pos = working_xpointer:match("^.*()/")
if last_slash_pos and last_slash_pos > 1 then
working_xpointer = working_xpointer:sub(1, last_slash_pos - 1)
cmp_result = ui.document:compareXPointers(last_xpointer, working_xpointer)
else
break
end
end
if cmp_result > 0 then
ui.link:addCurrentLocationToStack()
ui:handleEvent(Event:new("GotoXPointer", working_xpointer))
self:showSyncedMessage()
end
end
end
function SyncConfig:showSyncedMessage()
UIManager:show(InfoMessage:new{
text = _("Progress has been synchronized."),
timeout = 3,
})
end
function SyncConfig:push(ui, settings, client, interactive, last_sync_timestamp)
local config = self:getCurrentBookConfig(ui)
if not config then return last_sync_timestamp end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pushing book config..."),
timeout = 1,
})
end
local payload = {
books = {},
notes = {},
configs = { config },
}
client:pushChanges(
payload,
function(success, _response)
if interactive then
if success then
UIManager:show(InfoMessage:new{
text = _("Book config pushed successfully"),
timeout = 2,
})
else
UIManager:show(InfoMessage:new{
text = _("Failed to push book config"),
timeout = 2,
})
end
end
end
)
if not interactive then
return os.time()
end
return last_sync_timestamp
end
function SyncConfig:pull(ui, settings, client, book_hash, meta_hash, interactive, logout_fn)
if interactive then
UIManager:show(InfoMessage:new{
text = _("Pulling book config..."),
timeout = 1,
})
end
client:pullChanges(
{
since = 0,
type = "configs",
book = book_hash,
meta_hash = meta_hash,
},
function(success, response)
if not success then
if response and response.error == "Not authenticated" then
if interactive then
UIManager:show(InfoMessage:new{
text = _("Authentication failed, please login again"),
timeout = 2,
})
end
if logout_fn then logout_fn() end
return
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("Failed to pull book config"),
timeout = 2,
})
end
return
end
local data = response.configs
if data and #data > 0 then
local config = data[1]
if config then
self:applyBookConfig(ui, config)
if interactive then
UIManager:show(InfoMessage:new{
text = _("Book config synchronized"),
timeout = 2,
})
end
return
end
end
if interactive then
UIManager:show(InfoMessage:new{
text = _("No saved config found for this book"),
timeout = 2,
})
end
end
)
end
return SyncConfig
+2
View File
@@ -59,6 +59,8 @@ CREATE TABLE public.book_notes (
id text NOT NULL,
type text NULL,
cfi text NULL,
xpointer0 text NULL,
xpointer1 text NULL,
text text NULL,
style text NULL,
color text NULL,