From b8ddb5475e98036af6e69cdb9ebba7bbbcd8c1c0 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Wed, 1 Apr 2026 21:35:48 +0800 Subject: [PATCH] feat(sync): add full sync option for annotations in koplugin, closes #3710 (#3718) Add "Full sync all annotations" menu item that pushes and pulls all annotations regardless of the last sync timestamp, enabling users to sync old highlights that were created before the plugin was installed. Changes: - Add full_sync parameter to push/pull that bypasses timestamp filter and uses since=0 for pulling all server annotations - Deduplicate by annotation ID alongside position-based dedup - Store server ID on pulled annotations and reuse it when pushing - Parse ISO 8601 timestamps from server to preserve original created/updated dates instead of using current time - Resolve KOReader page numbers from xpointers via getPageFromXPointer - Resolve Readest page numbers from CFI via getCFIProgress on pull Co-authored-by: Claude Opus 4.6 (1M context) --- .../src/app/reader/hooks/useNotesSync.ts | 13 +++- apps/readest.koplugin/main.lua | 27 +++++-- apps/readest.koplugin/syncannotations.lua | 77 +++++++++++++++---- 3 files changed, 94 insertions(+), 23 deletions(-) diff --git a/apps/readest-app/src/app/reader/hooks/useNotesSync.ts b/apps/readest-app/src/app/reader/hooks/useNotesSync.ts index f405138f..a93a2342 100644 --- a/apps/readest-app/src/app/reader/hooks/useNotesSync.ts +++ b/apps/readest-app/src/app/reader/hooks/useNotesSync.ts @@ -86,7 +86,18 @@ export const useNotesSync = (bookKey: string) => { ); } if (cfi) { - converted.push({ ...note, cfi, updatedAt: Date.now() }); + let page = note.page; + if (view) { + try { + const progress = await view.getCFIProgress(cfi); + if (progress) { + page = progress.location.current + 1; + } + } catch { + // Page resolution failed — keep original page + } + } + converted.push({ ...note, cfi, page, updatedAt: Date.now() }); } } catch { // Conversion failed — discard note diff --git a/apps/readest.koplugin/main.lua b/apps/readest.koplugin/main.lua index 67dc92fc..a9e56d5e 100644 --- a/apps/readest.koplugin/main.lua +++ b/apps/readest.koplugin/main.lua @@ -137,6 +137,15 @@ function ReadestSync:addToMainMenu(menu_items) callback = function() self:pullBookNotes(true) end, + }, + { + text = _("Full sync all annotations"), + enabled_func = function() + return self.settings.access_token ~= nil and self.ui.document ~= nil + end, + callback = function() + self:fullSyncBookNotes() + end, separator = true, }, { @@ -227,22 +236,22 @@ end -- ── Annotation sync ──────────────────────────────────────────────── -function ReadestSync:pushBookNotes(interactive) - if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookNotes(interactive) end) then +function ReadestSync:pushBookNotes(interactive, full_sync) + if interactive and NetworkMgr:willRerunWhenOnline(function() self:pushBookNotes(interactive, full_sync) end) then return end local client = self:ensureClient(interactive) if not client then return end - SyncAnnotations:push(self.ui, self.settings, client, interactive) + SyncAnnotations:push(self.ui, self.settings, client, interactive, full_sync) end -function ReadestSync:pullBookNotes(interactive) +function ReadestSync:pullBookNotes(interactive, full_sync) local book_hash, meta_hash = self:getBookIdentifiers() if not book_hash or not meta_hash then return end - if NetworkMgr:willRerunWhenOnline(function() self:pullBookNotes(interactive) end) then + if NetworkMgr:willRerunWhenOnline(function() self:pullBookNotes(interactive, full_sync) end) then return end @@ -250,10 +259,16 @@ function ReadestSync:pullBookNotes(interactive) if not client then return end SyncAnnotations:pull( - self.ui, self.settings, client, book_hash, meta_hash, self.dialog, interactive + self.ui, self.settings, client, book_hash, meta_hash, self.dialog, interactive, full_sync ) end +function ReadestSync:fullSyncBookNotes() + -- Push all annotations first, then pull all + self:pushBookNotes(true, true) + self:pullBookNotes(true, true) +end + -- ── Event handlers ───────────────────────────────────────────────── function ReadestSync:onReadestSyncToggleAutoSync(toggle) diff --git a/apps/readest.koplugin/syncannotations.lua b/apps/readest.koplugin/syncannotations.lua index c23969e0..e71e5b22 100644 --- a/apps/readest.koplugin/syncannotations.lua +++ b/apps/readest.koplugin/syncannotations.lua @@ -47,16 +47,28 @@ function SyncAnnotations:parseDatetimeToMs(dt) return os.time() * 1000 end +function SyncAnnotations:parseISODatetime(dt) + if not dt then return os.time() end + local y, m, d, h, min, s = dt:match("(%d+)-(%d+)-(%d+)T(%d+):(%d+):(%d+)") + if y then + return os.time({ + year = tonumber(y), month = tonumber(m), day = tonumber(d), + hour = tonumber(h), min = tonumber(min), sec = tonumber(s), + }) + end + return os.time() +end + function SyncAnnotations:generateNoteId(book_hash, note_type, pos0, pos1) local raw = "ko:" .. book_hash .. ":" .. note_type .. ":" .. (pos0 or "") .. ":" .. (pos1 or "") return sha2.md5(raw):sub(1, 7) end -function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash) +function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash, full_sync) local annotations = ui.annotation and ui.annotation.annotations if not annotations then return {} end - local last_sync = settings.last_notes_sync_at or 0 + local last_sync = full_sync and 0 or (settings.last_notes_sync_at or 0) local notes = {} for _, item in ipairs(annotations) do @@ -79,7 +91,7 @@ function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash) style = "squiggly" end - local id = self:generateNoteId(book_hash, "annotation", tostring(pos0), pos1 and tostring(pos1)) + local id = item.id or self:generateNoteId(book_hash, "annotation", tostring(pos0), pos1 and tostring(pos1)) table.insert(notes, { bookHash = book_hash, metaHash = meta_hash, @@ -98,7 +110,7 @@ function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash) elseif not item.drawer and type(item.page) == "string" then -- Bookmark: no drawer, position in page field (xpointer string) local page_xp = item.page - local id = self:generateNoteId(book_hash, "bookmark", page_xp) + local id = item.id or self:generateNoteId(book_hash, "bookmark", page_xp) table.insert(notes, { bookHash = book_hash, metaHash = meta_hash, @@ -117,13 +129,13 @@ function SyncAnnotations:getAnnotations(ui, settings, book_hash, meta_hash) return notes end -function SyncAnnotations:push(ui, settings, client, interactive) +function SyncAnnotations:push(ui, settings, client, interactive, full_sync) local book_hash = ui.doc_settings:readSetting("partial_md5_checksum") local meta_hash = ui.doc_settings:readSetting("readest_sync") or {} meta_hash = meta_hash.meta_hash_v1 if not book_hash or not meta_hash then return end - local annotations = self:getAnnotations(ui, settings, book_hash, meta_hash) + local annotations = self:getAnnotations(ui, settings, book_hash, meta_hash, full_sync) if #annotations == 0 then if interactive then UIManager:show(InfoMessage:new{ @@ -172,7 +184,7 @@ function SyncAnnotations:push(ui, settings, client, interactive) ) end -function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog, interactive) +function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog, interactive, full_sync) if ui.document.info.has_pages then if interactive then UIManager:show(InfoMessage:new{ @@ -185,14 +197,14 @@ function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog if interactive then UIManager:show(InfoMessage:new{ - text = _("Pulling annotations..."), + text = full_sync and _("Full sync: pulling all annotations...") or _("Pulling annotations..."), timeout = 1, }) end client:pullChanges( { - since = settings.last_notes_sync_at or 0, + since = full_sync and 0 or (settings.last_notes_sync_at or 0), type = "notes", book = book_hash, meta_hash = meta_hash, @@ -219,19 +231,37 @@ function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog return end - logger.dbg("ReadestSync: Pulled annotations from sync:", data) + logger.dbg("ReadestSync: Pulled annotations from sync:", #data) local annotation_mgr = ui.annotation if not annotation_mgr then return end - -- Build dedup sets: annotations by pos0|pos1, bookmarks by page xpointer + -- Build dedup sets: by ID, by pos0|pos1 for annotations, by page xpointer for bookmarks + local existing_ids = {} local existing_annotations = {} local existing_bookmarks = {} for _, item in ipairs(annotation_mgr.annotations) do + -- Use stored id if available + if item.id then + existing_ids[item.id] = true + end if item.drawer then - local key = tostring(item.pos0) .. "|" .. tostring(item.pos1 or "") + local pos0 = item.pos0 + local pos1 = item.pos1 + if type(pos0) == "table" then pos0 = nil end + if type(pos1) == "table" then pos1 = nil end + local key = tostring(pos0) .. "|" .. tostring(pos1 or "") existing_annotations[key] = true + -- Also generate ID for annotations without id + if not item.id and pos0 then + local id = self:generateNoteId(book_hash, "annotation", tostring(pos0), pos1 and tostring(pos1)) + existing_ids[id] = true + end elseif type(item.page) == "string" then existing_bookmarks[item.page] = true + if not item.id then + local id = self:generateNoteId(book_hash, "bookmark", item.page) + existing_ids[id] = true + end end end @@ -244,18 +274,31 @@ function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog local xp0 = note.xpointer0 if not xp0 then goto continue end + -- Deduplicate by server-provided ID + if note.id and existing_ids[note.id] then goto continue end + local note_type = note.type local item + local created = self:parseISODatetime(note.created_at) + local updated = self:parseISODatetime(note.updated_at) or created + local datetime_str = os.date("%Y-%m-%d %H:%M:%S", created) + local datetime_updated_str = os.date("%Y-%m-%d %H:%M:%S", updated) + + -- Resolve KOReader page number from xpointer + local pageno = ui.document:getPageFromXPointer(xp0) or note.page + if note_type == "bookmark" then if existing_bookmarks[xp0] then goto continue end item = { + id = note.id, page = xp0, text = note.text or "", note = note.note or "", - pageno = note.page, - datetime = os.date("%Y-%m-%d %H:%M:%S"), + pageno = pageno, + datetime = datetime_str, + datetime_updated = datetime_updated_str, } existing_bookmarks[xp0] = true else @@ -271,6 +314,7 @@ function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog end item = { + id = note.id, pos0 = xp0, pos1 = xp1 or xp0, page = xp0, @@ -278,8 +322,9 @@ function SyncAnnotations:pull(ui, settings, client, book_hash, meta_hash, dialog note = note.note or "", drawer = drawer, color = READEST_TO_KO_COLOR[note.color] or "yellow", - pageno = note.page, - datetime = os.date("%Y-%m-%d %H:%M:%S"), + pageno = pageno, + datetime = datetime_str, + datetime_updated = datetime_updated_str, } existing_annotations[key] = true end