diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0bb5bade..e035e0b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -118,6 +118,33 @@ jobs: echo "Uploading ${plugin_zip} to GitHub release" gh release upload ${{ needs.get-release.outputs.release_tag }} ${plugin_zip} --clobber + build-calibre-plugin: + needs: get-release + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: create calibre plugin zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version=${{ needs.get-release.outputs.release_version }} + + # Stamp PLUGIN_VERSION in __init__.py with the release version + # (from apps/readest-app/package.json, mirroring the koplugin's + # _meta.lua stamp above); the committed value is a dev placeholder. + version_tuple=$(echo "${version}" | awk -F. '{printf "(%d, %d, %d)", $1, $2, $3}') + perl -i -pe "s/^PLUGIN_VERSION = \(\d+, \d+, \d+\)/PLUGIN_VERSION = ${version_tuple}/" \ + apps/readest-calibre-plugin/__init__.py + + make -C apps/readest-calibre-plugin zip + plugin_zip="apps/readest-calibre-plugin/dist/Readest-${version}.calibre-plugin.zip" + + echo "Uploading ${plugin_zip} to GitHub release" + gh release upload ${{ needs.get-release.outputs.release_tag }} ${plugin_zip} --clobber + build-tauri: needs: get-release permissions: diff --git a/apps/readest-app/.claude/memory/MEMORY.md b/apps/readest-app/.claude/memory/MEMORY.md index 5842581d..2a14bd03 100644 --- a/apps/readest-app/.claude/memory/MEMORY.md +++ b/apps/readest-app/.claude/memory/MEMORY.md @@ -114,6 +114,7 @@ - OPDS: [Firefox strict-XML #4479](opds-firefox-strict-xml-4479.md) junk after `` slice to last close; [JSON search #4502](opds2-json-search-4502.md) expand `{?query}` BEFORE resolveURL; [HTML desc #4503](opds-html-description-4503.md) decode-once+sanitize; [self-link meta #4749](opds-self-link-metadata-4749.md) `rel:self` deref; [popular dedup #4782](opds-popular-catalog-dedup-4782.md) - [D-pad Navigation](dpad-navigation.md) — Android TV remote / arrow-key nav - [koplugin cover upload (#4374)](koplugin-cover-upload.md) — `extractLocalCover` via `getCoverImage` +- [Calibre plugin push #4863](calibre-plugin-push-4863.md) apps/readest-calibre-plugin; OAuth localhost relay + /sync carry-over ## Library Fixes - [Book action platform surfaces](book-actions-platform-surfaces.md) — context menu Tauri-desktop-only; cross-platform - [Tauri menu append race (#4389)](tauri-menu-append-race-4389.md) — single `await Menu.new({ items })` diff --git a/apps/readest-app/.claude/memory/calibre-plugin-push-4863.md b/apps/readest-app/.claude/memory/calibre-plugin-push-4863.md new file mode 100644 index 00000000..beac2b7f --- /dev/null +++ b/apps/readest-app/.claude/memory/calibre-plugin-push-4863.md @@ -0,0 +1,23 @@ +--- +name: calibre-plugin-push-4863 +description: "readest-calibre-plugin (#4863) pushes calibre books+metadata to Readest cloud; key protocol facts (OAuth localhost relay, /sync explicit-null carry-over)" +metadata: + node_type: memory + type: project + originSessionId: 5d4d83a0-0aee-4200-852f-555df5243bed +--- + +`apps/readest-calibre-plugin/` (added 2026-07-04, commit 29a42853d on dev) implements #4863: calibre GUI plugin pushing selected books + metadata into the Readest cloud, modeled on BookFusion's plugin. + +Design decisions and hard-won protocol facts: +- **Identity**: `Book.hash` = partial MD5 (KOReader algorithm; JS `1024 << -2` wraps to 0, so offsets are 0, 1024, 4096, ... 1024<<20). metaHash = `md5(NFC("title|authors,|ids,"))`, preferred id scheme uuid > calibre > isbn; Python impl verified byte-identical to `js-md5` output. +- **OPF embedding + uuid dedup** (v2, per maintainer request): metadata IS embedded into a temp copy at upload (`calibre.ebooks.metadata.meta.set_metadata` — deterministic for EPUB, writes custom columns as `calibre:user_metadata`). Dedup keys: calibre uuid in row `metadata.identifier` (survives byte changes) + `metadata.calibreSourceHash` = raw library-file partialMD5 (change detection, no local state; v1 rows fall back to `book_hash` which equals the raw hash). File changed → replace flow: upload new blob, push new row (carry-over) + tombstone old in one /sync POST, best-effort delete old cloud files. Metadata-only edit → row update, no re-upload (embedded OPF goes stale until next file upload). +- **POST /sync explicit-nulls absent fields** (transformBookToDB) — updates must carry over `uploadedAt`, `groupId/Name`, `progress`, `readingStatus*`, `coverHash` from the pulled server row (`wire.py::merge_for_push`); same lesson as koplugin syncbooks.lua. +- **Upload key** `Readest/Books/{hash}/{hash}.{ext}`; app's `{title}.{ext}` downloads resolve via download.ts hash+extension fallback. cover.png stores *original* bytes (app never converts formats, bookService.ts:568), so calibre's cover.jpg bytes upload as-is; coverHash = partialMD5 of those bytes. +- **OAuth from a non-app client works**: `{supabase}/auth/v1/authorize?provider=X&redirect_to=http://localhost:PORT` is whitelisted (readest-app's Flatpak/custom-OAuth production path uses it). Tokens arrive in the URL *fragment*; serve a page whose JS relays `location.hash` to `/callback?...` (tauri-plugin-oauth trick). Implemented in `oauth.py`. +- Pure modules (`api.py`, `wire.py`, `oauth.py`) are calibre-free; `make test` runs 56 unittests; `make zip` builds; smoke-test inside calibre with `calibre-debug -c` after `from calibre.customize.ui import find_plugin` (initializes the `calibre_plugins` namespace). + +- **Release packaging** (PR #4918): `build-calibre-plugin` job in release.yml mirrors the koplugin job; perl-stamps `PLUGIN_VERSION` from readest-app package.json, `make zip` → `Readest-.calibre-plugin.zip` release asset. Committed version stays the (0, 1, 0) dev placeholder. +- **Pushing workflow files**: gh's OAuth token lacks `workflow` scope (HTTPS push of .github/workflows/* rejected); SSH push works (transient hangs — retry with ConnectTimeout/ServerAliveInterval). + +Related: [[koplugin-cover-upload]], [[grimmory-native-sync]], [[ci-pr-delivery-and-push]] diff --git a/apps/readest-calibre-plugin/.gitignore b/apps/readest-calibre-plugin/.gitignore new file mode 100644 index 00000000..2ead01d0 --- /dev/null +++ b/apps/readest-calibre-plugin/.gitignore @@ -0,0 +1,2 @@ +dist/ +__pycache__/ diff --git a/apps/readest-calibre-plugin/Makefile b/apps/readest-calibre-plugin/Makefile new file mode 100644 index 00000000..5eedd42d --- /dev/null +++ b/apps/readest-calibre-plugin/Makefile @@ -0,0 +1,28 @@ +NAME = Readest +VERSION = $(shell python3 -c "import re; \ + m = re.search(r'PLUGIN_VERSION = \((\d+), (\d+), (\d+)\)', open('__init__.py').read()); \ + print('.'.join(m.groups()))") +ZIP = dist/$(NAME)-$(VERSION).calibre-plugin.zip + +FILES = __init__.py api.py config.py dialogs.py oauth.py ui.py wire.py worker.py \ + plugin-import-name-readest.txt images/icon.png + +.PHONY: zip test install clean + +zip: $(ZIP) + +$(ZIP): $(FILES) + mkdir -p dist + rm -f $(ZIP) + zip -q $(ZIP) $(FILES) + @echo Built $(ZIP) + +test: + python3 -m unittest discover -s tests + +# Load the plugin into a locally installed calibre for development. +install: zip + calibre-customize -a $(ZIP) + +clean: + rm -rf dist diff --git a/apps/readest-calibre-plugin/README.md b/apps/readest-calibre-plugin/README.md new file mode 100644 index 00000000..cd39315b --- /dev/null +++ b/apps/readest-calibre-plugin/README.md @@ -0,0 +1,98 @@ +# Readest calibre plugin + +A [calibre](https://calibre-ebook.com) GUI plugin that pushes selected books — +with their metadata — into your [Readest](https://readest.com) cloud library. +Re-pushing a book updates its existing entry instead of creating a duplicate, +so you can edit metadata in calibre and re-send it any time. + +Implements [readest/readest#4863](https://github.com/readest/readest/issues/4863). + +## Features + +- **Selective, manual push**: select books in calibre, click *Readest* in the + toolbar. Nothing syncs in the background. +- **Metadata included and embedded**: title, authors, series, tags, + description, publisher, language, identifiers — and optionally calibre + custom columns. Metadata is written both to the Readest library entry + (custom columns under `customColumns`) and into the uploaded file's OPF + (calibre's own embedding, so custom columns travel as + `calibre:user_metadata`). Your calibre library files are never modified — + embedding happens on a temporary copy. +- **Update on re-push**: books already in your Readest library are recognized + by their calibre uuid and only changed entries are rewritten; unchanged + books are skipped, and a changed file replaces the old entry instead of + duplicating it. Reading progress, grouping and reading status in Readest + are preserved. +- **Per-book status report**: uploaded / updated / up to date / failed, with a + storage-quota check (the push stops cleanly when your quota is exhausted). +- **Login like the apps**: email + password, or browser sign-in with Google, + Apple, GitHub or Discord (OAuth via a temporary localhost callback, the same + flow the desktop app uses). + +## Install + +Download `Readest-.calibre-plugin.zip` from the +[latest release](https://github.com/readest/readest/releases/latest), or build +it yourself: + +```sh +make zip # builds dist/Readest-.calibre-plugin.zip +calibre-customize -a dist/Readest-*.calibre-plugin.zip # or: make install +``` + +Or in calibre: *Preferences → Plugins → Load plugin from file*, then restart +calibre and add the *Readest* button to the main toolbar if it is not visible. + +Release zips are versioned from `apps/readest-app/package.json` by the release +workflow, which stamps `PLUGIN_VERSION` in `__init__.py` before zipping; the +version committed in git is a development placeholder. + +## Usage + +1. Click the *Readest* toolbar button menu → *Log in to Readest…* +2. Select the books to push (any number). +3. Click the *Readest* button (or menu → *Push selected books to Readest*). + +For each book the best Readest-supported format is pushed, preferring +`EPUB > PDF > AZW3 > MOBI > AZW > FB2 > FBZ > CBZ > TXT > MD`. + +## How updates and duplicates work + +At upload time the plugin embeds your current calibre metadata (including +custom columns) into a temporary copy of the book file, so the copy in your +Readest library is self-describing. Pushed books are then tracked by two keys: + +- the **calibre book uuid**, carried in the entry's metadata + (`urn:uuid:...`) — this recognizes "this calibre book is already in + Readest" across pushes, even when the file bytes change; +- a **fingerprint of the raw calibre file** (`calibreSourceHash`), which + detects whether the file itself changed since the last push — no local + state, so it works from any machine. + +Re-push behavior: + +- **Nothing changed** → skipped. +- **Metadata edited** → the Readest library entry is updated in place; the + file is not re-uploaded (its embedded OPF keeps the metadata from its + upload time). +- **File changed** (e.g. re-converted) → the new file is uploaded with fresh + embedded metadata and *replaces* the old entry: reading status, grouping, + progress and the library date carry over, the old entry is removed and its + cloud files are deleted. Notes and reading positions re-attach when + title/authors are unchanged (Readest matches book versions by metadata + identity). +- Books pushed by older plugin versions (or dragged into Readest manually) + are recognized too: their entry hash doubles as the raw-file fingerprint. + +## Development + +Pure-logic modules (`api.py`, `wire.py`, `oauth.py`) have no calibre or Qt +dependencies and are covered by unit tests: + +```sh +make test # python3 -m unittest discover -s tests +``` + +The wire protocol mirrors what the Readest apps and `readest.koplugin` use: +Supabase auth (`/auth/v1`), `GET/POST /api/sync` for library rows, and +`POST /api/storage/upload` + presigned PUT for file blobs. diff --git a/apps/readest-calibre-plugin/__init__.py b/apps/readest-calibre-plugin/__init__.py new file mode 100644 index 00000000..7fb5bfcd --- /dev/null +++ b/apps/readest-calibre-plugin/__init__.py @@ -0,0 +1,31 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +from calibre.customize import InterfaceActionBase + +PLUGIN_VERSION = (0, 1, 0) + + +class ReadestPlugin(InterfaceActionBase): + name = 'Readest Sync' + description = ( + 'Push selected books and their metadata into your Readest cloud library. ' + 'Re-pushing a book updates its existing entry instead of creating a duplicate.' + ) + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Bilingify LLC' + version = PLUGIN_VERSION + minimum_calibre_version = (6, 0, 0) + + actual_plugin = 'calibre_plugins.readest.ui:ReadestInterfacePlugin' + + def is_customizable(self): + return True + + def config_widget(self): + from calibre_plugins.readest.config import ConfigWidget + + return ConfigWidget() + + def save_settings(self, config_widget): + config_widget.save_settings() diff --git a/apps/readest-calibre-plugin/api.py b/apps/readest-calibre-plugin/api.py new file mode 100644 index 00000000..a25314c8 --- /dev/null +++ b/apps/readest-calibre-plugin/api.py @@ -0,0 +1,332 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +"""HTTP client + content hashes for the Readest cloud API. + +Standard-library only (no calibre / Qt imports) so it can be unit-tested +outside calibre. The endpoints and shapes mirror the ones already consumed by +readest.koplugin (readest-sync-api.json / supabase-auth-api.json) and served +by apps/readest-app/src/pages/api/. +""" + +import hashlib +import json +import time +import unicodedata +import urllib.error +import urllib.parse +import urllib.request + +DEFAULT_API_BASE = 'https://web.readest.com/api' +DEFAULT_SUPABASE_URL = 'https://readest.supabase.co' +# Public anon key, same one readest.koplugin ships (base64-encoded in main.lua). +DEFAULT_ANON_KEY = ( + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.' + 'eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZic3l4ZnVzampxZHhranFseXNjIiwicm9sZSI6ImFub24iLCJpYXQiOjE3' + 'MzQxMjM2NzEsImV4cCI6MjA0OTY5OTY3MX0.' + '3U5Uqaou_1SgrVe1eo9rApc0uKjqhpQdUXhvwUHmUfg' +) + +TIMEOUT = 30 +UPLOAD_TIMEOUT = 600 + + +class ReadestAPIError(Exception): + def __init__(self, message, status=None): + super().__init__(message) + self.status = status + + +class AuthRequiredError(ReadestAPIError): + """No valid session — the user must log in.""" + + +class QuotaExceededError(ReadestAPIError): + """The server rejected an upload for insufficient storage quota.""" + + +# --------------------------------------------------------------------------- +# Content hashes +# --------------------------------------------------------------------------- + + +def _partial_md5_ranges(size): + """Chunk ranges of apps/readest-app/src/utils/md5.ts::partialMD5. + + 1024-byte chunks at offsets 0, 1024, 4096, ..., 1024 << 20 (the JS loop + runs i in -1..10; `1024 << -2` wraps to 0 under JS 32-bit shift). + """ + ranges = [] + for i in range(-1, 11): + offset = 0 if i == -1 else 1024 << (2 * i) + start = min(size, offset) + if start >= size: + break + ranges.append((start, min(start + 1024, size))) + return ranges + + +def partial_md5(file_or_path, size=None): + """Readest's Book.hash: partial MD5 of a file (KOReader-compatible).""" + if isinstance(file_or_path, str): + with open(file_or_path, 'rb') as f: + f.seek(0, 2) + return partial_md5(f, f.tell()) + f = file_or_path + hasher = hashlib.md5() + for start, end in _partial_md5_ranges(size): + f.seek(start) + hasher.update(f.read(end - start)) + return hasher.hexdigest() + + +def partial_md5_bytes(data): + hasher = hashlib.md5() + for start, end in _partial_md5_ranges(len(data)): + hasher.update(data[start:end]) + return hasher.hexdigest() + + +def _normalize_identifier(identifier): + # Mirrors utils/book.ts::normalizeIdentifier. + if 'urn:' in identifier: + return identifier.rsplit(':', 1)[-1] + if ':' in identifier: + return identifier.split(':', 1)[1] + return identifier + + +def _identifiers_list(identifiers): + # Mirrors utils/book.ts::getIdentifiersList / getPreferredIdentifier. + if not identifiers: + return [] + for scheme in ('uuid', 'calibre', 'isbn'): + for identifier in identifiers: + if scheme in identifier.lower(): + return [_normalize_identifier(identifier)] + return [_normalize_identifier(i) for i in identifiers if i] + + +def meta_hash(title, authors, identifiers): + """Readest's Book.metaHash: md5 over "title|authors|identifiers" (NFC). + + `identifiers` are raw identifier strings, scheme-prefixed where known + (e.g. "urn:uuid:...", "isbn:..."). + """ + source = '%s|%s|%s' % ( + title or '', + ','.join(authors or []), + ','.join(_identifiers_list(identifiers)), + ) + return hashlib.md5(unicodedata.normalize('NFC', source).encode('utf-8')).hexdigest() + + +# --------------------------------------------------------------------------- +# HTTP client +# --------------------------------------------------------------------------- + + +def _default_transport(request, timeout): + try: + with urllib.request.urlopen(request, timeout=timeout) as res: + return res.status, res.read() + except urllib.error.HTTPError as err: + return err.code, err.read() + + +class ReadestClient: + """Supabase auth + Readest sync/storage API client. + + tokens: dict with access_token / refresh_token / expires_at (epoch s) / + expires_in (s), persisted through the on_tokens callback whenever they + change. `transport(request, timeout) -> (status, body_bytes)` is + injectable for tests. + """ + + def __init__( + self, + api_base=DEFAULT_API_BASE, + supabase_url=DEFAULT_SUPABASE_URL, + anon_key=DEFAULT_ANON_KEY, + tokens=None, + on_tokens=None, + transport=None, + ): + self.api_base = api_base.rstrip('/') + self.supabase_url = supabase_url.rstrip('/') + self.anon_key = anon_key + self.tokens = dict(tokens) if tokens else None + self.on_tokens = on_tokens + self.transport = transport or _default_transport + + # -- plumbing ----------------------------------------------------------- + + def _request(self, method, url, headers, body=None, timeout=TIMEOUT): + data = None + if body is not None: + data = json.dumps(body).encode('utf-8') + headers = dict(headers, **{'Content-Type': 'application/json'}) + request = urllib.request.Request(url, data=data, headers=headers, method=method) + status, payload = self.transport(request, timeout) + parsed = None + if payload: + try: + parsed = json.loads(payload.decode('utf-8')) + except (ValueError, UnicodeDecodeError): + parsed = None + return status, parsed, payload + + @staticmethod + def _error_message(parsed, payload, fallback): + if isinstance(parsed, dict): + for key in ('error_description', 'msg', 'message', 'error'): + value = parsed.get(key) + if isinstance(value, str) and value: + return value + if payload: + return payload.decode('utf-8', 'replace')[:200] + return fallback + + # -- Supabase auth ------------------------------------------------------ + + def _auth_headers(self): + return {'apikey': self.anon_key, 'Accept': 'application/json'} + + def _store_tokens(self, body): + self.tokens = { + 'access_token': body.get('access_token'), + 'refresh_token': body.get('refresh_token'), + 'expires_at': body.get('expires_at'), + 'expires_in': body.get('expires_in'), + } + if self.on_tokens: + self.on_tokens(dict(self.tokens)) + + def sign_in_password(self, email, password): + status, parsed, payload = self._request( + 'POST', + self.supabase_url + '/auth/v1/token?grant_type=password', + self._auth_headers(), + body={'email': email, 'password': password}, + ) + if status != 200 or not isinstance(parsed, dict): + raise ReadestAPIError(self._error_message(parsed, payload, 'Login failed'), status) + self._store_tokens(parsed) + return parsed.get('user') or {} + + def set_session(self, tokens): + """Adopt tokens obtained externally (browser OAuth callback).""" + self._store_tokens(tokens) + + def refresh(self): + refresh_token = self.tokens and self.tokens.get('refresh_token') + if not refresh_token: + raise AuthRequiredError('Not logged in') + status, parsed, payload = self._request( + 'POST', + self.supabase_url + '/auth/v1/token?grant_type=refresh_token', + self._auth_headers(), + body={'refresh_token': refresh_token}, + ) + if status != 200 or not isinstance(parsed, dict): + raise AuthRequiredError( + self._error_message(parsed, payload, 'Session expired, please log in again'), + status, + ) + self._store_tokens(parsed) + + def ensure_fresh_token(self): + # Mirrors readest_syncauth.lua: refresh once less than half the TTL + # remains; a token past its final minute is unusable without refresh. + if not self.tokens or not self.tokens.get('access_token'): + raise AuthRequiredError('Not logged in') + expires_at = self.tokens.get('expires_at') or 0 + expires_in = self.tokens.get('expires_in') or 3600 + if expires_at < time.time() + max(60, expires_in / 2): + self.refresh() + + def get_user(self): + self.ensure_fresh_token() + headers = dict(self._auth_headers()) + headers['Authorization'] = 'Bearer ' + self.tokens['access_token'] + status, parsed, payload = self._request( + 'GET', self.supabase_url + '/auth/v1/user', headers + ) + if status != 200 or not isinstance(parsed, dict): + raise AuthRequiredError(self._error_message(parsed, payload, 'Not logged in'), status) + return parsed + + def sign_out(self): + if not self.tokens or not self.tokens.get('access_token'): + return + headers = dict(self._auth_headers()) + headers['Authorization'] = 'Bearer ' + self.tokens['access_token'] + try: + self._request('POST', self.supabase_url + '/auth/v1/logout', headers, body={}) + except Exception: + pass # best-effort; local tokens are cleared regardless + self.tokens = None + if self.on_tokens: + self.on_tokens(None) + + # -- Readest API -------------------------------------------------------- + + def _api(self, method, path, body=None): + self.ensure_fresh_token() + headers = { + 'Authorization': 'Bearer ' + self.tokens['access_token'], + 'Accept': 'application/json', + } + status, parsed, payload = self._request(method, self.api_base + path, headers, body=body) + if status in (401, 403): + message = self._error_message(parsed, payload, 'Not authenticated') + if 'quota' in message.lower(): + raise QuotaExceededError(message, status) + raise AuthRequiredError(message, status) + if status != 200: + raise ReadestAPIError( + self._error_message(parsed, payload, 'Request failed (%s)' % status), status + ) + return parsed + + def pull_books(self, since=0): + result = self._api('GET', '/sync?type=books&since=%d' % since) + return (result or {}).get('books') or [] + + def push_books(self, records): + return self._api('POST', '/sync', body={'books': records, 'notes': [], 'configs': []}) + + def get_upload_url(self, file_name, file_size, book_hash): + return self._api( + 'POST', + '/storage/upload', + body={'fileName': file_name, 'fileSize': file_size, 'bookHash': book_hash}, + ) + + def list_files(self, book_hash): + result = self._api('GET', '/storage/list?bookHash=' + urllib.parse.quote(book_hash)) + return (result or {}).get('files') or [] + + def delete_file(self, file_key): + return self._api( + 'DELETE', '/storage/delete?fileKey=' + urllib.parse.quote(file_key, safe='') + ) + + def put_file(self, url, fileobj, size): + """PUT raw bytes to a presigned URL (no auth headers).""" + request = urllib.request.Request( + url, + data=fileobj, + headers={'Content-Length': str(size)}, + method='PUT', + ) + status, payload = self.transport(request, UPLOAD_TIMEOUT) + if status not in (200, 201, 204): + message = 'Upload failed (%s)' % status + if payload: + # S3/R2 error responses are XML; surface the tag. + text = payload.decode('utf-8', 'replace') + start, end = text.find(''), text.find('') + if 0 <= start < end: + message += ': ' + text[start + 6 : end] + raise ReadestAPIError(message, status) diff --git a/apps/readest-calibre-plugin/config.py b/apps/readest-calibre-plugin/config.py new file mode 100644 index 00000000..9cfde330 --- /dev/null +++ b/apps/readest-calibre-plugin/config.py @@ -0,0 +1,50 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +from calibre.utils.config import JSONConfig +from qt.core import QCheckBox, QLabel, QLineEdit, QVBoxLayout, QWidget + +from calibre_plugins.readest.api import DEFAULT_API_BASE, DEFAULT_SUPABASE_URL + +prefs = JSONConfig('plugins/readest') + +prefs.defaults['api_base'] = DEFAULT_API_BASE +prefs.defaults['supabase_url'] = DEFAULT_SUPABASE_URL +prefs.defaults['tokens'] = None # {access_token, refresh_token, expires_at, expires_in} +prefs.defaults['user_email'] = None +prefs.defaults['include_custom_columns'] = True + + +def save_tokens(tokens): + prefs['tokens'] = tokens + if tokens is None: + prefs['user_email'] = None + + +class ConfigWidget(QWidget): + def __init__(self): + QWidget.__init__(self) + layout = QVBoxLayout() + self.setLayout(layout) + + account = prefs['user_email'] + status = ('Logged in as %s.' % account) if account else 'Not logged in.' + layout.addWidget(QLabel(status + ' Use the Readest toolbar menu to log in or out.')) + + self.custom_columns_checkbox = QCheckBox('Include custom columns in pushed metadata') + self.custom_columns_checkbox.setChecked(bool(prefs['include_custom_columns'])) + layout.addWidget(self.custom_columns_checkbox) + + layout.addWidget(QLabel('API server:')) + self.api_base_edit = QLineEdit(self) + self.api_base_edit.setText(prefs['api_base']) + self.api_base_edit.setToolTip( + 'Only change this if you run a self-hosted Readest server.' + ) + layout.addWidget(self.api_base_edit) + + layout.addStretch() + + def save_settings(self): + prefs['include_custom_columns'] = self.custom_columns_checkbox.isChecked() + prefs['api_base'] = self.api_base_edit.text().strip() or DEFAULT_API_BASE diff --git a/apps/readest-calibre-plugin/dialogs.py b/apps/readest-calibre-plugin/dialogs.py new file mode 100644 index 00000000..f5b854a9 --- /dev/null +++ b/apps/readest-calibre-plugin/dialogs.py @@ -0,0 +1,225 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +from qt.core import ( + QDialog, + QDialogButtonBox, + QFormLayout, + QHBoxLayout, + QLabel, + QLineEdit, + QProgressBar, + QPushButton, + QTableWidget, + QTableWidgetItem, + QThread, + QUrl, + QVBoxLayout, + pyqtSignal, +) + +from calibre.gui2 import error_dialog, open_url + +from calibre_plugins.readest.api import ReadestAPIError +from calibre_plugins.readest.oauth import PROVIDERS, OAuthCallbackServer, build_authorize_url +from calibre_plugins.readest.worker import STATUS_LABELS, PushWorker + +OAUTH_WAIT_SECONDS = 300 + + +class _OAuthWaiter(QThread): + got_tokens = pyqtSignal(dict) + timed_out = pyqtSignal() + + def __init__(self, parent, server): + QThread.__init__(self, parent) + self.server = server + + def run(self): + tokens = self.server.wait(OAUTH_WAIT_SECONDS) + if tokens: + self.got_tokens.emit(tokens) + else: + self.timed_out.emit() + + +class LoginDialog(QDialog): + """Email/password login plus browser OAuth (Google, Apple, GitHub, Discord).""" + + def __init__(self, parent, client): + QDialog.__init__(self, parent) + self.client = client + self.user = None + self.oauth_server = None + self.oauth_waiter = None + + self.setWindowTitle('Log in to Readest') + layout = QVBoxLayout() + self.setLayout(layout) + + form = QFormLayout() + self.email_edit = QLineEdit(self) + self.password_edit = QLineEdit(self) + self.password_edit.setEchoMode(QLineEdit.EchoMode.Password) + form.addRow('Email:', self.email_edit) + form.addRow('Password:', self.password_edit) + layout.addLayout(form) + + self.login_btn = QPushButton('Log in', self) + self.login_btn.clicked.connect(self.password_login) + layout.addWidget(self.login_btn) + + layout.addWidget(QLabel('Or sign in with your browser:')) + providers_layout = QHBoxLayout() + for provider in PROVIDERS: + btn = QPushButton(provider.capitalize(), self) + btn.clicked.connect(lambda _=False, p=provider: self.oauth_login(p)) + providers_layout.addWidget(btn) + layout.addLayout(providers_layout) + + self.status_label = QLabel('') + self.status_label.setWordWrap(True) + layout.addWidget(self.status_label) + + buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Cancel) + buttons.rejected.connect(self.reject) + layout.addWidget(buttons) + + def password_login(self): + email = self.email_edit.text().strip() + password = self.password_edit.text() + if not email or not password: + self.status_label.setText('Please enter both email and password.') + return + self.status_label.setText('Logging in…') + self.login_btn.setEnabled(False) + try: + self.user = self.client.sign_in_password(email, password) + except ReadestAPIError as err: + self.status_label.setText('Login failed: %s' % err) + self.login_btn.setEnabled(True) + return + self.accept() + + def oauth_login(self, provider): + self.stop_oauth() + self.oauth_server = OAuthCallbackServer() + port = self.oauth_server.start() + self.oauth_waiter = _OAuthWaiter(self, self.oauth_server) + self.oauth_waiter.got_tokens.connect(self.oauth_finished) + self.oauth_waiter.timed_out.connect( + lambda: self.status_label.setText('Browser login timed out. Try again.') + ) + self.oauth_waiter.start() + self.status_label.setText('Waiting for the browser login to complete…') + open_url(QUrl(build_authorize_url(self.client.supabase_url, provider, port))) + + def oauth_finished(self, tokens): + if tokens.get('error'): + self.status_label.setText( + 'Login failed: %s' % (tokens.get('error_description') or tokens['error']) + ) + return + if not tokens.get('access_token') or not tokens.get('refresh_token'): + self.status_label.setText('Login failed: the browser callback carried no session.') + return + self.client.set_session(tokens) + try: + self.user = self.client.get_user() + except ReadestAPIError as err: + self.status_label.setText('Login failed: %s' % err) + return + self.accept() + + def stop_oauth(self): + # Disconnect before stopping: stop() wakes the waiter thread, and its + # signals must not fire into a dialog that is going away. + if self.oauth_waiter: + try: + self.oauth_waiter.got_tokens.disconnect() + self.oauth_waiter.timed_out.disconnect() + except TypeError: + pass + if self.oauth_server: + self.oauth_server.stop() + self.oauth_server = None + if self.oauth_waiter: + self.oauth_waiter.wait(2000) + self.oauth_waiter = None + + def done(self, result): + self.stop_oauth() + QDialog.done(self, result) + + +class PushDialog(QDialog): + """Per-book status table for a push run, modeled on BookFusion's sync log.""" + + def __init__(self, parent, db, book_ids, client, include_custom_columns): + QDialog.__init__(self, parent) + self.db = db + self.worker = PushWorker(self, db, book_ids, client, include_custom_columns) + self.worker.progress.connect(self.on_progress) + self.worker.book_status.connect(self.on_book_status) + self.worker.done.connect(self.on_done) + + self.setWindowTitle('Push to Readest') + self.setMinimumSize(520, 380) + layout = QVBoxLayout() + self.setLayout(layout) + + count = len(book_ids) + layout.addWidget( + QLabel('Pushing %d %s to your Readest library…' % (count, _plural(count))) + ) + + self.progress_bar = QProgressBar(self) + self.progress_bar.setRange(0, count) + layout.addWidget(self.progress_bar) + + self.table = QTableWidget(0, 3, self) + self.table.setHorizontalHeaderLabels(['Book', 'Status', 'Details']) + self.table.horizontalHeader().setStretchLastSection(True) + self.table.setColumnWidth(0, 220) + self.table.setEditTriggers(QTableWidget.EditTrigger.NoEditTriggers) + layout.addWidget(self.table) + + self.summary_label = QLabel('') + self.summary_label.setWordWrap(True) + layout.addWidget(self.summary_label) + + self.buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Cancel) + self.buttons.rejected.connect(self.reject) + layout.addWidget(self.buttons) + + self.worker.start() + + def on_progress(self, done, total): + self.progress_bar.setValue(done) + + def on_book_status(self, book_id, status, detail): + title = self.db.field_for('title', book_id) or 'Unknown' + row = self.table.rowCount() + self.table.insertRow(row) + self.table.setItem(row, 0, QTableWidgetItem(title)) + self.table.setItem(row, 1, QTableWidgetItem(STATUS_LABELS.get(status, status))) + self.table.setItem(row, 2, QTableWidgetItem(detail)) + self.table.scrollToBottom() + + def on_done(self, ok, message): + self.progress_bar.setValue(self.progress_bar.maximum()) + self.summary_label.setText(message) + self.buttons.setStandardButtons(QDialogButtonBox.StandardButton.Close) + if not ok and self.table.rowCount() == 0: + error_dialog(self, 'Push to Readest failed', message, show=True) + + def reject(self): + if self.worker.isRunning(): + self.worker.cancel() + self.summary_label.setText('Canceling…') + return + QDialog.reject(self) + + +def _plural(count): + return 'book' if count == 1 else 'books' diff --git a/apps/readest-calibre-plugin/images/icon.png b/apps/readest-calibre-plugin/images/icon.png new file mode 100644 index 00000000..2d8aa82d Binary files /dev/null and b/apps/readest-calibre-plugin/images/icon.png differ diff --git a/apps/readest-calibre-plugin/oauth.py b/apps/readest-calibre-plugin/oauth.py new file mode 100644 index 00000000..4628df86 --- /dev/null +++ b/apps/readest-calibre-plugin/oauth.py @@ -0,0 +1,106 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +"""Browser OAuth sign-in via a localhost callback server. + +Supabase redirects OAuth logins to the whitelisted http://localhost:{port} +with the session tokens in the URL *fragment* — the same flow readest-app's +desktop custom-OAuth mode uses (tauri-plugin-oauth). Fragments never reach an +HTTP server, so the first response serves a page whose script forwards the +fragment as query parameters to /callback. Standard-library only. +""" + +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, HTTPServer + +PROVIDERS = ('google', 'apple', 'github', 'discord') + +LANDING_PAGE = b""" +Readest Login +

Completing login…

+ +""" + +DONE_PAGE = b""" +Readest Login +

Login complete. You can close this tab and return to calibre.

+""" + + +def build_authorize_url(supabase_url, provider, port): + return '%s/auth/v1/authorize?%s' % ( + supabase_url.rstrip('/'), + urllib.parse.urlencode({'provider': provider, 'redirect_to': 'http://localhost:%d' % port}), + ) + + +def parse_callback_query(query): + """Extract session tokens (or an OAuth error) from the callback query.""" + params = urllib.parse.parse_qs(query) + result = {} + for key in ('access_token', 'refresh_token', 'error', 'error_description'): + if key in params: + result[key] = params[key][0] + for key in ('expires_at', 'expires_in'): + if key in params: + try: + result[key] = int(params[key][0]) + except ValueError: + pass + return result + + +class _Handler(BaseHTTPRequestHandler): + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == '/callback': + tokens = parse_callback_query(parsed.query) + self._respond(DONE_PAGE) + if tokens: + self.server.oauth_result = tokens + self.server.oauth_event.set() + else: + self._respond(LANDING_PAGE) + + def _respond(self, page): + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.send_header('Content-Length', str(len(page))) + self.end_headers() + self.wfile.write(page) + + def log_message(self, *args): + pass # keep calibre's stdout clean + + +class OAuthCallbackServer: + def __init__(self): + self._server = None + self._thread = None + + def start(self): + """Bind to an ephemeral port and serve in a daemon thread.""" + self._server = HTTPServer(('127.0.0.1', 0), _Handler) + self._server.oauth_result = None + self._server.oauth_event = threading.Event() + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + return self._server.server_address[1] + + def wait(self, timeout): + """Tokens dict once the callback arrives, or None on timeout/stop.""" + server = self._server + if server is None or not server.oauth_event.wait(timeout): + return None + return server.oauth_result + + def stop(self): + server, self._server = self._server, None + if server: + server.oauth_event.set() # wake any wait()er (returns None) + server.shutdown() + server.server_close() diff --git a/apps/readest-calibre-plugin/plugin-import-name-readest.txt b/apps/readest-calibre-plugin/plugin-import-name-readest.txt new file mode 100644 index 00000000..e69de29b diff --git a/apps/readest-calibre-plugin/tests/test_client.py b/apps/readest-calibre-plugin/tests/test_client.py new file mode 100644 index 00000000..f105ad33 --- /dev/null +++ b/apps/readest-calibre-plugin/tests/test_client.py @@ -0,0 +1,234 @@ +import io +import json +import os +import sys +import time +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from api import ( # noqa: E402 + AuthRequiredError, + QuotaExceededError, + ReadestAPIError, + ReadestClient, +) + +API_BASE = 'https://web.example.com/api' +SUPABASE = 'https://sb.example.com' +ANON_KEY = 'anon-key' + + +class FakeTransport: + """Records requests; replies from a queue of (status, json_body) tuples.""" + + def __init__(self): + self.requests = [] + self.responses = [] + + def queue(self, status, body): + self.responses.append((status, body)) + + def __call__(self, request, timeout): + data = request.data + if hasattr(data, 'read'): + data = data.read() + self.requests.append( + { + 'method': request.get_method(), + 'url': request.full_url, + 'headers': {k.lower(): v for k, v in request.header_items()}, + 'body': data, + } + ) + status, body = self.responses.pop(0) + payload = body if isinstance(body, bytes) else json.dumps(body).encode('utf-8') + return status, payload + + +def make_client(transport, tokens=None, saved=None): + def on_tokens(t): + if saved is not None: + saved.append(t) + + return ReadestClient( + api_base=API_BASE, + supabase_url=SUPABASE, + anon_key=ANON_KEY, + tokens=tokens, + on_tokens=on_tokens, + transport=transport, + ) + + +def valid_tokens(expires_in=3600): + return { + 'access_token': 'at', + 'refresh_token': 'rt', + 'expires_at': int(time.time()) + expires_in, + 'expires_in': expires_in, + } + + +class SignInTest(unittest.TestCase): + def test_password_sign_in_stores_tokens(self): + transport = FakeTransport() + saved = [] + transport.queue( + 200, + { + 'access_token': 'new-at', + 'refresh_token': 'new-rt', + 'expires_at': 1234, + 'expires_in': 3600, + 'user': {'id': 'u1', 'email': 'a@b.c'}, + }, + ) + client = make_client(transport, saved=saved) + user = client.sign_in_password('a@b.c', 'pw') + + req = transport.requests[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['url'], f'{SUPABASE}/auth/v1/token?grant_type=password') + self.assertEqual(req['headers']['apikey'], ANON_KEY) + self.assertEqual(json.loads(req['body']), {'email': 'a@b.c', 'password': 'pw'}) + self.assertEqual(user['id'], 'u1') + self.assertEqual(saved[-1]['access_token'], 'new-at') + + def test_failed_sign_in_raises_with_message(self): + transport = FakeTransport() + transport.queue(400, {'error_description': 'Invalid login credentials'}) + client = make_client(transport) + with self.assertRaises(ReadestAPIError) as ctx: + client.sign_in_password('a@b.c', 'bad') + self.assertIn('Invalid login credentials', str(ctx.exception)) + + +class TokenRefreshTest(unittest.TestCase): + def test_fresh_token_not_refreshed(self): + transport = FakeTransport() + transport.queue(200, {'books': []}) + client = make_client(transport, tokens=valid_tokens()) + client.pull_books() + self.assertEqual(len(transport.requests), 1) # no refresh round-trip + + def test_expiring_token_refreshed_before_call(self): + transport = FakeTransport() + saved = [] + transport.queue( + 200, + {'access_token': 'at2', 'refresh_token': 'rt2', 'expires_at': 99, 'expires_in': 3600}, + ) + transport.queue(200, {'books': []}) + tokens = valid_tokens() + tokens['expires_at'] = int(time.time()) + 10 # nearly expired + client = make_client(transport, tokens=tokens, saved=saved) + client.pull_books() + + self.assertEqual( + transport.requests[0]['url'], + f'{SUPABASE}/auth/v1/token?grant_type=refresh_token', + ) + self.assertEqual(json.loads(transport.requests[0]['body']), {'refresh_token': 'rt'}) + self.assertEqual(transport.requests[1]['headers']['authorization'], 'Bearer at2') + self.assertEqual(saved[-1]['access_token'], 'at2') + + def test_no_tokens_raises_auth_required(self): + client = make_client(FakeTransport()) + with self.assertRaises(AuthRequiredError): + client.pull_books() + + +class SyncTest(unittest.TestCase): + def test_pull_books(self): + transport = FakeTransport() + transport.queue(200, {'books': [{'book_hash': 'h1'}]}) + client = make_client(transport, tokens=valid_tokens()) + books = client.pull_books() + + req = transport.requests[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['url'], f'{API_BASE}/sync?type=books&since=0') + self.assertEqual(req['headers']['authorization'], 'Bearer at') + self.assertEqual(books, [{'book_hash': 'h1'}]) + + def test_push_books(self): + transport = FakeTransport() + transport.queue(200, {'books': [{'book_hash': 'h1'}]}) + client = make_client(transport, tokens=valid_tokens()) + client.push_books([{'hash': 'h1', 'title': 'T'}]) + + req = transport.requests[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['url'], f'{API_BASE}/sync') + body = json.loads(req['body']) + self.assertEqual(body['books'][0]['hash'], 'h1') + self.assertEqual(body['notes'], []) + self.assertEqual(body['configs'], []) + + +class StorageTest(unittest.TestCase): + def test_get_upload_url(self): + transport = FakeTransport() + transport.queue(200, {'uploadUrl': 'https://s3/put', 'fileKey': 'k'}) + client = make_client(transport, tokens=valid_tokens()) + res = client.get_upload_url('Readest/Books/h/h.epub', 5, 'h') + + body = json.loads(transport.requests[0]['body']) + self.assertEqual(body['fileName'], 'Readest/Books/h/h.epub') + self.assertEqual(body['fileSize'], 5) + self.assertEqual(body['bookHash'], 'h') + self.assertEqual(res['uploadUrl'], 'https://s3/put') + + def test_quota_exceeded(self): + transport = FakeTransport() + transport.queue(403, {'error': 'Insufficient storage quota', 'usage': 1}) + client = make_client(transport, tokens=valid_tokens()) + with self.assertRaises(QuotaExceededError): + client.get_upload_url('f', 5, 'h') + + def test_list_files(self): + transport = FakeTransport() + transport.queue(200, {'files': [{'file_key': 'u/Readest/Books/h/h.epub'}]}) + client = make_client(transport, tokens=valid_tokens()) + files = client.list_files('h') + + req = transport.requests[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['url'], f'{API_BASE}/storage/list?bookHash=h') + self.assertEqual(files, [{'file_key': 'u/Readest/Books/h/h.epub'}]) + + def test_delete_file(self): + transport = FakeTransport() + transport.queue(200, {'success': True}) + client = make_client(transport, tokens=valid_tokens()) + client.delete_file('u/Readest/Books/h/h.epub') + + req = transport.requests[0] + self.assertEqual(req['method'], 'DELETE') + self.assertEqual( + req['url'], + f'{API_BASE}/storage/delete?fileKey=u%2FReadest%2FBooks%2Fh%2Fh.epub', + ) + + def test_put_file_sends_content_length(self): + transport = FakeTransport() + transport.queue(200, b'') + client = make_client(transport, tokens=valid_tokens()) + client.put_file('https://s3/put', io.BytesIO(b'12345'), 5) + + req = transport.requests[0] + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['headers']['content-length'], '5') + self.assertEqual(req['body'], b'12345') + + def test_put_file_failure_raises(self): + transport = FakeTransport() + transport.queue(500, b'InternalError') + client = make_client(transport, tokens=valid_tokens()) + with self.assertRaises(ReadestAPIError): + client.put_file('https://s3/put', io.BytesIO(b'x'), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/readest-calibre-plugin/tests/test_hashes.py b/apps/readest-calibre-plugin/tests/test_hashes.py new file mode 100644 index 00000000..0f0a1b7e --- /dev/null +++ b/apps/readest-calibre-plugin/tests/test_hashes.py @@ -0,0 +1,116 @@ +import hashlib +import io +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from api import meta_hash, partial_md5, partial_md5_bytes # noqa: E402 + + +def reference_ranges(size): + """Chunk ranges of utils/md5.ts::partialMD5, computed independently. + + The JS loop runs i in -1..10 with start = min(size, 1024 << (2*i)); the + JS << operator wraps 1024 << -2 to 0, so i == -1 reads offset 0. + """ + ranges = [] + for i in range(-1, 11): + offset = 0 if i == -1 else 1024 << (2 * i) + start = min(size, offset) + if start >= size: + break + ranges.append((start, min(start + 1024, size))) + return ranges + + +def reference_hash(data): + hasher = hashlib.md5() + for start, end in reference_ranges(len(data)): + hasher.update(data[start:end]) + return hasher.hexdigest() + + +class PartialMd5Test(unittest.TestCase): + def check(self, data): + self.assertEqual(partial_md5(io.BytesIO(data), len(data)), reference_hash(data)) + self.assertEqual(partial_md5_bytes(data), reference_hash(data)) + + def test_small_file_reads_everything(self): + data = b'hello world' + self.assertEqual(partial_md5_bytes(data), hashlib.md5(data).hexdigest()) + + def test_exactly_one_chunk(self): + self.check(bytes(range(256)) * 4) # 1024 bytes + + def test_two_chunks(self): + self.check(os.urandom(3000)) + + def test_skips_middle_of_larger_file(self): + data = os.urandom(20000) + self.check(data) + # Sanity: 20000 bytes covers offsets 0, 1024, 4096, 16384 then stops. + self.assertEqual( + reference_ranges(20000), + [(0, 1024), (1024, 2048), (4096, 5120), (16384, 17408)], + ) + + def test_from_file_path(self, tmp_name='partial_md5_fixture.bin'): + import tempfile + + data = os.urandom(5000) + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, tmp_name) + with open(path, 'wb') as f: + f.write(data) + self.assertEqual(partial_md5(path), reference_hash(data)) + + +class MetaHashTest(unittest.TestCase): + def md5(self, source): + import unicodedata + + return hashlib.md5(unicodedata.normalize('NFC', source).encode('utf-8')).hexdigest() + + def test_basic(self): + self.assertEqual( + meta_hash('Foo', ['Alice', 'Bob'], ['urn:uuid:1234']), + self.md5('Foo|Alice,Bob|1234'), + ) + + def test_prefers_uuid_over_isbn(self): + self.assertEqual( + meta_hash('Foo', ['Alice'], ['isbn:9781234567890', 'uuid:abcd']), + self.md5('Foo|Alice|abcd'), + ) + + def test_calibre_scheme_preferred_over_isbn(self): + self.assertEqual( + meta_hash('Foo', ['Alice'], ['isbn:9781234567890', 'calibre:42']), + self.md5('Foo|Alice|42'), + ) + + def test_urn_identifier_sliced_after_last_colon(self): + self.assertEqual( + meta_hash('T', ['A'], ['urn:isbn:978-0-00-000000-0']), + self.md5('T|A|978-0-00-000000-0'), + ) + + def test_no_identifiers(self): + self.assertEqual(meta_hash('T', ['A'], []), self.md5('T|A|')) + + def test_plain_identifier_without_scheme(self): + self.assertEqual(meta_hash('T', ['A'], ['abcdef']), self.md5('T|A|abcdef')) + + def test_nfc_normalization(self): + decomposed = 'Cafe\u0301' # e + combining acute accent + composed = 'Caf\u00e9' + self.assertEqual( + meta_hash(decomposed, [], []), + hashlib.md5(f'{composed}||'.encode('utf-8')).hexdigest(), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/readest-calibre-plugin/tests/test_oauth.py b/apps/readest-calibre-plugin/tests/test_oauth.py new file mode 100644 index 00000000..20e909c5 --- /dev/null +++ b/apps/readest-calibre-plugin/tests/test_oauth.py @@ -0,0 +1,91 @@ +import os +import sys +import unittest +import urllib.parse +import urllib.request + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from oauth import OAuthCallbackServer, build_authorize_url, parse_callback_query # noqa: E402 + + +class AuthorizeUrlTest(unittest.TestCase): + def test_url_shape(self): + url = build_authorize_url('https://sb.example.com', 'google', 43210) + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + self.assertEqual(parsed.scheme, 'https') + self.assertEqual(parsed.netloc, 'sb.example.com') + self.assertEqual(parsed.path, '/auth/v1/authorize') + self.assertEqual(query['provider'], ['google']) + self.assertEqual(query['redirect_to'], ['http://localhost:43210']) + + +class ParseCallbackQueryTest(unittest.TestCase): + def test_tokens(self): + tokens = parse_callback_query( + 'access_token=at&refresh_token=rt&expires_at=123&expires_in=3600&token_type=bearer' + ) + self.assertEqual(tokens['access_token'], 'at') + self.assertEqual(tokens['refresh_token'], 'rt') + self.assertEqual(tokens['expires_at'], 123) + self.assertEqual(tokens['expires_in'], 3600) + + def test_error(self): + tokens = parse_callback_query('error=access_denied&error_description=Denied') + self.assertEqual(tokens['error'], 'access_denied') + self.assertEqual(tokens['error_description'], 'Denied') + + def test_missing_tokens(self): + self.assertEqual(parse_callback_query(''), {}) + + +class CallbackServerTest(unittest.TestCase): + def test_full_roundtrip(self): + server = OAuthCallbackServer() + port = server.start() + try: + # First request: Supabase redirects to "/" with tokens in the URL + # fragment, which never reaches the server — it must serve a page + # whose script forwards the fragment as query params. + with urllib.request.urlopen(f'http://127.0.0.1:{port}/', timeout=5) as res: + page = res.read().decode('utf-8') + self.assertIn('location.hash', page) + self.assertIn('/callback', page) + + with urllib.request.urlopen( + f'http://127.0.0.1:{port}/callback?access_token=at&refresh_token=rt' + '&expires_at=123&expires_in=3600', + timeout=5, + ) as res: + self.assertEqual(res.status, 200) + + tokens = server.wait(timeout=5) + self.assertIsNotNone(tokens) + self.assertEqual(tokens['access_token'], 'at') + self.assertEqual(tokens['refresh_token'], 'rt') + finally: + server.stop() + + def test_wait_timeout(self): + server = OAuthCallbackServer() + server.start() + try: + self.assertIsNone(server.wait(timeout=0.1)) + finally: + server.stop() + + def test_stop_wakes_waiter(self): + import threading + import time + + server = OAuthCallbackServer() + server.start() + threading.Timer(0.1, server.stop).start() + started = time.monotonic() + self.assertIsNone(server.wait(timeout=10)) + self.assertLess(time.monotonic() - started, 5) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/readest-calibre-plugin/tests/test_wire.py b/apps/readest-calibre-plugin/tests/test_wire.py new file mode 100644 index 00000000..5c5f9362 --- /dev/null +++ b/apps/readest-calibre-plugin/tests/test_wire.py @@ -0,0 +1,346 @@ +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from wire import ( # noqa: E402 + build_metadata, + build_wire_book, + index_rows_by_uuid, + iso_to_ms, + merge_for_push, + pick_format, + pick_server_row, + plan_push, + tombstone_record, +) + +NOW = 1_800_000_000_000 +SRC = 's' * 32 # partial MD5 of the raw calibre library file + +BOOK = { + 'title': 'The Test Book', + 'authors': ['Alice Author', 'Bob Writer'], + 'languages': ['eng'], + 'publisher': 'Test House', + 'pubdate': '2020-05-01T00:00:00+00:00', + 'comments': '

A very good book.

', + 'tags': ['Fiction', 'Test'], + 'series': 'Test Series', + 'series_index': 2.0, + 'uuid': 'cafebabe-0000-0000-0000-000000000001', + 'isbn': '9781234567897', + 'custom_columns': {'read_status': 'done'}, + 'source_hash': SRC, +} + + +def server_row(**overrides): + row = { + 'book_hash': 'a' * 32, + 'meta_hash': 'b' * 32, + 'format': 'EPUB', + 'title': 'The Test Book', + 'author': 'Alice Author, Bob Writer', + 'tags': ['Fiction', 'Test'], + 'group_id': 'g1', + 'group_name': 'Group One', + 'progress': [3, 100], + 'reading_status': 'reading', + 'reading_status_updated_at': '2024-01-02T00:00:00.000Z', + 'cover_hash': 'c' * 32, + 'cover_updated_at': '2024-01-02T00:00:00.000Z', + 'metadata': None, + 'created_at': '2024-01-01T00:00:00.000Z', + 'updated_at': '2024-01-02T00:00:00.000Z', + 'deleted_at': None, + 'uploaded_at': '2024-01-01T12:00:00.000Z', + } + row.update(overrides) + return row + + +def wire_for(book=BOOK, file_hash='a' * 32, fmt='EPUB', now=NOW): + return build_wire_book(book, file_hash, fmt, now) + + +def synced_row(wire): + """A server row that matches `wire` (as if we pushed it earlier).""" + return server_row( + title=wire['title'], + author=wire['author'], + tags=wire.get('tags'), + metadata=json.dumps(wire['metadata']), + ) + + +class PickFormatTest(unittest.TestCase): + def test_prefers_epub(self): + self.assertEqual(pick_format(['MOBI', 'EPUB', 'PDF']), 'EPUB') + + def test_case_insensitive(self): + self.assertEqual(pick_format(['azw3', 'txt']), 'AZW3') + + def test_unsupported_only(self): + self.assertIsNone(pick_format(['DOCX', 'LRF'])) + + def test_empty(self): + self.assertIsNone(pick_format([])) + + +class BuildMetadataTest(unittest.TestCase): + def test_fields(self): + meta = build_metadata(BOOK) + self.assertEqual(meta['title'], 'The Test Book') + self.assertEqual(meta['author'], ['Alice Author', 'Bob Writer']) + self.assertEqual(meta['language'], 'eng') + self.assertEqual(meta['publisher'], 'Test House') + self.assertEqual(meta['published'], '2020-05-01T00:00:00+00:00') + self.assertEqual(meta['description'], '

A very good book.

') + self.assertEqual(meta['subject'], ['Fiction', 'Test']) + self.assertEqual(meta['series'], 'Test Series') + self.assertEqual(meta['seriesIndex'], 2.0) + self.assertEqual(meta['identifier'], 'urn:uuid:cafebabe-0000-0000-0000-000000000001') + self.assertEqual(meta['isbn'], '9781234567897') + self.assertEqual(meta['customColumns'], {'read_status': 'done'}) + self.assertEqual(meta['calibreSourceHash'], SRC) + + def test_single_author_is_string(self): + meta = build_metadata(dict(BOOK, authors=['Solo'])) + self.assertEqual(meta['author'], 'Solo') + + def test_omits_empty_fields(self): + meta = build_metadata({'title': 'T', 'authors': []}) + self.assertNotIn('publisher', meta) + self.assertNotIn('series', meta) + self.assertNotIn('customColumns', meta) + self.assertNotIn('isbn', meta) + self.assertNotIn('calibreSourceHash', meta) + + def test_strips_nul_characters(self): + meta = build_metadata({'title': 'T\x00itle', 'authors': ['A\x00nn']}) + self.assertEqual(meta['title'], 'Title') + self.assertEqual(meta['author'], 'Ann') + + +class BuildWireBookTest(unittest.TestCase): + def test_record_shape(self): + wire = wire_for() + self.assertEqual(wire['hash'], 'a' * 32) + self.assertEqual(wire['bookHash'], 'a' * 32) + self.assertEqual(wire['format'], 'EPUB') + self.assertEqual(wire['title'], 'The Test Book') + self.assertEqual(wire['sourceTitle'], 'The Test Book') + self.assertEqual(wire['author'], 'Alice Author, Bob Writer') + self.assertEqual(wire['tags'], ['Fiction', 'Test']) + self.assertEqual(wire['createdAt'], NOW) + self.assertEqual(wire['updatedAt'], NOW) + self.assertEqual(len(wire['metaHash']), 32) + self.assertEqual(wire['metadata']['title'], 'The Test Book') + + def test_meta_hash_uses_uuid_identifier(self): + import hashlib + import unicodedata + + wire = wire_for() + source = 'The Test Book|Alice Author,Bob Writer|cafebabe-0000-0000-0000-000000000001' + expected = hashlib.md5(unicodedata.normalize('NFC', source).encode('utf-8')).hexdigest() + self.assertEqual(wire['metaHash'], expected) + + +class IsoToMsTest(unittest.TestCase): + def test_iso_with_ms(self): + self.assertEqual(iso_to_ms('2024-01-01T00:00:00.000Z'), 1704067200000) + + def test_iso_without_ms(self): + self.assertEqual(iso_to_ms('2024-01-01T00:00:00Z'), 1704067200000) + + def test_none(self): + self.assertIsNone(iso_to_ms(None)) + + +class PlanPushTest(unittest.TestCase): + def test_new_book(self): + plan = plan_push(None, wire_for(), 'c' * 32, SRC) + self.assertEqual(plan['action'], 'new') + self.assertTrue(plan['upload_cover']) + + def test_new_book_without_cover(self): + plan = plan_push(None, wire_for(), None, SRC) + self.assertEqual(plan['action'], 'new') + self.assertFalse(plan['upload_cover']) + + def test_unchanged_book_is_skipped(self): + wire = wire_for() + plan = plan_push(synced_row(wire), wire, 'c' * 32, SRC) + self.assertEqual(plan['action'], 'skip') + + def test_changed_metadata_is_update(self): + wire = wire_for(dict(BOOK, title='Renamed Title')) + row = synced_row(wire_for()) + plan = plan_push(row, wire, 'c' * 32, SRC) + self.assertEqual(plan['action'], 'update') + self.assertFalse(plan['upload_cover']) + + def test_changed_cover_only_is_update_with_cover(self): + wire = wire_for() + plan = plan_push(synced_row(wire), wire, 'd' * 32, SRC) + self.assertEqual(plan['action'], 'update') + self.assertTrue(plan['upload_cover']) + + def test_tombstoned_row_is_resurrected(self): + wire = wire_for() + row = synced_row(wire) + row['deleted_at'] = '2024-06-01T00:00:00.000Z' + plan = plan_push(row, wire, 'c' * 32, SRC) + self.assertEqual(plan['action'], 'update') + + def test_tags_change_is_update(self): + wire = wire_for(dict(BOOK, tags=['Fiction'])) + row = synced_row(wire_for()) + self.assertEqual(plan_push(row, wire, 'c' * 32, SRC)['action'], 'update') + + def test_missing_local_cover_does_not_force_update(self): + wire = wire_for() + plan = plan_push(synced_row(wire), wire, None, SRC) + self.assertEqual(plan['action'], 'skip') + + def test_row_without_file_is_replaced(self): + wire = wire_for() + row = synced_row(wire) + row['uploaded_at'] = None + plan = plan_push(row, wire, 'c' * 32, SRC) + self.assertEqual(plan['action'], 'replace') + self.assertTrue(plan['upload_cover']) # new hash namespace needs its own cover + + def test_changed_source_file_is_replaced(self): + wire = wire_for(dict(BOOK, source_hash='n' * 32)) + row = synced_row(wire_for()) # cloud copy built from SRC + plan = plan_push(row, wire, 'c' * 32, 'n' * 32) + self.assertEqual(plan['action'], 'replace') + + def test_v1_row_with_matching_raw_hash_is_not_replaced(self): + # v1 rows have no calibreSourceHash but their book_hash IS the raw + # file hash (v1 uploaded the file unmodified). + book = dict(BOOK) + del book['source_hash'] + v1_wire = wire_for(book) + row = synced_row(v1_wire) + row['book_hash'] = SRC + plan = plan_push(row, wire_for(), 'c' * 32, SRC) + self.assertEqual(plan['action'], 'update') # gains calibreSourceHash + + def test_v1_row_with_changed_file_is_replaced(self): + book = dict(BOOK) + del book['source_hash'] + row = synced_row(wire_for(book)) + row['book_hash'] = 'o' * 32 # raw hash of the OLD file + plan = plan_push(row, wire_for(), 'c' * 32, SRC) + self.assertEqual(plan['action'], 'replace') + + +class ServerRowLookupTest(unittest.TestCase): + def test_index_rows_by_uuid(self): + wire = wire_for() + row = synced_row(wire) + index = index_rows_by_uuid([row, server_row(book_hash='x' * 32, metadata='not json')]) + self.assertEqual(index, {'cafebabe-0000-0000-0000-000000000001': row}) + + def test_index_prefers_live_row(self): + wire = wire_for() + dead = synced_row(wire) + dead['deleted_at'] = '2024-06-01T00:00:00.000Z' + live = synced_row(wire) + live['book_hash'] = 'x' * 32 + for rows in ([dead, live], [live, dead]): + index = index_rows_by_uuid(rows) + self.assertEqual(index['cafebabe-0000-0000-0000-000000000001'], live) + + def test_index_prefers_newer_row(self): + wire = wire_for() + old = synced_row(wire) + old['updated_at'] = '2024-01-01T00:00:00.000Z' + new = synced_row(wire) + new['book_hash'] = 'x' * 32 + new['updated_at'] = '2024-06-01T00:00:00.000Z' + index = index_rows_by_uuid([old, new]) + self.assertEqual(index['cafebabe-0000-0000-0000-000000000001'], new) + + def test_pick_prefers_live_hash_match(self): + hash_row = server_row() + uuid_row = server_row(book_hash='x' * 32) + self.assertIs(pick_server_row(hash_row, uuid_row), hash_row) + + def test_pick_falls_back_to_live_uuid_row(self): + dead = server_row(deleted_at='2024-06-01T00:00:00.000Z') + live = server_row(book_hash='x' * 32) + self.assertIs(pick_server_row(dead, live), live) + + def test_pick_returns_tombstone_when_nothing_live(self): + dead = server_row(deleted_at='2024-06-01T00:00:00.000Z') + self.assertIs(pick_server_row(None, dead), dead) + self.assertIsNone(pick_server_row(None, None)) + + +class TombstoneRecordTest(unittest.TestCase): + def test_shape(self): + rec = tombstone_record(server_row(), NOW) + self.assertEqual(rec['hash'], 'a' * 32) + self.assertEqual(rec['bookHash'], 'a' * 32) + self.assertEqual(rec['format'], 'EPUB') + self.assertEqual(rec['title'], 'The Test Book') + self.assertEqual(rec['author'], 'Alice Author, Bob Writer') + self.assertEqual(rec['createdAt'], iso_to_ms('2024-01-01T00:00:00.000Z')) + self.assertEqual(rec['updatedAt'], NOW) + self.assertEqual(rec['deletedAt'], NOW) + + +class MergeForPushTest(unittest.TestCase): + def test_new_book_merge(self): + wire = wire_for() + rec = merge_for_push(wire, None, NOW, uploaded_at_ms=NOW, cover_hash='e' * 32) + self.assertEqual(rec['createdAt'], NOW) + self.assertEqual(rec['updatedAt'], NOW) + self.assertEqual(rec['uploadedAt'], NOW) + self.assertEqual(rec['coverHash'], 'e' * 32) + self.assertEqual(rec['coverUpdatedAt'], NOW) + self.assertIsNone(rec['deletedAt']) + + def test_update_carries_server_fields(self): + wire = wire_for(dict(BOOK, title='Renamed')) + row = server_row(metadata=json.dumps(build_metadata(BOOK))) + rec = merge_for_push(wire, row, NOW) + # Fields the server would explicit-null if omitted must be carried over. + self.assertEqual(rec['groupId'], 'g1') + self.assertEqual(rec['groupName'], 'Group One') + self.assertEqual(rec['progress'], [3, 100]) + self.assertEqual(rec['readingStatus'], 'reading') + self.assertEqual(rec['readingStatusUpdatedAt'], iso_to_ms('2024-01-02T00:00:00.000Z')) + self.assertEqual(rec['uploadedAt'], iso_to_ms('2024-01-01T12:00:00.000Z')) + self.assertEqual(rec['coverHash'], 'c' * 32) + self.assertEqual(rec['coverUpdatedAt'], iso_to_ms('2024-01-02T00:00:00.000Z')) + self.assertEqual(rec['createdAt'], iso_to_ms('2024-01-01T00:00:00.000Z')) + # LWW: the push must win over the server row. + self.assertEqual(rec['updatedAt'], NOW) + # Our fresh metadata wins. + self.assertEqual(rec['title'], 'Renamed') + # Resurrects tombstones. + self.assertIsNone(rec['deletedAt']) + + def test_reupload_overrides_uploaded_at(self): + wire = wire_for() + row = server_row(uploaded_at=None) + rec = merge_for_push(wire, row, NOW, uploaded_at_ms=NOW) + self.assertEqual(rec['uploadedAt'], NOW) + + def test_new_cover_overrides_server_cover(self): + wire = wire_for() + rec = merge_for_push(wire, server_row(), NOW, cover_hash='f' * 32) + self.assertEqual(rec['coverHash'], 'f' * 32) + self.assertEqual(rec['coverUpdatedAt'], NOW) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/readest-calibre-plugin/ui.py b/apps/readest-calibre-plugin/ui.py new file mode 100644 index 00000000..81961b8e --- /dev/null +++ b/apps/readest-calibre-plugin/ui.py @@ -0,0 +1,115 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +from qt.core import QMenu + +from calibre.gui2 import error_dialog, info_dialog +from calibre.gui2.actions import InterfaceAction + +from calibre_plugins.readest.api import ReadestClient +from calibre_plugins.readest.config import prefs, save_tokens +from calibre_plugins.readest.dialogs import LoginDialog, PushDialog + + +def make_client(): + return ReadestClient( + api_base=prefs['api_base'], + supabase_url=prefs['supabase_url'], + tokens=prefs['tokens'], + on_tokens=save_tokens, + ) + + +class ReadestInterfacePlugin(InterfaceAction): + name = 'Readest Sync' + action_spec = ( + 'Readest', + None, + 'Push selected books and metadata to your Readest library', + None, + ) + + def genesis(self): + self.push_action = self.create_action( + spec=('Push selected books to Readest', None, None, None), + attr='Push selected books to Readest', + ) + self.push_action.triggered.connect(self.push_selected) + + self.login_action = self.create_action( + spec=('Log in to Readest…', None, None, None), attr='Log in to Readest' + ) + self.login_action.triggered.connect(self.login) + + self.logout_action = self.create_action( + spec=('Log out', None, None, None), attr='Log out from Readest' + ) + self.logout_action.triggered.connect(self.logout) + + self.config_action = self.create_action( + spec=('Customize plugin…', None, None, None), attr='Customize Readest plugin' + ) + self.config_action.triggered.connect(self.show_config) + + self.menu = QMenu(self.gui) + self.menu.addAction(self.push_action) + self.menu.addSeparator() + self.menu.addAction(self.login_action) + self.menu.addAction(self.logout_action) + self.menu.addAction(self.config_action) + self.menu.aboutToShow.connect(self.update_menu) + + self.qaction.setMenu(self.menu) + self.qaction.setIcon(get_icons('images/icon.png', 'Readest Sync')) + self.qaction.triggered.connect(self.push_selected) + + def update_menu(self): + logged_in = bool(prefs['tokens']) + email = prefs['user_email'] + self.login_action.setVisible(not logged_in) + self.logout_action.setVisible(logged_in) + if logged_in and email: + self.logout_action.setText('Log out (%s)' % email) + self.push_action.setEnabled(len(self.selected_book_ids()) > 0) + + def selected_book_ids(self): + return self.gui.library_view.get_selected_ids() + + def push_selected(self): + book_ids = self.selected_book_ids() + if not book_ids: + return error_dialog( + self.gui, 'No books selected', 'Select the books to push to Readest.', show=True + ) + if not prefs['tokens'] and not self.login(): + return + PushDialog( + self.gui, + self.gui.current_db.new_api, + book_ids, + make_client(), + bool(prefs['include_custom_columns']), + ).exec() + + def login(self): + dialog = LoginDialog(self.gui, make_client()) + if dialog.exec() != dialog.DialogCode.Accepted: + return False + user = dialog.user or {} + prefs['user_email'] = user.get('email') + info_dialog( + self.gui, + 'Readest', + 'Logged in as %s.' % (user.get('email') or 'your Readest account'), + show=True, + ) + return True + + def logout(self): + try: + make_client().sign_out() + finally: + save_tokens(None) + + def show_config(self): + self.interface_action_base_plugin.do_user_config(self.gui) diff --git a/apps/readest-calibre-plugin/wire.py b/apps/readest-calibre-plugin/wire.py new file mode 100644 index 00000000..8ec84237 --- /dev/null +++ b/apps/readest-calibre-plugin/wire.py @@ -0,0 +1,292 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +"""Calibre metadata → Readest wire records, and per-book push planning. + +Standard-library only (no calibre / Qt imports) so it can be unit-tested +outside calibre. The wire shape is the camelCase Book record consumed by +POST /api/sync (apps/readest-app/src/utils/transform.ts::transformBookToDB, +inverted — same as readest.koplugin's row_to_wire). +""" + +import json +from datetime import datetime, timezone + +try: # inside calibre the plugin loads as a package + from calibre_plugins.readest.api import meta_hash +except ImportError: # plain imports for the unit tests + from api import meta_hash + +# Readest-supported formats, in upload preference order. +FORMAT_PRIORITY = ('EPUB', 'PDF', 'AZW3', 'MOBI', 'AZW', 'FB2', 'FBZ', 'CBZ', 'TXT', 'MD') + +EXTS = { + 'EPUB': 'epub', + 'PDF': 'pdf', + 'MOBI': 'mobi', + 'AZW': 'azw', + 'AZW3': 'azw3', + 'CBZ': 'cbz', + 'FB2': 'fb2', + 'FBZ': 'fbz', + 'TXT': 'txt', + 'MD': 'md', +} + +CLOUD_BOOKS_SUBDIR = 'Readest/Books' + + +def pick_format(available): + """Best Readest-supported format among calibre's, or None.""" + upper = {f.upper() for f in available} + for fmt in FORMAT_PRIORITY: + if fmt in upper: + return fmt + return None + + +def book_file_name(file_hash, fmt): + # Matches getRemoteBookFilename for S3 ({hash}/{hash}.{ext}); R2 + # deployments resolve it through the download API's hash+extension + # fallback, same as the koplugin. + return '%s/%s/%s.%s' % (CLOUD_BOOKS_SUBDIR, file_hash, file_hash, EXTS[fmt]) + + +def cover_file_name(file_hash): + return '%s/%s/cover.png' % (CLOUD_BOOKS_SUBDIR, file_hash) + + +def _clean(value): + # The server strips NUL from strings (utils/sanitize.ts::sanitizeString); + # strip locally too so pushed and pulled values compare equal. + if isinstance(value, str): + return value.replace('\x00', '') + return value + + +def build_metadata(book): + """BookMetadata JSON for the books row (libs/document.ts::BookMetadata). + + `book` is a plain dict extracted from calibre's Metadata object: + title, authors, languages, publisher, pubdate, comments, tags, series, + series_index, uuid, isbn, custom_columns. + """ + authors = [_clean(a) for a in book.get('authors') or [] if a] + meta = { + 'title': _clean(book.get('title')) or '', + 'author': authors[0] if len(authors) == 1 else authors, + } + languages = book.get('languages') or [] + if isinstance(languages, str): + languages = [languages] + if languages: + meta['language'] = languages[0] if len(languages) == 1 else languages + optional = { + 'publisher': _clean(book.get('publisher')), + 'published': _clean(book.get('pubdate')), + 'description': _clean(book.get('comments')), + 'subject': [_clean(t) for t in book.get('tags') or []] or None, + 'series': _clean(book.get('series')), + 'isbn': _clean(book.get('isbn')), + # Partial MD5 of the RAW calibre library file. The uploaded blob has + # metadata embedded so its hash (book_hash) shifts with every embed; + # this stable fingerprint is how a later push detects "the file + # itself changed" without any machine-local state. + 'calibreSourceHash': book.get('source_hash'), + } + if book.get('series'): + optional['seriesIndex'] = book.get('series_index') + if book.get('uuid'): + optional['identifier'] = 'urn:uuid:%s' % book['uuid'] + if book.get('custom_columns'): + optional['customColumns'] = book['custom_columns'] + for key, value in optional.items(): + if value not in (None, '', []): + meta[key] = value + return meta + + +def _author_string(book): + return ', '.join(_clean(a) for a in book.get('authors') or [] if a) + + +def build_wire_book(book, file_hash, fmt, now_ms): + """Base wire record for a calibre book (before server-row merging).""" + identifiers = [] + if book.get('uuid'): + identifiers.append('urn:uuid:%s' % book['uuid']) + if book.get('isbn'): + identifiers.append('isbn:%s' % book['isbn']) + title = _clean(book.get('title')) or '' + authors = [_clean(a) for a in book.get('authors') or [] if a] + return { + 'hash': file_hash, + 'bookHash': file_hash, + 'metaHash': meta_hash(title, authors, identifiers), + 'format': fmt, + 'title': title, + 'sourceTitle': title, + 'author': _author_string(book), + 'tags': [_clean(t) for t in book.get('tags') or []], + 'metadata': build_metadata(book), + 'createdAt': now_ms, + 'updatedAt': now_ms, + } + + +def iso_to_ms(iso): + if not iso: + return None + text = iso.replace('Z', '+00:00') + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + + +def _parse_row_metadata(row): + raw = row.get('metadata') + if isinstance(raw, dict): + return raw + if isinstance(raw, str) and raw: + try: + return json.loads(raw) + except ValueError: + return None + return None + + +def _row_matches_wire(row, wire): + if (row.get('title') or '') != wire['title']: + return False + if (row.get('author') or '') != wire['author']: + return False + if (row.get('tags') or []) != (wire.get('tags') or []): + return False + return _parse_row_metadata(row) == wire['metadata'] + + +def row_source_hash(row): + """The raw-file fingerprint a server row was built from. + + v2 rows carry it as metadata.calibreSourceHash; v1 rows uploaded the raw + file unmodified, so their book_hash IS the raw hash. + """ + meta = _parse_row_metadata(row) or {} + return meta.get('calibreSourceHash') or row.get('book_hash') + + +def metadata_uuid(meta): + """Calibre book uuid from a metadata dict's urn:uuid identifier.""" + identifier = (meta or {}).get('identifier') + if isinstance(identifier, str) and 'urn:uuid:' in identifier: + return identifier.rsplit(':', 1)[-1].lower() + return None + + +def _prefer_row(row, over): + row_live, over_live = not row.get('deleted_at'), not over.get('deleted_at') + if row_live != over_live: + return row_live + return (iso_to_ms(row.get('updated_at')) or 0) > (iso_to_ms(over.get('updated_at')) or 0) + + +def index_rows_by_uuid(rows): + """Map calibre uuid -> best server row (live over tombstoned, then newest). + + This is the identity that survives file-content changes: the uploaded blob + has metadata embedded, so book_hash shifts whenever the file or its + metadata does, but the calibre uuid in metadata.identifier stays put. + """ + best = {} + for row in rows: + uuid = metadata_uuid(_parse_row_metadata(row)) + if not uuid: + continue + current = best.get(uuid) + if current is None or _prefer_row(row, current): + best[uuid] = row + return best + + +def pick_server_row(hash_row, uuid_row): + """Choose the row a push should target: live hash match, then live uuid + match, then whatever tombstone is left (for resurrection).""" + for row in (hash_row, uuid_row): + if row is not None and not row.get('deleted_at'): + return row + return hash_row if hash_row is not None else uuid_row + + +def plan_push(server_row, wire, local_cover_hash, source_hash): + """Decide what to do for one book. + + Returns {'action': 'new' | 'replace' | 'update' | 'skip', + 'upload_cover': bool}. + - new: no server row — embed metadata, upload file, insert row + - replace: the raw file changed (or the row has no file blob) — upload a + fresh embedded blob under its new hash, tombstone the old row + - update: file unchanged, but metadata/cover/tombstone differ — row only + - skip: file + metadata + cover all unchanged + """ + if server_row is None: + return {'action': 'new', 'upload_cover': bool(local_cover_hash)} + if not server_row.get('uploaded_at') or row_source_hash(server_row) != source_hash: + # A replaced book gets a new hash namespace, so it needs its own cover. + return {'action': 'replace', 'upload_cover': bool(local_cover_hash)} + cover_changed = bool(local_cover_hash) and local_cover_hash != server_row.get('cover_hash') + changed = ( + not _row_matches_wire(server_row, wire) + or bool(server_row.get('deleted_at')) + or cover_changed + ) + return {'action': 'update' if changed else 'skip', 'upload_cover': cover_changed} + + +def tombstone_record(row, now_ms): + """Wire record that soft-deletes a replaced server row.""" + return { + 'hash': row['book_hash'], + 'bookHash': row['book_hash'], + 'metaHash': row.get('meta_hash'), + 'format': row.get('format'), + 'title': row.get('title') or '', + 'author': row.get('author') or '', + 'createdAt': iso_to_ms(row.get('created_at')) or now_ms, + 'updatedAt': now_ms, + 'deletedAt': now_ms, + } + + +def merge_for_push(wire, server_row, now_ms, uploaded_at_ms=None, cover_hash=None): + """Final record for POST /sync. + + The server explicit-nulls any field absent from the wire record + (transformBookToDB), so an update must carry over the server row's + groupId/groupName, progress, readingStatus, uploadedAt and cover fields — + the koplugin learned this the hard way (see syncbooks.lua:291-299). + """ + record = dict(wire) + record['updatedAt'] = now_ms + record['deletedAt'] = None # an explicit push (re)activates the book + row = server_row or {} + record['createdAt'] = iso_to_ms(row.get('created_at')) or wire['createdAt'] + if row.get('group_id'): + record['groupId'] = row['group_id'] + if row.get('group_name'): + record['groupName'] = row['group_name'] + if row.get('progress') is not None: + record['progress'] = row['progress'] + if row.get('reading_status'): + record['readingStatus'] = row['reading_status'] + record['readingStatusUpdatedAt'] = iso_to_ms(row.get('reading_status_updated_at')) + record['uploadedAt'] = ( + uploaded_at_ms if uploaded_at_ms is not None else iso_to_ms(row.get('uploaded_at')) + ) + if cover_hash: + record['coverHash'] = cover_hash + record['coverUpdatedAt'] = now_ms + elif row.get('cover_hash'): + record['coverHash'] = row['cover_hash'] + record['coverUpdatedAt'] = iso_to_ms(row.get('cover_updated_at')) + return record diff --git a/apps/readest-calibre-plugin/worker.py b/apps/readest-calibre-plugin/worker.py new file mode 100644 index 00000000..c65535ff --- /dev/null +++ b/apps/readest-calibre-plugin/worker.py @@ -0,0 +1,267 @@ +__license__ = 'AGPL v3' +__copyright__ = '2026, Bilingify LLC' + +"""Background push worker: one QThread, sequential per-book processing.""" + +import io +import os +import shutil +import tempfile +import time +import traceback + +from qt.core import QThread, pyqtSignal + +from calibre_plugins.readest.api import ( + AuthRequiredError, + QuotaExceededError, + partial_md5, + partial_md5_bytes, +) +from calibre_plugins.readest.wire import ( + EXTS, + book_file_name, + build_wire_book, + cover_file_name, + index_rows_by_uuid, + merge_for_push, + pick_format, + pick_server_row, + plan_push, + tombstone_record, +) + +STATUS_LABELS = { + 'uploaded': 'Uploaded', + 'replaced': 'Replaced', + 'updated': 'Updated', + 'skipped': 'Up to date', + 'failed': 'Failed', +} + + +def _jsonable(value): + if hasattr(value, 'isoformat'): + return value.isoformat() + if isinstance(value, (list, tuple)): + return [_jsonable(v) for v in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def _custom_columns(mi): + columns = {} + for key in mi.custom_field_keys(): + try: + value = mi.get(key) + except Exception: + continue + if value in (None, '', []) or value == (): + continue + columns[key.lstrip('#')] = _jsonable(value) + return columns + + +def _book_dict(mi, include_custom_columns, source_hash): + pubdate = getattr(mi, 'pubdate', None) + # calibre uses year-101 dates as "undefined". + if pubdate is not None and getattr(pubdate, 'year', 0) < 1000: + pubdate = None + return { + 'title': mi.title, + 'authors': [a for a in (mi.authors or []) if a], + 'languages': list(mi.languages or []), + 'publisher': mi.publisher, + 'pubdate': pubdate.isoformat() if pubdate else None, + 'comments': mi.comments, + 'tags': sorted(mi.tags or []), + 'series': mi.series, + 'series_index': mi.series_index if mi.series else None, + 'uuid': getattr(mi, 'uuid', None), + 'isbn': (mi.get_identifiers() or {}).get('isbn'), + 'custom_columns': _custom_columns(mi) if include_custom_columns else None, + 'source_hash': source_hash, + } + + +def _embed_metadata_copy(path, mi, fmt): + """Copy the library file to a temp path and embed `mi` into it. + + The library file itself is never touched. Formats without a metadata + writer (or a failing writer) fall back to the unmodified copy. + """ + fd, tmp = tempfile.mkstemp(suffix='.' + EXTS[fmt], prefix='readest-') + os.close(fd) + shutil.copyfile(path, tmp) + try: + from calibre.ebooks.metadata.meta import set_metadata + + with open(tmp, 'r+b') as stream: + set_metadata(stream, mi, EXTS[fmt]) + except Exception: + traceback.print_exc() + return tmp + + +class PushWorker(QThread): + progress = pyqtSignal(int, int) # done, total + book_status = pyqtSignal(int, str, str) # book_id, status key, detail + done = pyqtSignal(bool, str) # ok, message + + def __init__(self, parent, db, book_ids, client, include_custom_columns): + QThread.__init__(self, parent) + self.db = db # calibre new_api (thread-safe) + self.book_ids = list(book_ids) + self.client = client + self.include_custom_columns = include_custom_columns + self.canceled = False + + def cancel(self): + self.canceled = True + + def run(self): + try: + rows = [r for r in self.client.pull_books() if r.get('book_hash')] + self.by_hash = {r['book_hash']: r for r in rows} + self.by_uuid = index_rows_by_uuid(rows) + except AuthRequiredError as err: + self.done.emit(False, 'Please log in to Readest first. (%s)' % err) + return + except Exception as err: + self.done.emit(False, 'Could not reach Readest: %s' % err) + return + + counts = {} + for index, book_id in enumerate(self.book_ids): + if self.canceled: + self.done.emit(False, 'Canceled.') + return + try: + status, detail = self._push_one(book_id) + except QuotaExceededError as err: + self.book_status.emit(book_id, 'failed', str(err)) + self.done.emit(False, 'Readest storage quota exceeded — push stopped.') + return + except AuthRequiredError as err: + self.book_status.emit(book_id, 'failed', str(err)) + self.done.emit(False, 'Session expired — please log in again.') + return + except Exception as err: + traceback.print_exc() + status, detail = 'failed', str(err) + counts[status] = counts.get(status, 0) + 1 + self.book_status.emit(book_id, status, detail) + self.progress.emit(index + 1, len(self.book_ids)) + + summary = ', '.join( + '%d %s' % (counts[key], STATUS_LABELS[key].lower()) + for key in ('uploaded', 'replaced', 'updated', 'skipped', 'failed') + if key in counts + ) + self.done.emit('failed' not in counts, summary or 'Nothing to push.') + + def _upload(self, file_name, fileobj, size, book_hash): + upload = self.client.get_upload_url(file_name, size, book_hash) + self.client.put_file(upload['uploadUrl'], fileobj, size) + + def _upload_cover(self, book_hash, cover_bytes): + self._upload(cover_file_name(book_hash), io.BytesIO(cover_bytes), len(cover_bytes), book_hash) + + def _delete_cloud_files(self, book_hash): + # Best-effort quota reclaim for a replaced book, mirroring the + # koplugin's deleteCloudFiles: list to learn exact keys, then DELETE. + try: + for record in self.client.list_files(book_hash): + if record.get('file_key'): + self.client.delete_file(record['file_key']) + except Exception: + traceback.print_exc() + + def _remember(self, record, uuid): + # Keep the lookup maps current so a duplicate calibre entry later in + # this run resolves to what we just pushed. + row = { + 'book_hash': record['hash'], + 'title': record['title'], + 'author': record['author'], + 'tags': record.get('tags'), + 'metadata': record['metadata'], + 'uploaded_at': 'pushed', + 'cover_hash': record.get('coverHash'), + } + self.by_hash[record['hash']] = row + if uuid: + self.by_uuid[uuid] = row + + def _push_one(self, book_id): + mi = self.db.get_metadata(book_id) + fmt = pick_format(self.db.formats(book_id)) + if fmt is None: + return 'failed', 'No Readest-supported format (EPUB, PDF, ...)' + path = self.db.format_abspath(book_id, fmt) + if not path or not os.path.exists(path): + return 'failed', 'Book file is missing from the calibre library' + + source_hash = partial_md5(path) + cover_bytes = self.db.cover(book_id) + cover_hash = partial_md5_bytes(cover_bytes) if cover_bytes else None + + now_ms = int(time.time() * 1000) + book = _book_dict(mi, self.include_custom_columns, source_hash) + wire = build_wire_book(book, source_hash, fmt, now_ms) + uuid = (book.get('uuid') or '').lower() + server_row = pick_server_row( + self.by_hash.get(source_hash), self.by_uuid.get(uuid) if uuid else None + ) + plan = plan_push(server_row, wire, cover_hash, source_hash) + + if plan['action'] == 'skip': + return 'skipped', '' + + if plan['action'] in ('new', 'replace'): + blob_path = _embed_metadata_copy(path, mi, fmt) + try: + blob_hash = partial_md5(blob_path) + wire['hash'] = wire['bookHash'] = blob_hash + size = os.path.getsize(blob_path) + with open(blob_path, 'rb') as f: + self._upload(book_file_name(blob_hash, fmt), f, size, blob_hash) + finally: + try: + os.unlink(blob_path) + except OSError: + pass + if plan['upload_cover'] and cover_bytes: + self._upload_cover(blob_hash, cover_bytes) + records = [ + merge_for_push( + wire, + server_row, + now_ms, + uploaded_at_ms=now_ms, + cover_hash=cover_hash if plan['upload_cover'] and cover_bytes else None, + ) + ] + replaced_hash = None + if server_row is not None and server_row['book_hash'] != blob_hash: + replaced_hash = server_row['book_hash'] + records.append(tombstone_record(server_row, now_ms)) + self.client.push_books(records) + if replaced_hash: + self._delete_cloud_files(replaced_hash) + self._remember(records[0], uuid) + return ('replaced' if plan['action'] == 'replace' else 'uploaded'), '' + + # action == 'update': the cloud file is current, only the row (and + # possibly the cover) changed. Keep the row's hash namespace. + wire['hash'] = wire['bookHash'] = server_row['book_hash'] + pushed_cover = plan['upload_cover'] and cover_bytes + if pushed_cover: + self._upload_cover(server_row['book_hash'], cover_bytes) + record = merge_for_push( + wire, server_row, now_ms, cover_hash=cover_hash if pushed_cover else None + ) + self.client.push_books([record]) + self._remember(record, uuid) + return 'updated', ''