Files
readest/apps/readest-app/src/libs/crdt.ts
T
Huang Xin cbdc3b8f52 feat(sync): wire dictionary store through replica sync (follow-up to #4075) (#4076)
* feat(sync): cross-device dictionary sync

Custom MDict / StarDict / DICT / SLOB dictionaries now sync across
signed-in devices via the replica layer.

- Store mutations publish replica rows with field-level LWW + tombstones.
- Re-importing the same content (renamed or after delete) preserves the
  user's label and reincarnates the server row instead of duplicating.
- Manifest commits after binary upload so other devices never see a row
  whose binaries aren't on cloud storage yet.
- Pull-side orchestrator creates a placeholder dict, queues the binaries
  via TransferManager, and clears the unavailable flag on completion.
- Toast copy branches by transfer kind so dict uploads don't read
  "Book uploaded".

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(sync): boot pull and binary download path

- Defer the boot pull until TransferManager is initialized so download
  enqueues aren't dropped.
- Auto-persist the local dict store after applyRemoteDictionary; otherwise
  the next loadCustomDictionaries wipes the in-memory rows.
- Boot pull passes since=null so a device whose cursor advanced past
  unpersisted rows can still recover.
- Skip pulling when not authenticated instead of logging
  "SyncError: Not authenticated" on every boot of a signed-out device.
- downloadReplicaFile resolves the destination against the kind's base
  dir; binaries previously landed at the literal lfp and openFile then
  failed with "File not found".

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* refactor(sync): per-page useReplicaPull hook

Lifts the boot-time pull out of EnvContext into a hook each page mounts
for the kinds it needs: useReplicaPull({ kinds: ['dictionary'] }).
Library page and the shared Reader component opt in. The hook fires 10s
after page load (so feature mounts hydrate first), dedups per-kind
across navigation, and releases the slot on failure so a later mount
can retry. Future kinds plug into the hook's per-kind switch.

Also closes two refresh-loop bugs:

- Hydrate the dict store from settings BEFORE the apply loop, so the
  auto-persist doesn't clobber persisted rows that the in-memory store
  hadn't yet read. Library-page refresh was the visible victim.
- Skip the download queue when every manifest file is already on disk
  under the resolved bundle dir. Refreshing is a no-op; partial-
  download recovery still queues because some files would be missing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-06 21:39:38 +02:00

203 lines
5.9 KiB
TypeScript

import type { FieldEnvelope, FieldsObject, Hlc, ReplicaRow } from '@/types/replica';
const MAX_PHYSICAL_MS = 0xfffffffffffff;
const MAX_COUNTER = 0xffffffff;
export const hlcPack = (physicalMs: number, counter: number, deviceId: string): Hlc => {
if (physicalMs < 0 || physicalMs > MAX_PHYSICAL_MS) {
throw new RangeError(`physicalMs out of range: ${physicalMs}`);
}
if (counter < 0 || counter > MAX_COUNTER) {
throw new RangeError(`counter out of range: ${counter}`);
}
const ms = physicalMs.toString(16).padStart(13, '0');
const c = counter.toString(16).padStart(8, '0');
return `${ms}-${c}-${deviceId}` as Hlc;
};
export const hlcParse = (h: Hlc): { physicalMs: number; counter: number; deviceId: string } => {
const dash1 = h.indexOf('-');
const dash2 = h.indexOf('-', dash1 + 1);
if (dash1 < 0 || dash2 < 0) throw new Error(`malformed HLC: ${h}`);
return {
physicalMs: parseInt(h.slice(0, dash1), 16),
counter: parseInt(h.slice(dash1 + 1, dash2), 16),
deviceId: h.slice(dash2 + 1),
};
};
export const hlcCompare = (a: Hlc, b: Hlc): -1 | 0 | 1 => (a < b ? -1 : a > b ? 1 : 0);
export const hlcMax = (a: Hlc | null, b: Hlc | null): Hlc | null => {
if (!a) return b;
if (!b) return a;
return hlcCompare(a, b) >= 0 ? a : b;
};
export interface HlcSnapshot {
physicalMs: number;
counter: number;
}
export class HlcGenerator {
private physicalMs = 0;
private counter = 0;
constructor(
private readonly deviceId: string,
private readonly now: () => number = Date.now,
) {}
static restore(
snapshot: HlcSnapshot,
deviceId: string,
now: () => number = Date.now,
): HlcGenerator {
const gen = new HlcGenerator(deviceId, now);
gen.physicalMs = snapshot.physicalMs;
gen.counter = snapshot.counter;
return gen;
}
next(): Hlc {
const wallMs = this.now();
if (wallMs > this.physicalMs) {
this.physicalMs = wallMs;
this.counter = 0;
} else {
this.counter += 1;
if (this.counter > MAX_COUNTER) {
this.physicalMs += 1;
this.counter = 0;
}
}
return hlcPack(this.physicalMs, this.counter, this.deviceId);
}
observe(remote: Hlc): void {
const { physicalMs: rMs, counter: rC } = hlcParse(remote);
const wallMs = this.now();
const newMs = Math.max(this.physicalMs, rMs, wallMs);
if (newMs === this.physicalMs && newMs === rMs) {
this.counter = Math.max(this.counter, rC) + 1;
} else if (newMs === rMs) {
this.physicalMs = newMs;
this.counter = rC + 1;
} else if (newMs === this.physicalMs) {
this.counter += 1;
} else {
this.physicalMs = newMs;
this.counter = 0;
}
}
serialize(): HlcSnapshot {
return { physicalMs: this.physicalMs, counter: this.counter };
}
}
export const setField = <V>(
fields: FieldsObject,
name: string,
value: V,
hlc: Hlc,
deviceId: string,
): FieldsObject => ({
...fields,
[name]: { v: value, t: hlc, s: deviceId },
});
const pickWinner = (a: FieldEnvelope, b: FieldEnvelope): FieldEnvelope => {
const cmp = hlcCompare(a.t, b.t);
if (cmp > 0) return a;
if (cmp < 0) return b;
return a.s >= b.s ? a : b;
};
export const mergeFields = (local: FieldsObject, remote: FieldsObject): FieldsObject => {
const out: FieldsObject = { ...local };
for (const key of Object.keys(remote)) {
const lo = local[key];
const re = remote[key]!;
out[key] = lo ? pickWinner(lo, re) : re;
}
return out;
};
const computeUpdatedAt = (fields: FieldsObject, deletedAt: Hlc | null): Hlc => {
let max: Hlc | null = deletedAt;
for (const key of Object.keys(fields)) {
const env = fields[key]!;
max = hlcMax(max, env.t);
}
return (max ?? hlcPack(0, 0, '')) as Hlc;
};
export const removeReplica = (row: ReplicaRow, hlc: Hlc): ReplicaRow => {
const deletedAt = hlcMax(row.deleted_at_ts, hlc) ?? hlc;
return {
...row,
deleted_at_ts: deletedAt,
updated_at_ts: hlcMax(row.updated_at_ts, hlc) ?? hlc,
};
};
export const withReincarnation = (row: ReplicaRow, token: string): ReplicaRow => ({
...row,
reincarnation: token,
deleted_at_ts: null,
fields_jsonb: {},
});
export const mergeReplica = (local: ReplicaRow, remote: ReplicaRow): ReplicaRow => {
if (
local.user_id !== remote.user_id ||
local.kind !== remote.kind ||
local.replica_id !== remote.replica_id
) {
throw new Error('mergeReplica: identity mismatch');
}
const fields = mergeFields(local.fields_jsonb, remote.fields_jsonb);
const deleted_at_ts = hlcMax(local.deleted_at_ts, remote.deleted_at_ts);
const reincarnationCandidates = [
local.reincarnation ? { token: local.reincarnation, t: local.updated_at_ts } : null,
remote.reincarnation ? { token: remote.reincarnation, t: remote.updated_at_ts } : null,
].filter((c): c is { token: string; t: Hlc } => c !== null);
const winningReincarnation =
reincarnationCandidates.length === 0
? null
: reincarnationCandidates.reduce((a, b) => (hlcCompare(a.t, b.t) >= 0 ? a : b));
const reincarnation =
winningReincarnation &&
(!deleted_at_ts || hlcCompare(winningReincarnation.t, deleted_at_ts) > 0)
? winningReincarnation.token
: null;
const manifest_jsonb =
remote.manifest_jsonb === null
? local.manifest_jsonb
: local.manifest_jsonb === null
? remote.manifest_jsonb
: hlcCompare(remote.updated_at_ts, local.updated_at_ts) > 0
? remote.manifest_jsonb
: local.manifest_jsonb;
const schema_version = Math.max(local.schema_version, remote.schema_version);
const contentUpdatedAt = computeUpdatedAt(fields, deleted_at_ts);
const rowUpdatedAt = hlcMax(local.updated_at_ts, remote.updated_at_ts);
const updated_at_ts = hlcMax(contentUpdatedAt, rowUpdatedAt) ?? contentUpdatedAt;
return {
user_id: local.user_id,
kind: local.kind,
replica_id: local.replica_id,
fields_jsonb: fields,
manifest_jsonb,
deleted_at_ts,
reincarnation,
updated_at_ts,
schema_version,
};
};