diff --git a/apps/readest-app/src/services/send/conversion/convertToEpub.ts b/apps/readest-app/src/services/send/conversion/convertToEpub.ts index 38054ca2..465d4278 100644 --- a/apps/readest-app/src/services/send/conversion/convertToEpub.ts +++ b/apps/readest-app/src/services/send/conversion/convertToEpub.ts @@ -11,7 +11,7 @@ import { buildEpub } from './buildEpub'; import { bundleAssets } from './assetBundler'; import { generateCoverSvg } from './coverGenerator'; import { fetchAuthorImage, fetchBestFavicon } from './faviconFetcher'; -import { findSiteRule, type SiteRule } from './siteRules'; +import { findSiteRule, META_FALLBACK, type SiteRule } from './siteRules'; import { extractHeadings } from './toc'; import { ConversionError } from './types'; import type { ConvertibleMime, ConvertedBook, EpubChapter, EpubImage, TocEntry } from './types'; @@ -181,12 +181,18 @@ function extractWithSiteRule(rawHtml: string, rule: SiteRule): RuleExtracted | n } } } + // Rule selectors first, OpenGraph / Twitter Card meta tags second. + // The meta-tag layer is what saves us when the site ships a frontend + // redesign and our CSS hooks stop matching — every reputable publisher + // keeps these tags stable for crawlers. const ruleTitle = rule.title ? (doc.querySelector(rule.title)?.textContent?.trim() ?? '') : ''; const ruleByline = rule.byline ? (doc.querySelector(rule.byline)?.textContent?.trim() ?? '') : ''; + const title = ruleTitle || pickMetaContent(doc, META_FALLBACK.title) || ''; + const byline = ruleByline || pickMetaContent(doc, META_FALLBACK.byline) || ''; return { content: contentEl.innerHTML, - title: ruleTitle || undefined, - byline: ruleByline || undefined, + title: title || undefined, + byline: byline || undefined, }; } @@ -242,6 +248,40 @@ function extractArticleFallback(rawHtml: string, minTextChars: number): string | * hostname stripped of the leading `www.` so e.g. `https://nytimes.com/foo` * yields "nytimes.com". */ +/** + * Walk a list of meta-tag selectors and return the first non-empty + * `content` attribute. Used as the universal safety net when a per-site + * rule's selector missed (site re-skinned and our CSS hooks no longer + * match) — see `META_FALLBACK` in `siteRules.ts` for the canonical + * lists per field. + */ +function pickMetaContent(doc: Document, selectors: readonly string[]): string | null { + for (const sel of selectors) { + const value = doc.querySelector(sel)?.getAttribute('content')?.trim(); + if (value) return value; + } + return null; +} + +/** + * Variant of `pickMetaContent` that resolves the picked URL against the + * page URL — meta `og:image` values are often root-relative. Returns + * null if nothing matched or the URL was unparseable. + */ +function resolveMetaImage( + doc: Document, + selectors: readonly string[], + pageUrl: string, +): string | null { + const raw = pickMetaContent(doc, selectors); + if (!raw) return null; + try { + return new URL(raw, pageUrl).toString(); + } catch { + return null; + } +} + function extractSiteName(doc: Document, pageUrl: string): string { const metaSelectors = [ 'meta[property="og:site_name"]', @@ -284,14 +324,26 @@ async function buildArticleCover( const siteName = extractSiteName(doc, pageUrl); // Prefer an author profile image (richer than a site-wide favicon). - // When the site rule exposes an `authorImage` selector, resolve the URL - // and fetch it. Falls through to the favicon on any failure. + // Three layers, in order of richness: + // 1. The site rule's `authorImage` selector — handcrafted, e.g. the + // WeChat account avatar or the X profile picture. + // 2. OpenGraph / Twitter Card `og:image` — every publisher sets this + // for link unfurlers. Works when no rule matched, AND when the + // rule's selector missed because the site reshuffled its DOM. + // 3. Favicon (handled below). + // Site-rule miss is silent — `pickImageUrlFromSelector` returns null, + // we fall through to the meta layer. let authorImage: { bytes: ArrayBuffer; mime: string } | undefined; - if (rule?.authorImage) { - const url = pickImageUrlFromSelector(doc, rule.authorImage, pageUrl); - if (url) { - authorImage = (await fetchAuthorImage(url, pageUrl).catch(() => null)) ?? undefined; - } + const ruleImageUrl = rule?.authorImage + ? pickImageUrlFromSelector(doc, rule.authorImage, pageUrl) + : null; + const metaImageUrl = ruleImageUrl + ? null + : resolveMetaImage(doc, META_FALLBACK.authorImage, pageUrl); + const candidateImageUrl = ruleImageUrl || metaImageUrl; + if (candidateImageUrl) { + authorImage = + (await fetchAuthorImage(candidateImageUrl, pageUrl).catch(() => null)) ?? undefined; } // Favicon is only fetched when we don't already have an author image — @@ -337,10 +389,13 @@ function hostnameFallback(pageUrl: string): string { } } -/** Pull `` / first `<h1>` out of a full HTML document. */ +/** Pull a title from the document, in declining order of trust: + * OpenGraph / Twitter Card title → `<title>` → first `<h1>` → fallback. */ function extractHtmlTitle(html: string, fallback: string): string { try { const doc = new DOMParser().parseFromString(sanitizeForParsing(html), 'text/html'); + const meta = pickMetaContent(doc, META_FALLBACK.title); + if (meta) return meta; const t = doc.querySelector('title')?.textContent?.trim(); if (t) return t; const h1 = doc.querySelector('h1')?.textContent?.trim(); @@ -467,8 +522,23 @@ export async function convertPageToEpub(html: string, url: string): Promise<Conv 'parse_failed', ); } - const title = parsed.title?.trim() || extractHtmlTitle(html, url); - const byline = parsed.byline?.trim() || ''; + // Readability gives us content reliably but byline misses on sites with + // non-standard author markup (most SPAs, anything class-mangled). Meta + // tags fill the gap — see `META_FALLBACK` in siteRules.ts. + let metaDoc: Document | null = null; + try { + metaDoc = new DOMParser().parseFromString(sanitizeForParsing(html), 'text/html'); + } catch { + /* meta fallback unavailable; keep going with what Readability gave us */ + } + const title = + parsed.title?.trim() || + (metaDoc ? pickMetaContent(metaDoc, META_FALLBACK.title) : null) || + extractHtmlTitle(html, url); + const byline = + parsed.byline?.trim() || + (metaDoc ? pickMetaContent(metaDoc, META_FALLBACK.byline) : null) || + ''; const bundle = await bundleAssets(articleHtml, url); const cover = await buildArticleCover(html, url, title, byline, rule); return htmlToBook( diff --git a/apps/readest-app/src/services/send/conversion/siteRules.ts b/apps/readest-app/src/services/send/conversion/siteRules.ts index 462c0768..777593a2 100644 --- a/apps/readest-app/src/services/send/conversion/siteRules.ts +++ b/apps/readest-app/src/services/send/conversion/siteRules.ts @@ -64,8 +64,63 @@ const RULES: SiteRule[] = [ '.open_in_weixin_wrap', ], }, + { + name: 'x.com', + match: (u) => u.hostname === 'x.com' || u.hostname.endsWith('.x.com'), + content: '[data-testid="twitterArticleReadView"]', + title: '[data-testid="twitter-article-title"]', + // The User-Name container holds both the display name and the @handle. + // The handle's anchor is keyboard-hidden via `tabindex="-1"`; the + // display-name anchor has no tabindex. Picking the non-tabindexed + // anchor lets us grab just "mousepotato" without the "@iluciddreaming" + // suffix. The verified-account SVG sits next to the name but + // contributes no text content, so textContent stays clean. + byline: '[data-testid="User-Name"] a:not([tabindex])', + // `UserAvatar-Container-<handle>` — prefix match keeps the rule + // handle-agnostic. The inner <img> uses `_x96` size variant; the + // cover generator may upscale via URL rewriting. + authorImage: '[data-testid^="UserAvatar-Container-"] img', + strip: [ + // Reply / Repost / Like / Bookmark / Share / Views row at top. + // aria-label is locale-specific, so match by the structural fact + // that this group contains a "reply" button. + '[role="group"]:has([data-testid="reply"])', + ], + }, ]; +/** + * Universal meta-tag fallbacks for fields that a per-site rule can't + * extract — either because no rule matched the host, or because the + * site shipped a frontend redesign and our CSS hooks no longer find + * anything. Every reputable publisher emits these tags for crawlers + * and link unfurlers (Twitter Card, Slack, iMessage previews), so + * they're the most stable fallback short of giving up. + * + * Values live in the `content` attribute, not textContent — read with + * `el.getAttribute('content')`. The lists are ordered by reliability: + * try the first that matches. + * + * No `content` (body) fallback here — Readability already owns that + * path when a site rule's content selector misses, and Readability + * does a better job than any meta tag would. + */ +export const META_FALLBACK = { + title: ['meta[property="og:title"]', 'meta[name="twitter:title"]', 'meta[name="title"]'], + byline: [ + 'meta[name="author"]', + 'meta[property="article:author"]', + 'meta[name="twitter:creator"]', + 'meta[property="og:site_name"]', // last-ditch: at least name the publication + ], + authorImage: [ + 'meta[property="og:image"]', + 'meta[property="og:image:url"]', + 'meta[name="twitter:image"]', + 'meta[name="twitter:image:src"]', + ], +} as const; + /** Find the rule matching `url`, if any. Returns null on parse error. */ export function findSiteRule(url: string): SiteRule | null { try {