3730 lines
148 KiB
Python
3730 lines
148 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import copy
|
||
import datetime as dt
|
||
import hashlib
|
||
import html
|
||
import json
|
||
import os
|
||
import posixpath
|
||
import re
|
||
import shutil
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import webbrowser
|
||
import zipfile
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from typing import Any
|
||
import xml.etree.ElementTree as ET
|
||
|
||
from flask import Flask, jsonify, make_response, request, send_from_directory
|
||
|
||
try:
|
||
from version import VERSION_RULES, version
|
||
except ImportError:
|
||
VERSION_RULES = {
|
||
"PATCH": "修复 bug 或兼容性小修",
|
||
"MINOR": "新增向后兼容能力、API 字段或可选配置",
|
||
"MAJOR": "破坏兼容的 API、配置或行为变更",
|
||
}
|
||
version = "0.13.0"
|
||
|
||
|
||
P_RE = re.compile(r"(<p\b[^>]*>)(.*?)(</p>)", re.S | re.I)
|
||
SOURCE_BLOCK_RE = re.compile(
|
||
r"(?P<open><(?P<tag>p|h[1-6])\b[^>]*>)(?P<inner>.*?)(?P<close></(?P=tag)\s*>)",
|
||
re.S | re.I,
|
||
)
|
||
TAG_RE = re.compile(r"<[^>]+>")
|
||
IMG_TAG_RE = re.compile(r"<(?:img|image)\b[^>]*>", re.S | re.I)
|
||
ATTR_RE = re.compile(r"([:\w-]+)\s*=\s*(['\"])(.*?)\2", re.S)
|
||
JP_RE = re.compile(r"[\u3040-\u30ff\u3400-\u9fff々〆〤]")
|
||
KANA_RE = re.compile(r"[\u3040-\u30ff]")
|
||
JAPANESE_CONTEXT_RE = re.compile(r"[ぁ-ヿ々〆〤ー・「」『』【】()〈〉《》〔〕〜~…!?。]")
|
||
GRAY_RE = re.compile(
|
||
r"(?:color\s*:\s*(?:#?808080|gray|grey|rgb\(\s*128\s*,\s*128\s*,\s*128\s*\))|class\s*=\s*['\"][^'\"]*(?:gray|grey|jp|source)[^'\"]*['\"])",
|
||
re.I,
|
||
)
|
||
EXTERNAL_URI_RE = re.compile(r"^(?:[a-z][a-z0-9+.-]*:|//)", re.I)
|
||
IMAGE_EXTENSIONS = {".apng", ".avif", ".bmp", ".gif", ".jpeg", ".jpg", ".png", ".svg", ".webp"}
|
||
RASTER_IMAGE_EXTENSIONS = {".apng", ".avif", ".bmp", ".gif", ".jpeg", ".jpg", ".png", ".webp"}
|
||
|
||
DEFAULT_REVIEW_ROOT = "epub_review_sessions"
|
||
STATE_VERSION = 1
|
||
UPLOAD_DIR_NAME = "uploads"
|
||
DEFAULT_GPT_BASE_URL = "https://api.openai.com/v1"
|
||
DEFAULT_GPT_MODEL = "gpt-4o-mini"
|
||
DEFAULT_TRANSLATION_PROMPT = (
|
||
"你是日语轻小说到简体中文的审校重翻助手。只重翻当前目标语句/段落。"
|
||
"目标是自然、流畅、忠实的简体中文轻小说文风。意义优先,再调整为中文语序;"
|
||
"不要贴着日语句序硬译,也不要补写原文没有的信息。"
|
||
"术语、人名、地名、TRPG/系统词只在本段命中时服从给定术语;没有命中的术语不要写进译文。"
|
||
)
|
||
LEGACY_TRANSLATION_PROMPT_WITHOUT_GLOBAL_TERM = (
|
||
"你是日语轻小说到简体中文的审校重翻助手。目标是把当前目标语句译成自然、流畅、忠实的简体中文轻小说文风。"
|
||
"意义优先,再调整为中文语序;不要贴着日语句序硬译,也不要补写原文没有的信息。"
|
||
"术语、人名、地名、TRPG/系统词优先服从给定名词表。"
|
||
)
|
||
DEFAULT_FORMAT_PROMPT = (
|
||
"只输出目标语句的新中文译文 HTML 片段,不要解释、不要 Markdown、不要代码围栏。"
|
||
"不得合并或拆分段落。保留必要的 HTML 内联标签,尤其是有意义的 ruby,必须写成"
|
||
"<ruby><rb>中文</rb><rt>读音/外语</rt></ruby>,不要改成括号。"
|
||
"标点统一为简中出版格式:对话引号用「」;省略号用……;破折号用——;问号用?,感叹号用!,问叹连用用?!或!?;"
|
||
"逗号、句号、顿号、冒号、分号用,。、:;。不要用英文半角标点替代中文标点。"
|
||
)
|
||
DEFAULT_CHARACTER_PROMPT = (
|
||
"根据目标语句和少量前后文判断说话人、情绪和角色口吻。"
|
||
"本系列以第一人称轻小说叙述为主,主角聪明、爱吐槽、有TRPG玩家思维;对话要符合身份差异,"
|
||
"不要把所有角色写成同一种口吻,也不要过度网络化。"
|
||
)
|
||
GPT_PROMPT_DEFAULTS = {
|
||
"translation_prompt": DEFAULT_TRANSLATION_PROMPT,
|
||
"format_prompt": DEFAULT_FORMAT_PROMPT,
|
||
"character_prompt": DEFAULT_CHARACTER_PROMPT,
|
||
}
|
||
GPT_PROMPT_MAX_CHARS = {
|
||
"translation_prompt": 5000,
|
||
"format_prompt": 5000,
|
||
"character_prompt": 3000,
|
||
}
|
||
GLOSSARY_ENTRY_LIMIT = 80
|
||
GLOSSARY_PROMPT_LIMIT = 16
|
||
TRANSLATION_JOB_DIR_NAME = "translation_jobs"
|
||
TRANSLATION_OUTPUT_DIR_NAME = "translated_outputs"
|
||
TRANSLATION_LOG_LIMIT = 80
|
||
DEFAULT_TRANSLATION_LIMIT = 20
|
||
TRANSLATION_RANGE_LIMIT_MAX = 5000
|
||
TRANSLATION_CONTEXT_RADIUS = 1
|
||
LIBRARY_DB_NAME = "library.json"
|
||
|
||
|
||
def now_iso() -> str:
|
||
return dt.datetime.now(dt.timezone.utc).astimezone().isoformat(timespec="seconds")
|
||
|
||
|
||
def safe_slug(value: str, max_len: int = 80) -> str:
|
||
value = re.sub(r'[\\/:*?"<>|]+', "_", value).strip().strip(".")
|
||
value = re.sub(r"\s+", "_", value)
|
||
return value[:max_len] or "epub"
|
||
|
||
|
||
def find_free_port(start: int, host: str = "127.0.0.1") -> int:
|
||
connect_host = "127.0.0.1" if host in {"localhost", "0.0.0.0", "::"} else host
|
||
bind_host = "127.0.0.1" if host in {"localhost", "::"} else host
|
||
port_in_use = getattr(socket, "SO_EXCLUSIVEADDRUSE", None)
|
||
for port in range(start, start + 100):
|
||
try:
|
||
with socket.create_connection((connect_host, port), timeout=0.2):
|
||
continue
|
||
except OSError:
|
||
pass
|
||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||
if port_in_use is not None:
|
||
sock.setsockopt(socket.SOL_SOCKET, port_in_use, 1)
|
||
try:
|
||
sock.bind((bind_host, port))
|
||
except OSError:
|
||
continue
|
||
return port
|
||
raise RuntimeError(f"no free port found from {start}")
|
||
|
||
|
||
def display_host(host: str) -> str:
|
||
return "localhost" if host in {"127.0.0.1", "0.0.0.0", "::"} else host
|
||
|
||
|
||
def read_json(path: Path, default: Any) -> Any:
|
||
if not path.exists():
|
||
return copy.deepcopy(default)
|
||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||
|
||
|
||
def write_json(path: Path, data: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
|
||
def write_json_atomic(path: Path, data: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_name(f"{path.name}.tmp")
|
||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
tmp.replace(path)
|
||
|
||
|
||
def append_jsonl(path: Path, record: dict[str, Any]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("a", encoding="utf-8", newline="\n") as fh:
|
||
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def strip_tags(fragment: str) -> str:
|
||
fragment = re.sub(r"<rt\b[^>]*>.*?</rt>", "", fragment, flags=re.S | re.I)
|
||
return html.unescape(TAG_RE.sub("", fragment)).strip()
|
||
|
||
|
||
def strip_html_keep_ruby_text(fragment: str) -> str:
|
||
return html.unescape(TAG_RE.sub("", fragment or "")).strip()
|
||
|
||
|
||
def compact_text(value: str) -> str:
|
||
return re.sub(r"\s+", " ", html.unescape(value or "")).strip()
|
||
|
||
|
||
def file_sort_key(value: str) -> list[Any]:
|
||
parts = re.split(r"(\d+)", value.lower())
|
||
return [int(part) if part.isdigit() else part for part in parts]
|
||
|
||
|
||
def entry_dir(entry_name: str) -> str:
|
||
directory = posixpath.dirname(entry_name)
|
||
return f"{directory}/" if directory else ""
|
||
|
||
|
||
def normalize_href(base_dir: str, href: str) -> str:
|
||
href = urllib.parse.unquote((href or "").split("#", 1)[0])
|
||
if not href:
|
||
return ""
|
||
return posixpath.normpath(posixpath.join(base_dir, href)).lstrip("./")
|
||
|
||
|
||
def normalize_href_with_fragment(base_dir: str, href: str) -> tuple[str, str]:
|
||
href = urllib.parse.unquote(href or "")
|
||
path, fragment = (href.split("#", 1) + [""])[:2] if "#" in href else (href, "")
|
||
path = posixpath.normpath(posixpath.join(base_dir, path)).lstrip("./") if path else ""
|
||
return path, fragment
|
||
|
||
|
||
def read_xml_root(path: Path) -> ET.Element | None:
|
||
if not path.exists():
|
||
return None
|
||
try:
|
||
return ET.fromstring(path.read_text(encoding="utf-8", errors="replace"))
|
||
except ET.ParseError:
|
||
try:
|
||
return ET.fromstring(path.read_bytes())
|
||
except ET.ParseError:
|
||
return None
|
||
|
||
|
||
def local_name(tag: str) -> str:
|
||
return tag.rsplit("}", 1)[-1].lower()
|
||
|
||
|
||
def element_text(elem: ET.Element) -> str:
|
||
return compact_text("".join(elem.itertext()))
|
||
|
||
|
||
def is_japaneseish_fragment(fragment: str) -> bool:
|
||
return bool(KANA_RE.search(fragment or ""))
|
||
|
||
|
||
def is_japaneseish(fragment: str) -> bool:
|
||
text = strip_tags(fragment)
|
||
return bool(KANA_RE.search(text))
|
||
|
||
|
||
def is_gray_source(attrs: str, inner: str) -> bool:
|
||
if GRAY_RE.search(attrs):
|
||
return True
|
||
return is_japaneseish(inner)
|
||
|
||
|
||
def paragraph_visible_text(inner: str) -> str:
|
||
return compact_text(strip_tags(inner or ""))
|
||
|
||
|
||
def paragraph_prompt_text(inner: str) -> str:
|
||
return strip_tags(inner or "").strip()
|
||
|
||
|
||
def html_declares_japanese(text: str) -> bool:
|
||
return bool(re.search(r"\b(?:xml:)?lang\s*=\s*['\"]ja(?:[-_'\"]|$)", text or "", flags=re.I))
|
||
|
||
|
||
def source_file_has_japanese_context(text: str) -> bool:
|
||
return html_declares_japanese(text) or bool(KANA_RE.search(strip_html_keep_ruby_text(text or "")))
|
||
|
||
|
||
def cjk_core_text(text: str) -> str:
|
||
return re.sub(r"[^\u3400-\u9fff々〆〤]", "", text or "")
|
||
|
||
|
||
def is_translatable_source_block(inner: str, tag: str = "p", file_has_japanese_context: bool = False) -> bool:
|
||
visible = paragraph_visible_text(inner)
|
||
if not visible:
|
||
return False
|
||
if IMG_TAG_RE.search(inner) and not visible:
|
||
return False
|
||
prompt_text = strip_html_keep_ruby_text(inner)
|
||
if KANA_RE.search(prompt_text):
|
||
return True
|
||
if not JP_RE.search(prompt_text):
|
||
return False
|
||
if tag.lower().startswith("h") and file_has_japanese_context and JP_RE.search(prompt_text):
|
||
return True
|
||
if file_has_japanese_context and JAPANESE_CONTEXT_RE.search(prompt_text):
|
||
return True
|
||
return False
|
||
|
||
|
||
def is_translatable_source_paragraph(inner: str) -> bool:
|
||
return is_translatable_source_block(inner, "p", False)
|
||
|
||
|
||
class HtmlSanitizer(HTMLParser):
|
||
allowed_tags = {
|
||
"a",
|
||
"b",
|
||
"br",
|
||
"code",
|
||
"em",
|
||
"i",
|
||
"mark",
|
||
"rb",
|
||
"rp",
|
||
"rt",
|
||
"ruby",
|
||
"small",
|
||
"span",
|
||
"strong",
|
||
"sub",
|
||
"sup",
|
||
}
|
||
allowed_attrs = {
|
||
"a": {"href", "title"},
|
||
"mark": {"class", "data-review-mark"},
|
||
"span": {"class", "style", "data-review-mark"},
|
||
"ruby": {"class"},
|
||
"rt": {"class"},
|
||
"rb": {"class"},
|
||
"rp": {"class"},
|
||
}
|
||
void_tags = {"br"}
|
||
style_safe = re.compile(r"^(?:\s*(?:color|background-color|font-weight|font-style)\s*:\s*[-#(),.%\w\s]+\s*;?)*$", re.I)
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=False)
|
||
self.parts: list[str] = []
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
tag = tag.lower()
|
||
if tag not in self.allowed_tags:
|
||
return
|
||
safe_attrs = []
|
||
for key, value in attrs:
|
||
key = key.lower()
|
||
value = value or ""
|
||
if key not in self.allowed_attrs.get(tag, set()):
|
||
continue
|
||
if key.startswith("on"):
|
||
continue
|
||
if key == "href" and re.match(r"\s*(?:javascript|data):", value, re.I):
|
||
continue
|
||
if key == "style" and not self.style_safe.fullmatch(value):
|
||
continue
|
||
safe_attrs.append(f'{key}="{html.escape(value, quote=True)}"')
|
||
attr_text = (" " + " ".join(safe_attrs)) if safe_attrs else ""
|
||
if tag in self.void_tags:
|
||
self.parts.append(f"<{tag}{attr_text}/>")
|
||
else:
|
||
self.parts.append(f"<{tag}{attr_text}>")
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
tag = tag.lower()
|
||
if tag in self.allowed_tags and tag not in self.void_tags:
|
||
self.parts.append(f"</{tag}>")
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
self.parts.append(html.escape(data, quote=False))
|
||
|
||
def handle_entityref(self, name: str) -> None:
|
||
self.parts.append(f"&{name};")
|
||
|
||
def handle_charref(self, name: str) -> None:
|
||
self.parts.append(f"&#{name};")
|
||
|
||
|
||
def sanitize_cn_html(value: str) -> str:
|
||
value = re.sub(r"(?is)<\s*(script|style|iframe|object|embed)\b.*?</\s*\1\s*>", "", value)
|
||
sanitizer = HtmlSanitizer()
|
||
sanitizer.feed(value)
|
||
sanitizer.close()
|
||
return "".join(sanitizer.parts).strip()
|
||
|
||
|
||
def normalize_gpt_base_url(value: str) -> str:
|
||
value = (value or DEFAULT_GPT_BASE_URL).strip().rstrip("/")
|
||
if not value:
|
||
value = DEFAULT_GPT_BASE_URL
|
||
parsed = urllib.parse.urlparse(value)
|
||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||
raise ValueError("Base URL 必须是 http(s) 地址")
|
||
return value
|
||
|
||
|
||
def normalize_gpt_prompt(name: str, value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
text = GPT_PROMPT_DEFAULTS[name]
|
||
if name == "translation_prompt":
|
||
text = normalize_translation_prompt(text)
|
||
return text[: GPT_PROMPT_MAX_CHARS[name]]
|
||
|
||
|
||
def gpt_prompt_defaults() -> dict[str, str]:
|
||
return dict(GPT_PROMPT_DEFAULTS)
|
||
|
||
|
||
def compact_prompt_text(text: str) -> str:
|
||
return re.sub(r"\s+", "", text or "")
|
||
|
||
|
||
def normalize_translation_prompt(text: str) -> str:
|
||
text = strip_obsolete_global_term_rules(text)
|
||
if not text:
|
||
return DEFAULT_TRANSLATION_PROMPT
|
||
if compact_prompt_text(text) == compact_prompt_text(LEGACY_TRANSLATION_PROMPT_WITHOUT_GLOBAL_TERM):
|
||
return DEFAULT_TRANSLATION_PROMPT
|
||
scoped_clause = "术语、人名、地名、TRPG/系统词只在本段命中时服从给定术语;没有命中的术语不要写进译文。"
|
||
text = re.sub(
|
||
r"术语、人名、地名、TRPG/系统词优先服从给定名词表[。;;]?",
|
||
scoped_clause,
|
||
text,
|
||
)
|
||
return text
|
||
|
||
|
||
def strip_obsolete_global_term_rules(text: str) -> str:
|
||
"""Old saved prompts hard-coded one term into every request; keep terms scoped."""
|
||
parts = re.split(r"([。!?\n;;]+)", text)
|
||
kept: list[str] = []
|
||
for idx in range(0, len(parts), 2):
|
||
segment = parts[idx]
|
||
delimiter = parts[idx + 1] if idx + 1 < len(parts) else ""
|
||
has_falchion = re.search(r"Falchion|ファルシオン", segment, flags=re.I)
|
||
has_translation = "大砍刀" in segment or "偃月刀" in segment
|
||
if has_falchion and has_translation:
|
||
continue
|
||
kept.append(segment + delimiter)
|
||
text = "".join(kept)
|
||
text = re.sub(r"[;;]\s*([。!?])", r"\1", text)
|
||
text = re.sub(r"[;;]\s*$", "。", text)
|
||
text = re.sub(r"。{2,}", "。", text)
|
||
return text.strip()
|
||
|
||
|
||
def project_root() -> Path:
|
||
return Path(__file__).resolve().parents[2]
|
||
|
||
|
||
def default_glossary_path() -> Path:
|
||
root = project_root()
|
||
candidates = sorted(
|
||
path
|
||
for path in root.glob("*/mingcibiao.json")
|
||
if "epub_review_sessions" not in path.parts
|
||
)
|
||
if len(candidates) == 1:
|
||
return candidates[0]
|
||
return root / "mingcibiao.json"
|
||
|
||
|
||
def normalize_glossary_path_value(value: str = "") -> str:
|
||
raw = str(value or "").strip()
|
||
path = Path(raw).expanduser() if raw else default_glossary_path()
|
||
if not path.is_absolute():
|
||
path = project_root() / path
|
||
if path.suffix.lower() != ".json":
|
||
raise ValueError("术语表路径必须指向 .json 文件")
|
||
return str(path.resolve())
|
||
|
||
|
||
def update_gpt_config_fields(review_root: Path, fields: dict[str, Any]) -> None:
|
||
existing = read_json(gpt_config_path(review_root), {})
|
||
if not isinstance(existing, dict):
|
||
existing = {}
|
||
existing.update(fields)
|
||
existing["updated_at"] = now_iso()
|
||
write_json(gpt_config_path(review_root), existing)
|
||
|
||
|
||
def read_gpt_config(review_root: Path) -> dict[str, Any]:
|
||
data = read_json(gpt_config_path(review_root), {})
|
||
api_key = str(data.get("api_key") or os.environ.get("OPENAI_API_KEY") or "").strip()
|
||
base_url = str(data.get("base_url") or os.environ.get("OPENAI_BASE_URL") or DEFAULT_GPT_BASE_URL).strip()
|
||
model = str(data.get("model") or os.environ.get("OPENAI_MODEL") or DEFAULT_GPT_MODEL).strip()
|
||
prompts = {
|
||
name: normalize_gpt_prompt(name, data.get(name, default))
|
||
for name, default in GPT_PROMPT_DEFAULTS.items()
|
||
}
|
||
return {
|
||
"base_url": normalize_gpt_base_url(base_url),
|
||
"model": model or DEFAULT_GPT_MODEL,
|
||
"api_key": api_key,
|
||
"configured": bool(api_key),
|
||
"key_source": "config" if str(data.get("api_key") or "").strip() else ("env" if os.environ.get("OPENAI_API_KEY") else ""),
|
||
"updated_at": data.get("updated_at", ""),
|
||
"glossary_path": normalize_glossary_path_value(str(data.get("glossary_path") or "")),
|
||
**prompts,
|
||
}
|
||
|
||
|
||
def public_gpt_config(review_root: Path) -> dict[str, Any]:
|
||
config = read_gpt_config(review_root)
|
||
return {
|
||
"configured": config["configured"],
|
||
"base_url": config["base_url"],
|
||
"model": config["model"],
|
||
"key_source": config["key_source"],
|
||
"updated_at": config.get("updated_at", ""),
|
||
"translation_prompt": config["translation_prompt"],
|
||
"format_prompt": config["format_prompt"],
|
||
"character_prompt": config["character_prompt"],
|
||
"glossary_path": config["glossary_path"],
|
||
"prompt_defaults": gpt_prompt_defaults(),
|
||
}
|
||
|
||
|
||
def save_gpt_config(review_root: Path, data: dict[str, Any]) -> dict[str, Any]:
|
||
base_url = normalize_gpt_base_url(str(data.get("base_url") or DEFAULT_GPT_BASE_URL))
|
||
model = str(data.get("model") or DEFAULT_GPT_MODEL).strip() or DEFAULT_GPT_MODEL
|
||
api_key = str(data.get("api_key") or "").strip()
|
||
keep_existing = bool(data.get("keep_existing", False))
|
||
existing = read_json(gpt_config_path(review_root), {})
|
||
if keep_existing and not api_key:
|
||
api_key = str(existing.get("api_key") or "").strip()
|
||
prompts = {
|
||
name: normalize_gpt_prompt(
|
||
name,
|
||
data.get(name) if name in data else existing.get(name, default),
|
||
)
|
||
for name, default in GPT_PROMPT_DEFAULTS.items()
|
||
}
|
||
record = {
|
||
"base_url": base_url,
|
||
"model": model,
|
||
"api_key": api_key,
|
||
"glossary_path": normalize_glossary_path_value(str(data.get("glossary_path") or existing.get("glossary_path") or "")),
|
||
**prompts,
|
||
"updated_at": now_iso(),
|
||
}
|
||
write_json(gpt_config_path(review_root), record)
|
||
return public_gpt_config(review_root)
|
||
|
||
|
||
def glossary_path(review_root: Path) -> Path:
|
||
data = read_json(gpt_config_path(review_root), {})
|
||
configured = str(data.get("glossary_path") or "").strip()
|
||
return Path(normalize_glossary_path_value(configured))
|
||
|
||
|
||
def read_glossary(review_root: Path) -> dict[str, str]:
|
||
path = glossary_path(review_root)
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
data = read_json(path, {})
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError(f"术语表 JSON 解析失败:{exc}") from exc
|
||
if not isinstance(data, dict):
|
||
raise ValueError("术语表必须是 JSON 对象")
|
||
glossary: dict[str, str] = {}
|
||
for key, value in data.items():
|
||
if isinstance(key, str) and isinstance(value, str):
|
||
glossary[key] = value
|
||
return glossary
|
||
|
||
|
||
def public_glossary(review_root: Path) -> dict[str, Any]:
|
||
path = glossary_path(review_root)
|
||
glossary = read_glossary(review_root)
|
||
entries = [
|
||
{
|
||
"source": key,
|
||
"target": value,
|
||
"clean_target": value.split("#", 1)[0].strip(),
|
||
}
|
||
for key, value in glossary.items()
|
||
]
|
||
return {
|
||
"path": str(path),
|
||
"exists": path.exists(),
|
||
"count": len(entries),
|
||
"entries": entries,
|
||
"updated_at": dt.datetime.fromtimestamp(path.stat().st_mtime).astimezone().isoformat(timespec="seconds") if path.exists() else "",
|
||
}
|
||
|
||
|
||
def save_glossary(review_root: Path, entries: list[dict[str, Any]]) -> dict[str, Any]:
|
||
path = glossary_path(review_root)
|
||
if path.exists() and not path.is_file():
|
||
raise ValueError("术语表路径不是文件")
|
||
glossary: dict[str, str] = {}
|
||
for entry in entries:
|
||
if not isinstance(entry, dict):
|
||
raise ValueError("术语条目必须是对象")
|
||
source = str(entry.get("source") or "").strip()
|
||
target = str(entry.get("target") or "").strip()
|
||
if not source:
|
||
continue
|
||
if not target:
|
||
raise ValueError(f"术语“{source}”缺少译名")
|
||
if len(source) > 300:
|
||
raise ValueError(f"术语“{source[:80]}...”过长")
|
||
if len(target) > 1200:
|
||
raise ValueError(f"术语“{source}”的译名/备注过长")
|
||
if source in glossary:
|
||
raise ValueError(f"术语“{source}”重复")
|
||
glossary[source] = target
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
write_json(path, glossary)
|
||
return public_glossary(review_root)
|
||
|
||
|
||
def search_glossary_entries(glossary: dict[str, str], term: str, limit: int = GLOSSARY_ENTRY_LIMIT) -> list[str]:
|
||
needle = term.strip().lower()
|
||
if not needle:
|
||
return []
|
||
selected: list[str] = []
|
||
for key, value in glossary.items():
|
||
clean_value = value.split("#", 1)[0].strip()
|
||
if needle in key.lower() or needle in value.lower():
|
||
selected.append(f"{key} => {clean_value or value}")
|
||
if len(selected) >= limit:
|
||
break
|
||
return selected
|
||
|
||
|
||
def parenthetical_aliases(value: str) -> list[str]:
|
||
aliases: list[str] = []
|
||
for match in re.finditer(r"\(([^()]{1,80})\)|(([^()]{1,80}))", value):
|
||
alias = (match.group(1) or match.group(2) or "").strip()
|
||
if alias:
|
||
aliases.append(alias)
|
||
return aliases
|
||
|
||
|
||
def strip_parenthetical_aliases(value: str) -> str:
|
||
return re.sub(r"\([^()]{1,80}\)|([^()]{1,80})", "", value).strip()
|
||
|
||
|
||
def glossary_match_variants(key: str, value: str) -> list[str]:
|
||
variants: list[str] = []
|
||
key = key.strip()
|
||
clean_value = value.split("#", 1)[0].strip()
|
||
if key:
|
||
variants.append(key)
|
||
variants.extend(parenthetical_aliases(key))
|
||
variants.extend(parenthetical_aliases(clean_value))
|
||
deduped: list[str] = []
|
||
for item in variants:
|
||
if len(item) < 2 or item in deduped:
|
||
continue
|
||
deduped.append(item)
|
||
return deduped
|
||
|
||
|
||
def glossary_prompt_line(key: str, value: str, source: str) -> str:
|
||
clean_value = value.split("#", 1)[0].strip()
|
||
value_aliases = parenthetical_aliases(clean_value)
|
||
keep_parenthetical = bool(parenthetical_aliases(key)) or any(alias in source for alias in value_aliases)
|
||
translation = clean_value if keep_parenthetical else strip_parenthetical_aliases(clean_value) or clean_value
|
||
return f"{key} => {translation}"
|
||
|
||
|
||
def glossary_matches_for_prompt(review_root: Path, row: dict[str, Any], limit: int = GLOSSARY_PROMPT_LIMIT) -> list[str]:
|
||
try:
|
||
glossary = read_glossary(review_root)
|
||
except ValueError:
|
||
raise
|
||
except Exception as exc:
|
||
raise ValueError(f"读取术语表失败:{exc}") from exc
|
||
if not glossary:
|
||
return []
|
||
source = "\n".join(
|
||
[
|
||
str(row.get("jp_text") or ""),
|
||
strip_html_keep_ruby_text(str(row.get("jp_html") or "")),
|
||
]
|
||
)
|
||
selected: list[str] = []
|
||
for key, value in glossary.items():
|
||
if not isinstance(key, str) or not isinstance(value, str):
|
||
continue
|
||
clean_value = value.split("#", 1)[0].strip()
|
||
if not clean_value:
|
||
continue
|
||
if any(variant in source for variant in glossary_match_variants(key, value)):
|
||
selected.append(glossary_prompt_line(key, value, source))
|
||
if len(selected) >= limit:
|
||
break
|
||
return selected[:limit]
|
||
|
||
|
||
def sanitize_source_html_for_prompt(fragment: str) -> str:
|
||
allowed = {"p", "br", "ruby", "rb", "rt", "rp", "span", "em", "strong", "b", "i"}
|
||
|
||
def clean_tag(match: re.Match[str]) -> str:
|
||
raw = match.group(0)
|
||
closing = raw.startswith("</")
|
||
name_match = re.match(r"</?\s*([a-zA-Z0-9:-]+)", raw)
|
||
if not name_match:
|
||
return ""
|
||
name = name_match.group(1).lower()
|
||
if name not in allowed:
|
||
return ""
|
||
if closing:
|
||
return f"</{name}>"
|
||
if name == "br" or raw.rstrip().endswith("/>"):
|
||
return f"<{name} />"
|
||
return f"<{name}>"
|
||
|
||
return re.sub(r"<[^>]+>", clean_tag, fragment or "")
|
||
|
||
|
||
def glossary_mapping_from_data(data: Any) -> dict[str, str]:
|
||
glossary: dict[str, str] = {}
|
||
if isinstance(data, dict) and isinstance(data.get("terms"), list):
|
||
for item in data.get("terms", []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
source = str(item.get("source") or "").strip()
|
||
target = str(item.get("target") or "").strip()
|
||
if not source or not target:
|
||
continue
|
||
target_rt = str(item.get("target_rt") or "").strip()
|
||
mode = str(item.get("mode") or "").strip()
|
||
value = f"{target}({target_rt})" if target_rt and mode == "ruby" else target
|
||
glossary[source] = value
|
||
aliases = item.get("aliases") or []
|
||
if isinstance(aliases, list):
|
||
for alias in aliases:
|
||
alias_text = str(alias or "").strip()
|
||
if alias_text and alias_text not in glossary:
|
||
glossary[alias_text] = value
|
||
rt = str(item.get("rt") or "").strip()
|
||
if rt and rt not in glossary:
|
||
glossary[rt] = value
|
||
return glossary
|
||
if isinstance(data, dict):
|
||
for key, value in data.items():
|
||
if isinstance(key, str) and isinstance(value, str):
|
||
glossary[key] = value
|
||
return glossary
|
||
|
||
|
||
def read_glossary_mapping_from_path(path: Path) -> dict[str, str]:
|
||
if not str(path):
|
||
return {}
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
data = read_json(path, {})
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError(f"术语表 JSON 解析失败:{exc}") from exc
|
||
return glossary_mapping_from_data(data)
|
||
|
||
|
||
def glossary_matches_for_source(
|
||
glossary: dict[str, str],
|
||
source_html: str,
|
||
source_text: str = "",
|
||
limit: int = GLOSSARY_PROMPT_LIMIT,
|
||
) -> list[str]:
|
||
if not glossary:
|
||
return []
|
||
source = "\n".join([str(source_text or ""), strip_html_keep_ruby_text(source_html or "")])
|
||
selected: list[str] = []
|
||
for key, value in glossary.items():
|
||
if not isinstance(key, str) or not isinstance(value, str):
|
||
continue
|
||
clean_value = value.split("#", 1)[0].strip()
|
||
if not clean_value:
|
||
continue
|
||
if any(variant in source for variant in glossary_match_variants(key, value)):
|
||
selected.append(glossary_prompt_line(key, value, source))
|
||
if len(selected) >= limit:
|
||
break
|
||
return selected[:limit]
|
||
|
||
|
||
def truncate_prompt_text(value: Any, limit: int = 1600) -> str:
|
||
text = compact_text(str(value or ""))
|
||
if len(text) <= limit:
|
||
return text
|
||
return text[: limit - 1].rstrip() + "…"
|
||
|
||
|
||
def row_translation_html(row: dict[str, Any]) -> str:
|
||
return str(row.get("current_html") or row.get("cn_html") or row.get("cn_text") or "")
|
||
|
||
|
||
def row_source_html(row: dict[str, Any]) -> str:
|
||
return str(row.get("jp_html") or row.get("jp_text") or "")
|
||
|
||
|
||
def row_source_html_for_prompt(row: dict[str, Any]) -> str:
|
||
allowed = {"p", "br", "ruby", "rb", "rt", "rp", "span", "em", "strong", "b", "i"}
|
||
|
||
def clean_tag(match: re.Match[str]) -> str:
|
||
raw = match.group(0)
|
||
closing = raw.startswith("</")
|
||
name_match = re.match(r"</?\s*([a-zA-Z0-9:-]+)", raw)
|
||
if not name_match:
|
||
return ""
|
||
name = name_match.group(1).lower()
|
||
if name not in allowed:
|
||
return ""
|
||
if closing:
|
||
return f"</{name}>"
|
||
if name == "br" or raw.rstrip().endswith("/>"):
|
||
return f"<{name} />"
|
||
return f"<{name}>"
|
||
|
||
return re.sub(r"<[^>]+>", clean_tag, row_source_html(row))
|
||
|
||
|
||
def build_retranslate_context(row: dict[str, Any], rows: list[dict[str, Any]], radius: int = 1) -> list[str]:
|
||
same_file_rows = [item for item in rows if item.get("file") == row.get("file")]
|
||
index = next((idx for idx, item in enumerate(same_file_rows) if item.get("id") == row.get("id")), -1)
|
||
if index < 0:
|
||
return []
|
||
start = max(0, index - radius)
|
||
end = min(len(same_file_rows), index + radius + 1)
|
||
context: list[str] = []
|
||
for idx in range(start, end):
|
||
item = same_file_rows[idx]
|
||
if item.get("id") == row.get("id"):
|
||
continue
|
||
direction = "前文" if idx < index else "后文"
|
||
context.append(f"{direction} {item.get('id', '')}: {truncate_prompt_text(item.get('jp_text'), 260)}")
|
||
return context
|
||
|
||
|
||
def normalize_candidate_punctuation(fragment: str) -> str:
|
||
protected: list[str] = []
|
||
|
||
def protect(match: re.Match[str]) -> str:
|
||
protected.append(match.group(0))
|
||
return f"\uE000{len(protected) - 1}\uE001"
|
||
|
||
protected_pattern = (
|
||
r"<[^>]+>|"
|
||
r"&(?:#[0-9]+|#x[0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]+);|"
|
||
r"https?://[^\s<>\"]+|"
|
||
r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|"
|
||
r"\b\d+(?:[.,]\d+)+\b|"
|
||
r"\b[A-Za-z0-9]+(?:\.[A-Za-z0-9]+)+\b"
|
||
)
|
||
text = re.sub(protected_pattern, protect, fragment)
|
||
text = text.replace("“", "「").replace("”", "」")
|
||
text = text.replace("『", "「").replace("』", "」")
|
||
text = text.replace("...", "……").replace("…", "……")
|
||
text = re.sub(r"(?:……)+", "……", text)
|
||
text = re.sub(r"[-]{2,}", "——", text)
|
||
text = text.translate(str.maketrans({
|
||
",": ",",
|
||
".": "。",
|
||
":": ":",
|
||
";": ";",
|
||
}))
|
||
text = text.replace("?", "?").replace("!", "!")
|
||
text = re.sub(r"?+!+", "?!", text)
|
||
text = re.sub(r"!+?+", "!?", text)
|
||
text = re.sub(r"?{2,}", "?", text)
|
||
text = re.sub(r"!{2,}", "!", text)
|
||
|
||
def restore(match: re.Match[str]) -> str:
|
||
return protected[int(match.group(1))]
|
||
|
||
return re.sub(r"\uE000(\d+)\uE001", restore, text)
|
||
|
||
|
||
def restore_glossary_ruby_candidates(fragment: str, glossary_lines: list[str]) -> str:
|
||
pairs: list[tuple[str, str]] = []
|
||
for line in glossary_lines:
|
||
value = line.split("=>", 1)[1].strip() if "=>" in line else line
|
||
match = re.fullmatch(r"(.+?)\(([^()<>]{1,80})\)", value)
|
||
if match:
|
||
rb, rt = match.group(1).strip(), match.group(2).strip()
|
||
if rb and rt and len(rb) <= 80:
|
||
pairs.append((rb, rt))
|
||
if not pairs:
|
||
return fragment
|
||
protected: list[str] = []
|
||
|
||
def protect(match: re.Match[str]) -> str:
|
||
protected.append(match.group(0))
|
||
return f"\uE010{len(protected) - 1}\uE011"
|
||
|
||
text = re.sub(r"<ruby\b.*?</ruby>|<[^>]+>", protect, fragment, flags=re.S | re.I)
|
||
for rb, rt in sorted(pairs, key=lambda item: len(item[0]), reverse=True):
|
||
pattern = re.escape(f"{rb}({rt})")
|
||
ruby = f"<ruby><rb>{html.escape(rb)}</rb><rt>{html.escape(rt)}</rt></ruby>"
|
||
text = re.sub(pattern, ruby, text)
|
||
|
||
def restore(match: re.Match[str]) -> str:
|
||
return protected[int(match.group(1))]
|
||
|
||
return re.sub(r"\uE010(\d+)\uE011", restore, text)
|
||
|
||
|
||
def build_retranslate_messages(
|
||
row: dict[str, Any],
|
||
rows: list[dict[str, Any]],
|
||
config: dict[str, Any],
|
||
glossary_lines: list[str],
|
||
instruction: str = "",
|
||
) -> list[dict[str, str]]:
|
||
glossary_text = "\n".join(f"- {item}" for item in glossary_lines)
|
||
context_lines = build_retranslate_context(row, rows)
|
||
system_prompt = "\n\n".join(
|
||
[
|
||
"你是日语轻小说到简体中文的审校重翻助手。只处理一个目标语句/段落。",
|
||
"只输出新的简体中文译文 HTML 片段,不解释,不输出 Markdown。",
|
||
"不得合并、拆分、补写或删除目标段落信息。",
|
||
]
|
||
)
|
||
user_parts = [
|
||
"【翻译内容提示词】",
|
||
str(config.get("translation_prompt") or DEFAULT_TRANSLATION_PROMPT).strip(),
|
||
"",
|
||
"【目标语句:日文原文 HTML】",
|
||
row_source_html_for_prompt(row),
|
||
]
|
||
if glossary_text:
|
||
user_parts.extend(["", "【本段命中术语】", glossary_text])
|
||
if context_lines:
|
||
user_parts.extend(["", "【少量上下文】", "\n".join(f"- {item}" for item in context_lines)])
|
||
user_parts.extend(
|
||
[
|
||
"",
|
||
"【角色状态与口吻提示词】",
|
||
str(config.get("character_prompt") or DEFAULT_CHARACTER_PROMPT).strip(),
|
||
]
|
||
)
|
||
user_parts.extend(
|
||
[
|
||
"【格式与标点提示词】",
|
||
str(config.get("format_prompt") or DEFAULT_FORMAT_PROMPT).strip(),
|
||
]
|
||
)
|
||
if instruction.strip():
|
||
user_parts.extend(["", "【本次额外要求】", instruction.strip()[:800]])
|
||
user_parts.extend(
|
||
[
|
||
"",
|
||
"请只输出目标语句的新中文译文 HTML 片段。",
|
||
]
|
||
)
|
||
return [
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": "\n".join(user_parts)},
|
||
]
|
||
|
||
|
||
def call_openai_compatible_chat(
|
||
config: dict[str, Any],
|
||
messages: list[dict[str, str]],
|
||
temperature: float = 0.2,
|
||
timeout: int = 90,
|
||
) -> str:
|
||
api_key = str(config.get("api_key") or "").strip()
|
||
if not api_key:
|
||
raise ValueError("尚未配置 GPT API Key")
|
||
base_url = normalize_gpt_base_url(str(config.get("base_url") or DEFAULT_GPT_BASE_URL))
|
||
url = f"{base_url}/chat/completions"
|
||
payload = {
|
||
"model": str(config.get("model") or DEFAULT_GPT_MODEL),
|
||
"messages": messages,
|
||
"temperature": temperature,
|
||
}
|
||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
url,
|
||
data=body,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {api_key}",
|
||
},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||
result = json.loads(response.read().decode("utf-8", errors="replace"))
|
||
except urllib.error.HTTPError as exc:
|
||
detail = exc.read().decode("utf-8", errors="replace")[:1200]
|
||
raise RuntimeError(f"GPT 请求失败:HTTP {exc.code} {detail}") from exc
|
||
except urllib.error.URLError as exc:
|
||
raise RuntimeError(f"GPT 请求失败:{exc.reason}") from exc
|
||
choices = result.get("choices") or []
|
||
if not choices:
|
||
raise RuntimeError("GPT 返回为空")
|
||
content = ((choices[0].get("message") or {}).get("content") or "").strip()
|
||
if content.startswith("```"):
|
||
content = re.sub(r"^```(?:html|xml)?\s*", "", content, flags=re.I).strip()
|
||
content = re.sub(r"\s*```$", "", content).strip()
|
||
if not content:
|
||
raise RuntimeError("GPT 没有返回译文")
|
||
return content
|
||
|
||
|
||
def session_root_for(review_root: Path, epub_path: Path) -> Path:
|
||
digest = hashlib.sha1(str(epub_path.resolve()).encode("utf-8")).hexdigest()[:10]
|
||
return review_root / f"{safe_slug(epub_path.stem)}_{digest}"
|
||
|
||
|
||
def app_manifest_path(review_root: Path) -> Path:
|
||
return review_root / "app_state.json"
|
||
|
||
|
||
def gpt_config_path(review_root: Path) -> Path:
|
||
return review_root / "gpt_config.json"
|
||
|
||
|
||
def session_manifest_path(session_root: Path) -> Path:
|
||
return session_root / "review_state" / "session.json"
|
||
|
||
|
||
def library_db_path(review_root: Path) -> Path:
|
||
return review_root / LIBRARY_DB_NAME
|
||
|
||
|
||
def series_config_path(review_root: Path, series_id: str) -> Path:
|
||
return review_root / "series_configs" / f"{safe_slug(series_id, max_len=120)}.json"
|
||
|
||
|
||
def session_public_id(session_root: Path) -> str:
|
||
return session_root.name
|
||
|
||
|
||
def session_root_from_id(review_root: Path, session_id: str) -> Path:
|
||
safe_id = safe_slug(session_id, max_len=240)
|
||
if safe_id != session_id:
|
||
raise ValueError("invalid session id")
|
||
root = (review_root / session_id).resolve()
|
||
try:
|
||
root.relative_to(review_root.resolve())
|
||
except ValueError as exc:
|
||
raise ValueError("invalid session id") from exc
|
||
return root
|
||
|
||
|
||
def write_session_manifest(session_root: Path, epub_path: Path, source_name: str = "") -> None:
|
||
existing = read_json(session_manifest_path(session_root), {})
|
||
metadata = read_epub_metadata(session_root)
|
||
metadata["source_name"] = source_name or epub_path.name
|
||
record = {
|
||
"id": session_public_id(session_root),
|
||
"source_name": source_name or epub_path.name,
|
||
"source_epub": str(epub_path),
|
||
"session_root": str(session_root),
|
||
"created_at": existing.get("created_at") or now_iso(),
|
||
"updated_at": now_iso(),
|
||
"metadata": metadata,
|
||
}
|
||
write_json(session_manifest_path(session_root), record)
|
||
|
||
|
||
def update_app_manifest(review_root: Path, session_root: Path | None) -> None:
|
||
data = read_json(app_manifest_path(review_root), {})
|
||
data["active_session_id"] = session_public_id(session_root) if session_root else ""
|
||
data["updated_at"] = now_iso()
|
||
write_json(app_manifest_path(review_root), data)
|
||
|
||
|
||
def session_epub_path(session_root: Path) -> Path:
|
||
manifest = read_json(session_manifest_path(session_root), {})
|
||
source_epub = manifest.get("source_epub") or read_json(
|
||
session_root / "review_state" / "state.json", {}
|
||
).get("source_epub")
|
||
if source_epub:
|
||
path = Path(source_epub)
|
||
if path.exists():
|
||
return path
|
||
fallback = session_root / "source.epub"
|
||
if fallback.exists():
|
||
return fallback
|
||
return Path(source_epub or fallback)
|
||
|
||
|
||
def normalize_reading_position(data: Any) -> dict[str, Any]:
|
||
if not isinstance(data, dict):
|
||
return {}
|
||
|
||
def field(name: str, limit: int = 160) -> str:
|
||
return str(data.get(name) or "").strip()[:limit]
|
||
|
||
def nonnegative_int(name: str, limit: int = 10_000_000) -> int:
|
||
try:
|
||
value = int(float(data.get(name) or 0))
|
||
except (TypeError, ValueError):
|
||
value = 0
|
||
return max(0, min(value, limit))
|
||
|
||
mode = field("mode", 20)
|
||
if mode not in {"reading", "polish"}:
|
||
mode = "reading"
|
||
scope = field("scope", 20)
|
||
if scope not in {"chapter", "part", "image", "none"}:
|
||
scope = "none"
|
||
position = {
|
||
"mode": mode,
|
||
"scope": scope,
|
||
"chapter_id": field("chapter_id"),
|
||
"part_id": field("part_id"),
|
||
"image_item_id": field("image_item_id"),
|
||
"row_id": field("row_id"),
|
||
"scroll_top": nonnegative_int("scroll_top"),
|
||
"offset": nonnegative_int("offset", 5000),
|
||
"updated_at": field("updated_at", 80) or now_iso(),
|
||
}
|
||
return position
|
||
|
||
|
||
def read_reading_position(session_root: Path) -> dict[str, Any]:
|
||
state = read_json(session_root / "review_state" / "state.json", {})
|
||
return normalize_reading_position(state.get("reading_position", {}))
|
||
|
||
|
||
def summarize_session(session_root: Path) -> dict[str, Any]:
|
||
manifest = read_json(session_manifest_path(session_root), {})
|
||
state = read_json(session_root / "review_state" / "state.json", {})
|
||
rows = merge_rows(session_root) if (session_root / "review_state" / "rows.json").exists() else []
|
||
touched = [row for row in rows if rowTouched_py(row)]
|
||
source_epub = manifest.get("source_epub") or state.get("source_epub", "")
|
||
source_name = manifest.get("source_name") or (Path(source_epub).name if source_epub else session_root.name)
|
||
metadata = manifest.get("metadata") if isinstance(manifest.get("metadata"), dict) else {}
|
||
if not metadata:
|
||
metadata = read_epub_metadata(session_root)
|
||
manifest["metadata"] = metadata
|
||
write_json(session_manifest_path(session_root), manifest)
|
||
exports_dir = session_root / "exports"
|
||
latest_export = ""
|
||
if exports_dir.exists():
|
||
exports = sorted(exports_dir.glob("*.epub"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
latest_export = str(exports[0]) if exports else ""
|
||
updated_at = state.get("updated_at") or manifest.get("updated_at") or ""
|
||
session_id = session_public_id(session_root)
|
||
cover_asset_path = str(metadata.get("cover_asset_path") or "")
|
||
return {
|
||
"id": session_id,
|
||
"version": version,
|
||
"source_name": source_name,
|
||
"source_epub": source_epub,
|
||
"session_root": str(session_root),
|
||
"metadata": metadata,
|
||
"title": metadata.get("title") or source_name,
|
||
"series": metadata.get("series") or "未分类",
|
||
"series_id": metadata.get("series_id") or safe_slug(metadata.get("series") or "未分类", max_len=120),
|
||
"volume": metadata.get("volume", ""),
|
||
"authors": metadata.get("authors", []),
|
||
"publisher": metadata.get("publisher", ""),
|
||
"language": metadata.get("language", ""),
|
||
"date": metadata.get("date", ""),
|
||
"cover_asset_path": cover_asset_path,
|
||
"cover_url": session_asset_url(session_id, cover_asset_path) if cover_asset_path else "",
|
||
"row_count": len(rows),
|
||
"touched_count": len(touched),
|
||
"marked_count": sum(1 for row in rows if row.get("marked")),
|
||
"updated_at": updated_at,
|
||
"created_at": state.get("created_at") or manifest.get("created_at") or "",
|
||
"feedback_md": str(session_root / "feedback" / "feedback_for_codex.md"),
|
||
"feedback_jsonl": str(session_root / "feedback" / "translation_feedback.jsonl"),
|
||
"latest_export": latest_export,
|
||
"reading_position": normalize_reading_position(state.get("reading_position", {})),
|
||
}
|
||
|
||
|
||
def rowTouched_py(row: dict[str, Any]) -> bool:
|
||
return bool(
|
||
row.get("edited")
|
||
or row.get("marked")
|
||
or row.get("comment")
|
||
or row.get("learn_note")
|
||
or row.get("issue_type")
|
||
or row.get("severity")
|
||
or row.get("tags")
|
||
)
|
||
|
||
|
||
def list_review_sessions(review_root: Path) -> list[dict[str, Any]]:
|
||
if not review_root.exists():
|
||
return []
|
||
sessions = []
|
||
for child in review_root.iterdir():
|
||
if not child.is_dir() or child.name in {UPLOAD_DIR_NAME, TRANSLATION_JOB_DIR_NAME, TRANSLATION_OUTPUT_DIR_NAME, "series_configs"}:
|
||
continue
|
||
if not (child / "review_state" / "state.json").exists():
|
||
continue
|
||
try:
|
||
sessions.append(summarize_session(child))
|
||
except Exception:
|
||
continue
|
||
sessions.sort(key=lambda item: item.get("updated_at") or item.get("created_at") or "", reverse=True)
|
||
return sessions
|
||
|
||
|
||
def build_library_index(review_root: Path) -> dict[str, Any]:
|
||
sessions = list_review_sessions(review_root)
|
||
series_map: dict[str, dict[str, Any]] = {}
|
||
for session in sessions:
|
||
series_id = str(session.get("series_id") or "uncategorized")
|
||
series = series_map.setdefault(
|
||
series_id,
|
||
{
|
||
"id": series_id,
|
||
"title": session.get("series") or "未分类",
|
||
"count": 0,
|
||
"books": [],
|
||
"authors": set(),
|
||
"publishers": set(),
|
||
"languages": set(),
|
||
},
|
||
)
|
||
series["count"] += 1
|
||
series["books"].append(session["id"])
|
||
for author in session.get("authors") or []:
|
||
if author:
|
||
series["authors"].add(author)
|
||
if session.get("publisher"):
|
||
series["publishers"].add(session["publisher"])
|
||
if session.get("language"):
|
||
series["languages"].add(session["language"])
|
||
series_list = []
|
||
for item in series_map.values():
|
||
series_list.append(
|
||
{
|
||
**item,
|
||
"authors": sorted(item["authors"]),
|
||
"publishers": sorted(item["publishers"]),
|
||
"languages": sorted(item["languages"]),
|
||
}
|
||
)
|
||
series_list.sort(key=lambda item: item["title"])
|
||
library = {
|
||
"version": version,
|
||
"updated_at": now_iso(),
|
||
"book_count": len(sessions),
|
||
"series_count": len(series_list),
|
||
"sessions": sessions,
|
||
"series": series_list,
|
||
}
|
||
write_json_atomic(library_db_path(review_root), library)
|
||
return library
|
||
|
||
|
||
def read_series_config(review_root: Path, series_id: str) -> dict[str, Any]:
|
||
path = series_config_path(review_root, series_id)
|
||
data = read_json(path, {})
|
||
return {
|
||
"series_id": series_id,
|
||
"translation_prompt": str(data.get("translation_prompt") or ""),
|
||
"format_prompt": str(data.get("format_prompt") or ""),
|
||
"character_prompt": str(data.get("character_prompt") or ""),
|
||
"glossary_path": str(data.get("glossary_path") or ""),
|
||
"updated_at": str(data.get("updated_at") or ""),
|
||
}
|
||
|
||
|
||
def normalize_optional_prompt(name: str, value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return ""
|
||
return normalize_gpt_prompt(name, text)
|
||
|
||
|
||
def normalize_optional_glossary_path(value: Any) -> str:
|
||
raw = str(value or "").strip()
|
||
if not raw:
|
||
return ""
|
||
return normalize_glossary_path_value(raw)
|
||
|
||
|
||
def save_series_config(review_root: Path, series_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||
record = {
|
||
"series_id": series_id,
|
||
"translation_prompt": normalize_optional_prompt("translation_prompt", data.get("translation_prompt")),
|
||
"format_prompt": normalize_optional_prompt("format_prompt", data.get("format_prompt")),
|
||
"character_prompt": normalize_optional_prompt("character_prompt", data.get("character_prompt")),
|
||
"glossary_path": normalize_optional_glossary_path(data.get("glossary_path")),
|
||
"updated_at": now_iso(),
|
||
}
|
||
write_json_atomic(series_config_path(review_root, series_id), record)
|
||
return record
|
||
|
||
|
||
def validate_epub_file(path: Path) -> None:
|
||
if path.suffix.lower() != ".epub":
|
||
raise ValueError("只支持 .epub 文件")
|
||
if not zipfile.is_zipfile(path):
|
||
raise ValueError("文件不是有效的 EPUB/ZIP 包")
|
||
|
||
|
||
def copy_upload_to_review_root(file_storage: Any, review_root: Path) -> Path:
|
||
filename = safe_slug(Path(file_storage.filename or "uploaded.epub").name, max_len=140)
|
||
if not filename.lower().endswith(".epub"):
|
||
filename = f"{filename}.epub"
|
||
upload_dir = review_root / UPLOAD_DIR_NAME
|
||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||
stem = safe_slug(Path(filename).stem, max_len=100)
|
||
target = upload_dir / f"{dt.datetime.now().strftime('%Y%m%d_%H%M%S')}_{stem}.epub"
|
||
try:
|
||
file_storage.save(target)
|
||
validate_epub_file(target)
|
||
return target
|
||
except Exception:
|
||
target.unlink(missing_ok=True)
|
||
raise
|
||
|
||
|
||
def extracted_path(session_root: Path, entry_name: str) -> Path:
|
||
root = (session_root / "extracted").resolve()
|
||
target = (root / entry_name).resolve()
|
||
try:
|
||
target.relative_to(root)
|
||
except ValueError as exc:
|
||
raise ValueError(f"unsafe epub entry path: {entry_name}") from exc
|
||
return target
|
||
|
||
|
||
def extract_epub(epub_path: Path, session_root: Path, reset: bool = False) -> None:
|
||
extracted = session_root / "extracted"
|
||
if reset and session_root.exists():
|
||
shutil.rmtree(session_root)
|
||
if extracted.exists():
|
||
return
|
||
session_root.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(epub_path, session_root / "source.epub")
|
||
extracted.mkdir(parents=True, exist_ok=True)
|
||
entries = []
|
||
with zipfile.ZipFile(epub_path, "r") as zf:
|
||
for info in zf.infolist():
|
||
entries.append(
|
||
{
|
||
"filename": info.filename,
|
||
"compress_type": int(info.compress_type),
|
||
"date_time": list(info.date_time),
|
||
"is_dir": info.is_dir(),
|
||
}
|
||
)
|
||
target = extracted_path(session_root, info.filename)
|
||
if info.is_dir():
|
||
target.mkdir(parents=True, exist_ok=True)
|
||
continue
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
target.write_bytes(zf.read(info.filename))
|
||
write_json(session_root / "review_state" / "epub_entries.json", entries)
|
||
|
||
|
||
def find_opf_entry(session_root: Path) -> str:
|
||
extracted = session_root / "extracted"
|
||
container = extracted / "META-INF" / "container.xml"
|
||
root = read_xml_root(container)
|
||
if root is not None:
|
||
for elem in root.iter():
|
||
if local_name(elem.tag) == "rootfile":
|
||
full_path = elem.attrib.get("full-path") or elem.attrib.get("full_path")
|
||
if full_path:
|
||
return posixpath.normpath(full_path).lstrip("./")
|
||
entries = read_json(session_root / "review_state" / "epub_entries.json", [])
|
||
for item in entries:
|
||
name = item.get("filename", "")
|
||
if not item.get("is_dir") and name.lower().endswith(".opf"):
|
||
return name
|
||
return ""
|
||
|
||
|
||
def parse_opf_metadata(session_root: Path) -> dict[str, Any]:
|
||
opf_entry = find_opf_entry(session_root)
|
||
if not opf_entry:
|
||
return {"opf_entry": "", "opf_dir": "", "spine": [], "metadata": {}}
|
||
opf_path = extracted_path(session_root, opf_entry)
|
||
root = read_xml_root(opf_path)
|
||
if root is None:
|
||
return {"opf_entry": opf_entry, "opf_dir": entry_dir(opf_entry), "spine": [], "metadata": {}}
|
||
|
||
opf_dir = entry_dir(opf_entry)
|
||
manifest: dict[str, dict[str, str]] = {}
|
||
spine_idrefs: list[str] = []
|
||
metadata_values: dict[str, list[str]] = {
|
||
"title": [],
|
||
"creator": [],
|
||
"language": [],
|
||
"publisher": [],
|
||
"date": [],
|
||
"identifier": [],
|
||
"subject": [],
|
||
"description": [],
|
||
}
|
||
cover_id = ""
|
||
for elem in root.iter():
|
||
name = local_name(elem.tag)
|
||
if name in metadata_values and elem.text:
|
||
value = compact_text(elem.text)
|
||
if value:
|
||
metadata_values[name].append(value)
|
||
elif name == "meta":
|
||
meta_name = (elem.attrib.get("name") or "").strip().lower()
|
||
if meta_name == "cover":
|
||
cover_id = elem.attrib.get("content", "").strip()
|
||
if name == "item":
|
||
item_id = elem.attrib.get("id", "")
|
||
href = elem.attrib.get("href", "")
|
||
if item_id and href:
|
||
manifest[item_id] = {
|
||
"href": normalize_href(opf_dir, href),
|
||
"media_type": elem.attrib.get("media-type", ""),
|
||
"properties": elem.attrib.get("properties", ""),
|
||
}
|
||
elif name == "itemref":
|
||
idref = elem.attrib.get("idref", "")
|
||
if idref:
|
||
spine_idrefs.append(idref)
|
||
|
||
spine = []
|
||
for idref in spine_idrefs:
|
||
item = manifest.get(idref)
|
||
if not item:
|
||
continue
|
||
href = item["href"]
|
||
if href.lower().endswith((".xhtml", ".html", ".htm")):
|
||
spine.append(href)
|
||
|
||
cover_entry = ""
|
||
if cover_id and cover_id in manifest:
|
||
cover_entry = manifest[cover_id].get("href", "")
|
||
if not cover_entry:
|
||
for item in manifest.values():
|
||
if "cover-image" in item.get("properties", "").split() and is_raster_image_asset(item.get("href", "")):
|
||
cover_entry = item.get("href", "")
|
||
break
|
||
if not cover_entry:
|
||
for item in manifest.values():
|
||
href = item.get("href", "")
|
||
media_type = item.get("media_type", "")
|
||
if media_type.startswith("image/") and is_raster_image_asset(href):
|
||
cover_entry = href
|
||
break
|
||
|
||
metadata = {
|
||
key: values[0] if values else ""
|
||
for key, values in metadata_values.items()
|
||
if key not in {"creator", "identifier", "subject"}
|
||
}
|
||
metadata["creators"] = metadata_values["creator"]
|
||
metadata["identifiers"] = metadata_values["identifier"]
|
||
metadata["subjects"] = metadata_values["subject"]
|
||
metadata["cover_asset_path"] = cover_entry if cover_entry and is_raster_image_asset(cover_entry) else ""
|
||
return {
|
||
"opf_entry": opf_entry,
|
||
"opf_dir": opf_dir,
|
||
"manifest": manifest,
|
||
"spine": spine,
|
||
"metadata": metadata,
|
||
}
|
||
|
||
|
||
def infer_volume_number(title: str) -> str:
|
||
text = compact_text(title)
|
||
patterns = [
|
||
r"(?:第\s*)?([0-90-9]+)\s*[巻卷册集]",
|
||
r"\bvol(?:ume)?\.?\s*([0-90-9]+)\b",
|
||
r"\b([0-90-9]+)\s*(?:[~~].*)?$",
|
||
]
|
||
for pattern in patterns:
|
||
match = re.search(pattern, text, flags=re.I)
|
||
if match:
|
||
return match.group(1).translate(str.maketrans("0123456789", "0123456789"))
|
||
return ""
|
||
|
||
|
||
def infer_series_title(title: str, source_name: str = "") -> str:
|
||
text = compact_text(title or Path(source_name).stem)
|
||
text = re.sub(r"_AI翻译_(?:中日双语|中文)_\d{8}_\d{6}$", "", text)
|
||
text = re.sub(r"\s*\([^)]*(?:文庫|ノベル|EPUB|nodrm)[^)]*\)\s*$", "", text, flags=re.I)
|
||
text = re.sub(r"\s*(?:第\s*)?[0-90-9]+\s*[巻卷册集]\s*(?:[~~].*)?$", "", text)
|
||
text = re.sub(r"\s+vol(?:ume)?\.?\s*[0-90-9]+\s*(?:[~~].*)?$", "", text, flags=re.I)
|
||
text = re.sub(r"\s+[0-90-9]+\s*(?:[~~].*)?$", "", text)
|
||
text = re.sub(r"\s+[0-90-9]+$", "", text)
|
||
return compact_text(text) or compact_text(title or source_name) or "未分类"
|
||
|
||
|
||
def read_epub_metadata(session_root: Path) -> dict[str, Any]:
|
||
opf = parse_opf_metadata(session_root)
|
||
metadata = dict(opf.get("metadata") or {})
|
||
manifest = read_json(session_manifest_path(session_root), {})
|
||
source_name = manifest.get("source_name") or Path(str(manifest.get("source_epub") or "")).name
|
||
title = compact_text(str(metadata.get("title") or "")) or Path(source_name).stem or session_root.name
|
||
creators = metadata.get("creators") if isinstance(metadata.get("creators"), list) else []
|
||
series_title = infer_series_title(title, source_name)
|
||
series_id = safe_slug(series_title, max_len=120)
|
||
volume = infer_volume_number(title) or infer_volume_number(source_name)
|
||
metadata.update(
|
||
{
|
||
"title": title,
|
||
"series": series_title,
|
||
"series_id": series_id,
|
||
"volume": volume,
|
||
"authors": [compact_text(str(item)) for item in creators if compact_text(str(item))],
|
||
"publisher": compact_text(str(metadata.get("publisher") or "")),
|
||
"language": compact_text(str(metadata.get("language") or "")),
|
||
"date": compact_text(str(metadata.get("date") or "")),
|
||
"source_name": source_name,
|
||
}
|
||
)
|
||
return metadata
|
||
|
||
|
||
def session_asset_url(session_id: str, entry_name: str) -> str:
|
||
return f"/api/session/{urllib.parse.quote(session_id)}/asset/{urllib.parse.quote(entry_name, safe='/')}"
|
||
|
||
|
||
def translation_source_stats(session_root: Path) -> dict[str, Any]:
|
||
html_files = html_entries_in_reading_order(session_root)
|
||
total_blocks = 0
|
||
total_paragraphs = 0
|
||
total_headings = 0
|
||
nonempty_blocks = 0
|
||
kana_blocks = 0
|
||
image_blocks = 0
|
||
skipped_nav_files = 0
|
||
skipped_nav_blocks = 0
|
||
skipped_blank = 0
|
||
skipped_image = 0
|
||
skipped_no_japanese = 0
|
||
skipped_no_japanese_examples: list[dict[str, Any]] = []
|
||
translatable_by_file: dict[str, int] = {}
|
||
for entry_name in html_files:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
matches = list(SOURCE_BLOCK_RE.finditer(text))
|
||
file_has_japanese_context = source_file_has_japanese_context(text)
|
||
total_blocks += len(matches)
|
||
total_paragraphs += sum(1 for match in matches if match.group("tag").lower() == "p")
|
||
total_headings += sum(1 for match in matches if match.group("tag").lower().startswith("h"))
|
||
nav = html_looks_like_navigation(entry_name, text)
|
||
if nav:
|
||
skipped_nav_files += 1
|
||
skipped_nav_blocks += len(matches)
|
||
for match in matches:
|
||
tag = match.group("tag").lower()
|
||
inner = match.group("inner")
|
||
visible = paragraph_visible_text(inner)
|
||
if visible:
|
||
nonempty_blocks += 1
|
||
else:
|
||
skipped_blank += 1
|
||
if IMG_TAG_RE.search(inner):
|
||
image_blocks += 1
|
||
if not visible:
|
||
skipped_image += 1
|
||
if KANA_RE.search(strip_html_keep_ruby_text(inner)):
|
||
kana_blocks += 1
|
||
included = (not nav) and is_translatable_source_block(inner, tag, file_has_japanese_context)
|
||
if included:
|
||
translatable_by_file[entry_name] = translatable_by_file.get(entry_name, 0) + 1
|
||
elif not nav and visible and not IMG_TAG_RE.search(inner) and not KANA_RE.search(strip_html_keep_ruby_text(inner)):
|
||
skipped_no_japanese += 1
|
||
if len(skipped_no_japanese_examples) < 5:
|
||
skipped_no_japanese_examples.append(
|
||
{
|
||
"file": entry_name,
|
||
"tag": tag,
|
||
"text": truncate_prompt_text(visible, 80),
|
||
}
|
||
)
|
||
return {
|
||
"html_file_count": len(html_files),
|
||
"total_blocks": total_blocks,
|
||
"total_paragraphs": total_paragraphs,
|
||
"total_headings": total_headings,
|
||
"nonempty_blocks": nonempty_blocks,
|
||
"kana_blocks": kana_blocks,
|
||
"image_blocks": image_blocks,
|
||
"skipped_nav_files": skipped_nav_files,
|
||
"skipped_nav_blocks": skipped_nav_blocks,
|
||
"skipped_blank": skipped_blank,
|
||
"skipped_image": skipped_image,
|
||
"skipped_no_japanese": skipped_no_japanese,
|
||
"skipped_no_japanese_examples": skipped_no_japanese_examples,
|
||
"translatable_by_file": translatable_by_file,
|
||
}
|
||
|
||
|
||
def html_entries_in_reading_order(session_root: Path) -> list[str]:
|
||
entries = read_json(session_root / "review_state" / "epub_entries.json", [])
|
||
all_html = [
|
||
item["filename"]
|
||
for item in entries
|
||
if not item.get("is_dir")
|
||
and item["filename"].lower().endswith((".xhtml", ".html", ".htm"))
|
||
]
|
||
opf = parse_opf_metadata(session_root)
|
||
ordered: list[str] = []
|
||
for item in opf.get("spine", []):
|
||
if item in all_html and item not in ordered:
|
||
ordered.append(item)
|
||
for item in sorted(all_html, key=file_sort_key):
|
||
basename = posixpath.basename(item).lower()
|
||
if item in ordered or basename in {"nav.xhtml", "toc.xhtml"}:
|
||
continue
|
||
ordered.append(item)
|
||
return ordered
|
||
|
||
|
||
def extract_heading_from_html(text: str) -> str:
|
||
for pattern in (
|
||
r"<h[1-6]\b[^>]*>(.*?)</h[1-6]>",
|
||
r"<title\b[^>]*>(.*?)</title>",
|
||
):
|
||
match = re.search(pattern, text, flags=re.S | re.I)
|
||
if not match:
|
||
continue
|
||
title = compact_text(strip_tags(match.group(1)))
|
||
if title:
|
||
return title
|
||
return ""
|
||
|
||
|
||
def extract_body_heading_from_html(text: str) -> str:
|
||
match = re.search(r"<h[1-6]\b[^>]*>(.*?)</h[1-6]>", text, flags=re.S | re.I)
|
||
if not match:
|
||
return ""
|
||
return compact_text(strip_tags(match.group(1)))
|
||
|
||
|
||
def read_document_title(session_root: Path, entry_name: str) -> str:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
return ""
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
return extract_heading_from_html(text)
|
||
|
||
|
||
def read_body_heading(session_root: Path, entry_name: str) -> str:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
return ""
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
return extract_body_heading_from_html(text)
|
||
|
||
|
||
def first_content_text(session_root: Path, entry_name: str) -> str:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
return ""
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
for match in P_RE.finditer(text):
|
||
inner = match.group(2)
|
||
if re.search(r"<img\b", inner, flags=re.I):
|
||
continue
|
||
label = compact_text(strip_tags(inner))
|
||
if label:
|
||
return label
|
||
return ""
|
||
|
||
|
||
def tag_attrs(tag: str) -> dict[str, str]:
|
||
attrs: dict[str, str] = {}
|
||
for key, _quote, value in ATTR_RE.findall(tag):
|
||
attrs[key.lower()] = html.unescape(value or "")
|
||
return attrs
|
||
|
||
|
||
def part_display_label(entry_name: str, fallback_index: int) -> str:
|
||
stem = Path(entry_name).stem
|
||
for pattern in (r"part[-_]?(\d+)$", r"p[-_]?(\d+)$", r"^(\d+)$"):
|
||
match = re.search(pattern, stem, flags=re.I)
|
||
if match:
|
||
return f"part{int(match.group(1)):04d}"
|
||
if stem and stem.lower() not in {"titlepage", "cover"}:
|
||
return stem
|
||
return f"part{fallback_index:04d}"
|
||
|
||
|
||
def is_image_asset(entry_name: str) -> bool:
|
||
suffix = posixpath.splitext(entry_name.split("?", 1)[0])[1].lower()
|
||
return suffix in IMAGE_EXTENSIONS
|
||
|
||
|
||
def is_raster_image_asset(entry_name: str) -> bool:
|
||
suffix = posixpath.splitext(entry_name.split("?", 1)[0])[1].lower()
|
||
return suffix in RASTER_IMAGE_EXTENSIONS
|
||
|
||
|
||
def asset_url(entry_name: str) -> str:
|
||
return f"/api/asset/{urllib.parse.quote(entry_name, safe='/')}"
|
||
|
||
|
||
def extract_images_from_html(session_root: Path, entry_name: str) -> list[dict[str, Any]]:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
return []
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
images: list[dict[str, Any]] = []
|
||
for index, match in enumerate(IMG_TAG_RE.finditer(text), start=1):
|
||
attrs = tag_attrs(match.group(0))
|
||
href = attrs.get("src") or attrs.get("href") or attrs.get("xlink:href")
|
||
if not href or EXTERNAL_URI_RE.match(href):
|
||
continue
|
||
image_entry = normalize_href(entry_dir(entry_name), href)
|
||
if not image_entry or not is_raster_image_asset(image_entry):
|
||
continue
|
||
images.append(
|
||
{
|
||
"index": index,
|
||
"file": entry_name,
|
||
"file_label": Path(entry_name).name,
|
||
"asset_path": image_entry,
|
||
"asset_name": Path(image_entry).name,
|
||
"asset_url": asset_url(image_entry),
|
||
"alt": compact_text(attrs.get("alt") or attrs.get("title") or ""),
|
||
}
|
||
)
|
||
return images
|
||
|
||
|
||
def collect_asset_paths_from_structure(structure: dict[str, Any]) -> set[str]:
|
||
paths: set[str] = set()
|
||
for chapter in structure.get("chapters", []):
|
||
for image in chapter.get("images", []):
|
||
asset_path = image.get("asset_path", "")
|
||
if asset_path:
|
||
paths.add(asset_path)
|
||
for item in chapter.get("items", []):
|
||
for image in item.get("images", []):
|
||
asset_path = image.get("asset_path", "")
|
||
if asset_path:
|
||
paths.add(asset_path)
|
||
return paths
|
||
|
||
|
||
def image_item_count(chapters: list[dict[str, Any]]) -> int:
|
||
return sum(1 for chapter in chapters for item in chapter.get("items", []) if item.get("kind") == "image")
|
||
|
||
|
||
def nav_target_key(base_dir: str, href: str) -> tuple[str, str]:
|
||
path, fragment = normalize_href_with_fragment(base_dir, href)
|
||
return path, fragment
|
||
|
||
|
||
def parse_internal_toc(session_root: Path) -> list[dict[str, Any]]:
|
||
ordered_files = html_entries_in_reading_order(session_root)
|
||
ordered_set = set(ordered_files)
|
||
for entry_name in ordered_files:
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
if not re.search(r"<a\b[^>]*href=", text, flags=re.I):
|
||
continue
|
||
basename = posixpath.basename(entry_name).lower()
|
||
heading = extract_heading_from_html(text)
|
||
is_toc_like_name = basename in {"nav.xhtml", "toc.xhtml", "toc.html", "toc.htm"} or "toc" in basename
|
||
is_toc_like_title = bool(re.search(r"(目录|目次|contents?)", heading, flags=re.I))
|
||
matches = list(P_RE.finditer(text))
|
||
base_dir = entry_dir(entry_name)
|
||
candidate_items: list[dict[str, Any]] = []
|
||
i = 0
|
||
while i < len(matches):
|
||
source_attrs = matches[i].group(1)[2:-1]
|
||
source_inner = matches[i].group(2)
|
||
link_match = re.search(r"<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>(.*?)</a>", source_inner, flags=re.S | re.I)
|
||
if not link_match:
|
||
i += 1
|
||
continue
|
||
target_file, target_fragment = nav_target_key(base_dir, link_match.group(1))
|
||
if not target_file:
|
||
i += 1
|
||
continue
|
||
if target_file not in ordered_set:
|
||
i += 1
|
||
continue
|
||
source_title = compact_text(strip_tags(link_match.group(2))) or compact_text(strip_tags(source_inner))
|
||
title = source_title
|
||
if is_gray_source(source_attrs, source_inner) and i + 1 < len(matches):
|
||
next_attrs = matches[i + 1].group(1)[2:-1]
|
||
next_inner = matches[i + 1].group(2)
|
||
if not is_gray_source(next_attrs, next_inner):
|
||
next_link = re.search(r"<a\b[^>]*href=['\"]([^'\"]+)['\"][^>]*>(.*?)</a>", next_inner, flags=re.S | re.I)
|
||
same_target = True
|
||
if next_link:
|
||
next_target_file, next_target_fragment = nav_target_key(base_dir, next_link.group(1))
|
||
same_target = next_target_file == target_file and next_target_fragment == target_fragment
|
||
next_title = compact_text(strip_tags(next_link.group(2) if next_link else next_inner))
|
||
if next_title:
|
||
if same_target:
|
||
title = next_title
|
||
i += 1
|
||
if title:
|
||
candidate_items.append(
|
||
{
|
||
"title": title,
|
||
"href": target_file,
|
||
"fragment": target_fragment,
|
||
"children": [],
|
||
"source_title": source_title,
|
||
}
|
||
)
|
||
i += 1
|
||
if candidate_items:
|
||
internal_targets = sum(1 for item in candidate_items if item["href"] in ordered_set and item["href"] != entry_name)
|
||
link_density = internal_targets / max(1, len(matches))
|
||
if is_toc_like_name or is_toc_like_title or (len(candidate_items) >= 4 and link_density >= 0.35):
|
||
return candidate_items
|
||
return []
|
||
|
||
|
||
def parse_nav_toc(session_root: Path) -> list[dict[str, Any]]:
|
||
entries = read_json(session_root / "review_state" / "epub_entries.json", [])
|
||
nav_entries = [
|
||
item["filename"]
|
||
for item in entries
|
||
if not item.get("is_dir")
|
||
and posixpath.basename(item["filename"]).lower() in {"nav.xhtml", "toc.xhtml", "toc.html", "toc.htm"}
|
||
]
|
||
opf = parse_opf_metadata(session_root)
|
||
for manifest_item in opf.get("manifest", {}).values():
|
||
if "nav" in manifest_item.get("properties", "").split():
|
||
href = manifest_item.get("href", "")
|
||
if href and href not in nav_entries:
|
||
nav_entries.insert(0, href)
|
||
|
||
for entry_name in nav_entries:
|
||
path = extracted_path(session_root, entry_name)
|
||
root = read_xml_root(path)
|
||
if root is None:
|
||
continue
|
||
base_dir = entry_dir(entry_name)
|
||
toc_root: ET.Element | None = None
|
||
for elem in root.iter():
|
||
if local_name(elem.tag) != "nav":
|
||
continue
|
||
nav_type = elem.attrib.get("{http://www.idpf.org/2007/ops}type", "") or elem.attrib.get("epub:type", "")
|
||
if "toc" in nav_type.split() or toc_root is None:
|
||
toc_root = elem
|
||
if "toc" in nav_type.split():
|
||
break
|
||
if toc_root is None:
|
||
continue
|
||
top_ol = next((child for child in list(toc_root) if local_name(child.tag) == "ol"), None)
|
||
if top_ol is None:
|
||
top_ol = next((elem for elem in toc_root.iter() if local_name(elem.tag) == "ol"), None)
|
||
if top_ol is None:
|
||
continue
|
||
items = parse_nav_ol(top_ol, base_dir)
|
||
if items:
|
||
return items
|
||
return []
|
||
|
||
|
||
def parse_nav_ol(ol: ET.Element, base_dir: str) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
for li in list(ol):
|
||
if local_name(li.tag) != "li":
|
||
continue
|
||
label = ""
|
||
href = ""
|
||
fragment = ""
|
||
children: list[dict[str, Any]] = []
|
||
for child in list(li):
|
||
name = local_name(child.tag)
|
||
if name == "a":
|
||
label = element_text(child)
|
||
href, fragment = normalize_href_with_fragment(base_dir, child.attrib.get("href", ""))
|
||
elif name == "span" and not label:
|
||
label = element_text(child)
|
||
elif name == "ol":
|
||
children = parse_nav_ol(child, base_dir)
|
||
if not label:
|
||
label = element_text(li)
|
||
items.append({"title": label, "href": href, "fragment": fragment, "children": children})
|
||
return items
|
||
|
||
|
||
def flatten_nav_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
flattened: list[dict[str, Any]] = []
|
||
|
||
def walk(nodes: list[dict[str, Any]], depth: int) -> None:
|
||
for node in nodes:
|
||
href = node.get("href", "")
|
||
if href:
|
||
flattened.append({"title": node.get("title") or href, "href": href, "depth": depth})
|
||
walk(node.get("children", []), depth + 1)
|
||
|
||
walk(items, 0)
|
||
return flattened
|
||
|
||
|
||
def build_structure(session_root: Path) -> dict[str, Any]:
|
||
rows = merge_rows(session_root)
|
||
rows_by_file: dict[str, list[dict[str, Any]]] = {}
|
||
for row in rows:
|
||
rows_by_file.setdefault(row["file"], []).append(row)
|
||
|
||
all_ordered_files = html_entries_in_reading_order(session_root)
|
||
image_by_file = {
|
||
entry_name: images
|
||
for entry_name in all_ordered_files
|
||
if (images := extract_images_from_html(session_root, entry_name))
|
||
}
|
||
ordered_files = [file for file in all_ordered_files if file in rows_by_file]
|
||
for file in rows_by_file:
|
||
if file not in ordered_files:
|
||
ordered_files.append(file)
|
||
content_files = [file for file in all_ordered_files if file in rows_by_file or file in image_by_file]
|
||
for file in sorted(set(rows_by_file) | set(image_by_file), key=file_sort_key):
|
||
if file not in content_files:
|
||
content_files.append(file)
|
||
|
||
nav_tree = parse_nav_toc(session_root)
|
||
nav_flat = flatten_nav_items(nav_tree)
|
||
nav_titles: dict[str, str] = {}
|
||
for item in nav_flat:
|
||
href = item.get("href", "")
|
||
title = item.get("title", "")
|
||
if href and title and href not in nav_titles:
|
||
nav_titles[href] = title
|
||
|
||
start_titles: dict[str, str] = {}
|
||
chapter_specs: list[dict[str, Any]] = []
|
||
toc_items = parse_internal_toc(session_root)
|
||
ordered_index = {entry_name: index for index, entry_name in enumerate(all_ordered_files)}
|
||
content_file_set = set(content_files)
|
||
|
||
def first_content_file_at_or_after(entry_name: str) -> str:
|
||
start_pos = ordered_index.get(entry_name, -1)
|
||
if start_pos < 0:
|
||
return ""
|
||
for candidate in all_ordered_files[start_pos:]:
|
||
if candidate in content_file_set:
|
||
return candidate
|
||
return ""
|
||
|
||
for item in toc_items:
|
||
href = item.get("href", "")
|
||
if href not in ordered_index:
|
||
continue
|
||
start = first_content_file_at_or_after(href)
|
||
if start and item.get("title") and start not in start_titles:
|
||
start_titles[start] = item["title"]
|
||
|
||
if not start_titles:
|
||
for item in nav_flat:
|
||
href = item.get("href", "")
|
||
if href not in ordered_index:
|
||
continue
|
||
start = first_content_file_at_or_after(href)
|
||
if start and item.get("title") and start not in start_titles:
|
||
start_titles[start] = item["title"]
|
||
|
||
if not start_titles:
|
||
for entry_name in ordered_files:
|
||
heading = read_body_heading(session_root, entry_name)
|
||
if heading and entry_name not in start_titles and len(strip_tags(heading)) <= 80:
|
||
start_titles[entry_name] = heading
|
||
|
||
if start_titles:
|
||
starts = sorted(start_titles, key=lambda name: ordered_index.get(name, 10**9))
|
||
first_start_pos = ordered_index.get(starts[0], len(all_ordered_files)) if starts else len(all_ordered_files)
|
||
preface_files = [
|
||
entry_name
|
||
for entry_name in all_ordered_files[:first_start_pos]
|
||
if entry_name in rows_by_file or entry_name in image_by_file
|
||
]
|
||
if preface_files:
|
||
chapter_specs.append({"title": "卷首", "files": preface_files, "start": preface_files[0]})
|
||
for index, start in enumerate(starts):
|
||
start_pos = ordered_index.get(start, -1)
|
||
next_pos = ordered_index.get(starts[index + 1], len(all_ordered_files)) if index + 1 < len(starts) else len(all_ordered_files)
|
||
if start_pos < 0:
|
||
continue
|
||
files = [
|
||
entry_name
|
||
for entry_name in all_ordered_files[start_pos:next_pos]
|
||
if entry_name in rows_by_file or entry_name in image_by_file
|
||
]
|
||
if files:
|
||
chapter_specs.append({"title": start_titles.get(start, ""), "files": files, "start": files[0]})
|
||
|
||
grouped_files = {entry_name for spec in chapter_specs for entry_name in spec["files"]}
|
||
for entry_name in content_files:
|
||
if entry_name not in grouped_files:
|
||
chapter_specs.append({"title": "", "files": [entry_name], "start": entry_name})
|
||
chapter_specs.sort(key=lambda spec: ordered_index.get(spec["start"], 10**9))
|
||
|
||
part_serial = 0
|
||
image_serial = 0
|
||
|
||
def file_counts(file_rows: list[dict[str, Any]]) -> tuple[int, int]:
|
||
touched_count = sum(
|
||
1
|
||
for row in file_rows
|
||
if row["edited"]
|
||
or row["marked"]
|
||
or row["comment"]
|
||
or row["learn_note"]
|
||
or row["issue_type"]
|
||
or row["severity"]
|
||
or row["tags"]
|
||
)
|
||
marked_count = sum(1 for row in file_rows if row["marked"])
|
||
return touched_count, marked_count
|
||
|
||
def make_part(entry_name: str, title_override: str = "") -> dict[str, Any]:
|
||
nonlocal part_serial
|
||
part_serial += 1
|
||
file_rows = rows_by_file[entry_name]
|
||
title = part_display_label(entry_name, part_serial - 1)
|
||
touched_count, marked_count = file_counts(file_rows)
|
||
return {
|
||
"id": f"PT{part_serial:04d}",
|
||
"kind": "text",
|
||
"title": title,
|
||
"source_title": title_override
|
||
or nav_titles.get(entry_name)
|
||
or read_body_heading(session_root, entry_name)
|
||
or Path(entry_name).stem,
|
||
"file": entry_name,
|
||
"file_label": Path(entry_name).name,
|
||
"row_ids": [row["id"] for row in file_rows],
|
||
"row_count": len(file_rows),
|
||
"touched_count": touched_count,
|
||
"marked_count": marked_count,
|
||
"first_row_id": file_rows[0]["id"] if file_rows else "",
|
||
"last_row_id": file_rows[-1]["id"] if file_rows else "",
|
||
}
|
||
|
||
def make_image_item(
|
||
entry_name: str,
|
||
image: dict[str, Any],
|
||
title_override: str = "",
|
||
image_index: int = 1,
|
||
image_total: int = 1,
|
||
) -> dict[str, Any]:
|
||
nonlocal image_serial
|
||
image_serial += 1
|
||
suffix = f"-{image_index:02d}" if image_total > 1 else ""
|
||
label = part_display_label(entry_name, image_serial - 1)
|
||
is_cover = Path(entry_name).stem.lower() in {"titlepage", "cover"}
|
||
default_title = "封面" if is_cover else f"插图 {label}{suffix}"
|
||
base_title = title_override or nav_titles.get(entry_name) or read_body_heading(session_root, entry_name)
|
||
if is_cover:
|
||
title = default_title
|
||
elif title_override and base_title:
|
||
title = f"{base_title} · 插图{suffix}"
|
||
else:
|
||
title = base_title or default_title
|
||
image_item = {
|
||
**image,
|
||
"id": f"IMG{image_serial:04d}-{image['index']:02d}",
|
||
}
|
||
return {
|
||
"id": image_item["id"],
|
||
"kind": "image",
|
||
"title": title,
|
||
"file": entry_name,
|
||
"file_label": Path(entry_name).name,
|
||
"images": [image_item],
|
||
"row_count": 0,
|
||
"touched_count": 0,
|
||
"marked_count": 0,
|
||
"parts": [],
|
||
"first_image_id": image_item["id"],
|
||
"last_image_id": image_item["id"],
|
||
}
|
||
|
||
def image_chapter_from_item(chapter_id: str, item: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
**item,
|
||
"id": chapter_id,
|
||
"items": [item],
|
||
"_sort": ordered_index.get(item.get("file", ""), 10**9),
|
||
}
|
||
|
||
def chapter_from_items(chapter_id: str, title: str, items: list[dict[str, Any]]) -> dict[str, Any]:
|
||
parts = [item for item in items if item.get("kind") == "text"]
|
||
images = [image for item in items if item.get("kind") == "image" for image in item.get("images", [])]
|
||
return {
|
||
"id": chapter_id,
|
||
"kind": "text",
|
||
"title": title,
|
||
"row_count": sum(part["row_count"] for part in parts),
|
||
"touched_count": sum(part["touched_count"] for part in parts),
|
||
"marked_count": sum(part["marked_count"] for part in parts),
|
||
"parts": parts,
|
||
"images": images,
|
||
"items": items,
|
||
"_sort": min((ordered_index.get(item.get("file", ""), 10**9) for item in items), default=10**9),
|
||
}
|
||
|
||
def chapter_from_parts(chapter_id: str, title: str, parts: list[dict[str, Any]]) -> dict[str, Any]:
|
||
return chapter_from_items(chapter_id, title, parts)
|
||
|
||
def fold_image_chapters_into_text(input_chapters: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
folded: list[dict[str, Any]] = []
|
||
last_text_chapter: dict[str, Any] | None = None
|
||
for chapter in input_chapters:
|
||
if chapter.get("kind") != "image":
|
||
folded.append(chapter)
|
||
last_text_chapter = chapter if chapter.get("kind") == "text" else last_text_chapter
|
||
continue
|
||
if last_text_chapter is None:
|
||
folded.append(chapter)
|
||
continue
|
||
image_items = chapter.get("items") or [chapter]
|
||
last_text_chapter.setdefault("items", []).extend(image_items)
|
||
last_text_chapter.setdefault("images", []).extend(
|
||
image
|
||
for item in image_items
|
||
for image in item.get("images", [])
|
||
)
|
||
return folded
|
||
|
||
def flatten_node_files(node: dict[str, Any]) -> list[dict[str, str]]:
|
||
items: list[dict[str, str]] = []
|
||
href = node.get("href", "")
|
||
if href in rows_by_file or href in image_by_file:
|
||
items.append({"href": href, "title": node.get("title") or ""})
|
||
for child in node.get("children", []):
|
||
items.extend(flatten_node_files(child))
|
||
return items
|
||
|
||
chapters: list[dict[str, Any]] = []
|
||
assigned_files: set[str] = set()
|
||
chapter_serial = 0
|
||
|
||
def append_file_as_chapter(entry_name: str, title_override: str = "") -> None:
|
||
nonlocal chapter_serial
|
||
if entry_name in image_by_file:
|
||
images = image_by_file.get(entry_name, [])
|
||
for image_index, image in enumerate(images, start=1):
|
||
chapter_serial += 1
|
||
image_item = make_image_item(entry_name, image, title_override, image_index, len(images))
|
||
chapters.append(image_chapter_from_item(f"C{chapter_serial:04d}", image_item))
|
||
return
|
||
if entry_name in rows_by_file:
|
||
chapter_serial += 1
|
||
part = make_part(entry_name, title_override)
|
||
chapters.append(chapter_from_parts(f"C{chapter_serial:04d}", title_override or part["source_title"] or part["title"], [part]))
|
||
|
||
def append_grouped_files(title: str, files: list[str]) -> None:
|
||
nonlocal chapter_serial
|
||
chapter_items: list[dict[str, Any]] = []
|
||
text_title = title
|
||
|
||
def flush_chapter_items() -> None:
|
||
nonlocal chapter_serial, chapter_items, text_title
|
||
if not chapter_items:
|
||
return
|
||
text_parts = [item for item in chapter_items if item.get("kind") == "text"]
|
||
if not text_parts:
|
||
for item in chapter_items:
|
||
if item.get("kind") == "image":
|
||
chapter_serial += 1
|
||
chapters.append(image_chapter_from_item(f"C{chapter_serial:04d}", item))
|
||
chapter_items = []
|
||
text_title = ""
|
||
return
|
||
chapter_serial += 1
|
||
chapters.append(chapter_from_items(f"C{chapter_serial:04d}", text_title or text_parts[0]["source_title"] or text_parts[0]["title"], chapter_items))
|
||
chapter_items = []
|
||
text_title = ""
|
||
|
||
for file_index, entry_name in enumerate(files):
|
||
if entry_name in image_by_file:
|
||
images = image_by_file.get(entry_name, [])
|
||
for image_index, image in enumerate(images, start=1):
|
||
image_title = title if file_index == 0 and not chapter_items else ""
|
||
chapter_items.append(make_image_item(entry_name, image, image_title, image_index, len(images)))
|
||
elif entry_name in rows_by_file:
|
||
chapter_items.append(make_part(entry_name))
|
||
flush_chapter_items()
|
||
|
||
for spec in chapter_specs:
|
||
node_files = [entry_name for entry_name in spec.get("files", []) if entry_name not in assigned_files]
|
||
if not node_files:
|
||
continue
|
||
if len(node_files) == 1 and node_files[0] in image_by_file:
|
||
append_file_as_chapter(node_files[0], spec.get("title") or "")
|
||
else:
|
||
append_grouped_files(spec.get("title") or "", node_files)
|
||
assigned_files.update(node_files)
|
||
|
||
chapter_nodes = nav_tree
|
||
while (
|
||
len(chapter_nodes) == 1
|
||
and chapter_nodes[0].get("href") not in rows_by_file
|
||
and chapter_nodes[0].get("children")
|
||
):
|
||
chapter_nodes = chapter_nodes[0]["children"]
|
||
|
||
if not chapters:
|
||
for node in chapter_nodes:
|
||
node_files = []
|
||
seen_in_node: set[str] = set()
|
||
for item in flatten_node_files(node):
|
||
href = item["href"]
|
||
if href in assigned_files or href in seen_in_node:
|
||
continue
|
||
seen_in_node.add(href)
|
||
node_files.append(item)
|
||
if not node_files:
|
||
continue
|
||
append_grouped_files(node.get("title") or "", [item["href"] for item in node_files])
|
||
assigned_files.update(item["href"] for item in node_files)
|
||
|
||
for entry_name in content_files:
|
||
if entry_name in assigned_files:
|
||
continue
|
||
append_file_as_chapter(entry_name)
|
||
|
||
chapters.sort(key=lambda chapter: chapter.get("_sort", 10**9))
|
||
chapters = fold_image_chapters_into_text(chapters)
|
||
for index, chapter in enumerate(chapters, start=1):
|
||
chapter["id"] = f"C{index:04d}"
|
||
chapter.pop("_sort", None)
|
||
if chapter.get("kind") == "text":
|
||
for item_index, item in enumerate(chapter.get("items", []), start=1):
|
||
item["parent_chapter_id"] = chapter["id"]
|
||
item["display_level"] = 1
|
||
item["toc_title"] = item.get("title") or item.get("file_label") or item.get("id") or f"item{item_index:04d}"
|
||
else:
|
||
chapter.setdefault("display_level", 0)
|
||
|
||
return {
|
||
"chapters": chapters,
|
||
"chapter_count": len(chapters),
|
||
"text_chapter_count": sum(1 for chapter in chapters if chapter.get("kind") != "image"),
|
||
"image_chapter_count": image_item_count(chapters) or sum(1 for chapter in chapters if chapter.get("kind") == "image"),
|
||
"top_level_image_chapter_count": sum(1 for chapter in chapters if chapter.get("kind") == "image"),
|
||
"part_count": sum(len(chapter.get("parts", [])) for chapter in chapters),
|
||
"image_count": sum(
|
||
len(chapter.get("images", [])) if chapter.get("kind") == "image" else sum(
|
||
len(item.get("images", []))
|
||
for item in chapter.get("items", [])
|
||
if item.get("kind") == "image"
|
||
)
|
||
for chapter in chapters
|
||
),
|
||
"asset_count": len(collect_asset_paths_from_structure({"chapters": chapters})),
|
||
"row_count": len(rows),
|
||
}
|
||
|
||
|
||
def build_rows(session_root: Path) -> list[dict[str, Any]]:
|
||
html_entries = html_entries_in_reading_order(session_root)
|
||
rows: list[dict[str, Any]] = []
|
||
serial = 0
|
||
for file_order, entry_name in enumerate(html_entries, start=1):
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
continue
|
||
document_title = read_document_title(session_root, entry_name) or Path(entry_name).stem
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
matches = list(P_RE.finditer(text))
|
||
i = 0
|
||
file_serial = 0
|
||
while i < len(matches) - 1:
|
||
open_tag = matches[i].group(1)
|
||
attrs = open_tag[2:-1]
|
||
jp_inner = matches[i].group(2)
|
||
cn_inner = matches[i + 1].group(2)
|
||
next_open = matches[i + 1].group(1)
|
||
next_attrs = next_open[2:-1]
|
||
if is_gray_source(attrs, jp_inner) and not is_gray_source(next_attrs, cn_inner):
|
||
serial += 1
|
||
file_serial += 1
|
||
rows.append(
|
||
{
|
||
"id": f"R{serial:05d}",
|
||
"file": entry_name,
|
||
"file_order": file_order,
|
||
"file_row_index": file_serial,
|
||
"file_label": Path(entry_name).name,
|
||
"document_title": document_title,
|
||
"ja_p_index": i,
|
||
"cn_p_index": i + 1,
|
||
"jp_html": jp_inner,
|
||
"jp_text": strip_tags(jp_inner),
|
||
"cn_html": cn_inner,
|
||
"cn_text": strip_tags(cn_inner),
|
||
}
|
||
)
|
||
i += 2
|
||
else:
|
||
i += 1
|
||
write_json(session_root / "review_state" / "rows.json", rows)
|
||
return rows
|
||
|
||
|
||
def ensure_state(session_root: Path, epub_path: Path, reset: bool = False) -> dict[str, Any]:
|
||
extract_epub(epub_path, session_root, reset=reset)
|
||
rows_path = session_root / "review_state" / "rows.json"
|
||
rows = read_json(rows_path, None)
|
||
if rows is None:
|
||
rows = build_rows(session_root)
|
||
state_path = session_root / "review_state" / "state.json"
|
||
state = read_json(
|
||
state_path,
|
||
{
|
||
"version": STATE_VERSION,
|
||
"created_at": now_iso(),
|
||
"updated_at": now_iso(),
|
||
"source_epub": str(epub_path),
|
||
"edits": {},
|
||
},
|
||
)
|
||
state.setdefault("edits", {})
|
||
state["source_epub"] = str(epub_path)
|
||
write_json(state_path, state)
|
||
write_session_manifest(session_root, epub_path)
|
||
return state
|
||
|
||
|
||
def merge_rows(session_root: Path) -> list[dict[str, Any]]:
|
||
rows = read_json(session_root / "review_state" / "rows.json", [])
|
||
state = read_json(session_root / "review_state" / "state.json", {"edits": {}})
|
||
edits = state.get("edits", {})
|
||
merged = []
|
||
for row in rows:
|
||
item = dict(row)
|
||
edit = edits.get(row["id"], {})
|
||
item["current_html"] = edit.get("current_html", row["cn_html"])
|
||
item["marked"] = bool(edit.get("marked", False))
|
||
item["issue_type"] = edit.get("issue_type", "")
|
||
item["severity"] = edit.get("severity", "")
|
||
item["comment"] = edit.get("comment", "")
|
||
item["learn_note"] = edit.get("learn_note", "")
|
||
item["tags"] = edit.get("tags", "")
|
||
item["edited"] = item["current_html"] != row["cn_html"]
|
||
item["updated_at"] = edit.get("updated_at", "")
|
||
merged.append(item)
|
||
return merged
|
||
|
||
|
||
def write_feedback_summary(session_root: Path) -> Path:
|
||
rows = merge_rows(session_root)
|
||
marked = [
|
||
row
|
||
for row in rows
|
||
if row["edited"]
|
||
or row["marked"]
|
||
or row["comment"]
|
||
or row["learn_note"]
|
||
or row["issue_type"]
|
||
or row["severity"]
|
||
or row["tags"]
|
||
]
|
||
out = session_root / "feedback" / "feedback_for_codex.md"
|
||
lines = [
|
||
"# 双语 EPUB 审校反馈",
|
||
"",
|
||
f"- 更新时间:{now_iso()}",
|
||
f"- 有记录段落:{len(marked)}",
|
||
"",
|
||
"这些记录用于之后优化翻译、润色、术语和角色口吻。`修改前` 来自初始双语 EPUB,`修改后` 来自网页编辑器当前保存内容。",
|
||
"",
|
||
]
|
||
for row in marked:
|
||
lines.extend(
|
||
[
|
||
f"## {row['id']} · {row['file_label']} · P{row['cn_p_index']}",
|
||
"",
|
||
f"- 文件:`{row['file']}`",
|
||
f"- 问题类型:{row['issue_type'] or '未填写'}",
|
||
f"- 严重程度:{row['severity'] or '未填写'}",
|
||
f"- 标签:{row['tags'] or '无'}",
|
||
f"- 备注:{row['comment'] or '无'}",
|
||
f"- 可沉淀规则:{row['learn_note'] or '无'}",
|
||
"",
|
||
"日文原文:",
|
||
"",
|
||
row["jp_text"],
|
||
"",
|
||
"修改前:",
|
||
"",
|
||
strip_tags(row["cn_html"]),
|
||
"",
|
||
"修改后:",
|
||
"",
|
||
strip_tags(row["current_html"]),
|
||
"",
|
||
]
|
||
)
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
out.write_text("\n".join(lines), encoding="utf-8")
|
||
return out
|
||
|
||
|
||
def apply_edits_to_xhtml(session_root: Path) -> list[str]:
|
||
rows = merge_rows(session_root)
|
||
edited_rows = [row for row in rows if row["edited"]]
|
||
by_file: dict[str, list[dict[str, Any]]] = {}
|
||
for row in edited_rows:
|
||
by_file.setdefault(row["file"], []).append(row)
|
||
modified = []
|
||
for entry_name, items in by_file.items():
|
||
path = extracted_path(session_root, entry_name)
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
matches = list(P_RE.finditer(text))
|
||
replace_by_index = {int(row["cn_p_index"]): row["current_html"] for row in items}
|
||
parts: list[str] = []
|
||
last = 0
|
||
for index, match in enumerate(matches):
|
||
parts.append(text[last:match.start()])
|
||
if index in replace_by_index:
|
||
parts.append(match.group(1))
|
||
parts.append(replace_by_index[index])
|
||
parts.append(match.group(3))
|
||
else:
|
||
parts.append(match.group(0))
|
||
last = match.end()
|
||
parts.append(text[last:])
|
||
path.write_text("".join(parts), encoding="utf-8")
|
||
modified.append(entry_name)
|
||
write_feedback_summary(session_root)
|
||
return modified
|
||
|
||
|
||
def export_epub(session_root: Path) -> Path:
|
||
apply_edits_to_xhtml(session_root)
|
||
entries = read_json(session_root / "review_state" / "epub_entries.json", [])
|
||
manifest = read_json(session_manifest_path(session_root), {})
|
||
state_source = read_json(session_root / "review_state" / "state.json", {}).get("source_epub", "reviewed.epub")
|
||
source_name = manifest.get("source_name") or Path(state_source).name
|
||
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
out_dir = session_root / "exports"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
out_path = out_dir / f"{safe_slug(Path(source_name).stem)}_网页审校版_{ts}.epub"
|
||
with zipfile.ZipFile(out_path, "w") as zf:
|
||
names_written: set[str] = set()
|
||
mimetype = extracted_path(session_root, "mimetype")
|
||
if mimetype.exists():
|
||
zf.write(mimetype, "mimetype", compress_type=zipfile.ZIP_STORED)
|
||
names_written.add("mimetype")
|
||
for entry in entries:
|
||
name = entry["filename"]
|
||
if name in names_written:
|
||
continue
|
||
path = extracted_path(session_root, name)
|
||
if not path.exists():
|
||
continue
|
||
compress = zipfile.ZIP_STORED if name == "mimetype" else zipfile.ZIP_DEFLATED
|
||
if entry.get("is_dir"):
|
||
info = zipfile.ZipInfo(name)
|
||
zf.writestr(info, b"")
|
||
else:
|
||
zf.write(path, name, compress_type=compress)
|
||
names_written.add(name)
|
||
append_jsonl(
|
||
session_root / "feedback" / "translation_feedback.jsonl",
|
||
{"ts": now_iso(), "event": "export_epub", "output": str(out_path)},
|
||
)
|
||
return out_path
|
||
|
||
|
||
def translation_jobs_root(review_root: Path) -> Path:
|
||
return review_root / TRANSLATION_JOB_DIR_NAME
|
||
|
||
|
||
def translation_job_root(review_root: Path, job_id: str) -> Path:
|
||
safe_id = safe_slug(job_id, max_len=160)
|
||
if safe_id != job_id:
|
||
raise ValueError("invalid translation job id")
|
||
root = (translation_jobs_root(review_root) / job_id).resolve()
|
||
try:
|
||
root.relative_to(translation_jobs_root(review_root).resolve())
|
||
except ValueError as exc:
|
||
raise ValueError("invalid translation job id") from exc
|
||
return root
|
||
|
||
|
||
def translation_job_path(review_root: Path, job_id: str) -> Path:
|
||
return translation_job_root(review_root, job_id) / "job.json"
|
||
|
||
|
||
def make_translation_job_id(session_id: str) -> str:
|
||
seed = f"{session_id}-{now_iso()}-{time.time_ns()}"
|
||
digest = hashlib.sha1(seed.encode("utf-8")).hexdigest()[:8]
|
||
return f"tr_{dt.datetime.now().strftime('%Y%m%d_%H%M%S')}_{digest}"
|
||
|
||
|
||
def read_translation_job(review_root: Path, job_id: str) -> dict[str, Any]:
|
||
return read_json(translation_job_path(review_root, job_id), {})
|
||
|
||
|
||
def write_translation_job(review_root: Path, job: dict[str, Any]) -> None:
|
||
job["updated_at"] = now_iso()
|
||
write_json_atomic(translation_job_path(review_root, str(job["id"])), job)
|
||
|
||
|
||
def add_translation_job_log(job: dict[str, Any], message: str, level: str = "info") -> None:
|
||
logs = job.setdefault("logs", [])
|
||
logs.append({"ts": now_iso(), "level": level, "message": str(message)[:1200]})
|
||
if len(logs) > TRANSLATION_LOG_LIMIT:
|
||
del logs[: len(logs) - TRANSLATION_LOG_LIMIT]
|
||
|
||
|
||
def public_translation_job(job: dict[str, Any]) -> dict[str, Any]:
|
||
settings = job.get("settings") if isinstance(job.get("settings"), dict) else {}
|
||
return {
|
||
"id": job.get("id", ""),
|
||
"status": job.get("status", ""),
|
||
"source_session_id": job.get("source_session_id", ""),
|
||
"source_name": job.get("source_name", ""),
|
||
"created_at": job.get("created_at", ""),
|
||
"updated_at": job.get("updated_at", ""),
|
||
"started_at": job.get("started_at", ""),
|
||
"finished_at": job.get("finished_at", ""),
|
||
"progress": job.get("progress", {}),
|
||
"error": job.get("error", ""),
|
||
"logs": job.get("logs", []),
|
||
"output_epub": job.get("output_epub", ""),
|
||
"output_session_id": job.get("output_session_id", ""),
|
||
"settings_summary": {
|
||
"output_mode": settings.get("output_mode", "bilingual"),
|
||
"range_mode": settings.get("range_mode", "limit"),
|
||
"limit": settings.get("limit", DEFAULT_TRANSLATION_LIMIT),
|
||
"glossary_path": settings.get("glossary_path", ""),
|
||
"temperature": settings.get("temperature", 0.2),
|
||
},
|
||
}
|
||
|
||
|
||
def normalize_translation_job_settings(data: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
|
||
output_mode = str(data.get("output_mode") or "bilingual").strip().lower()
|
||
if output_mode not in {"bilingual", "translated"}:
|
||
raise ValueError("输出模式必须是 bilingual 或 translated")
|
||
range_mode = str(data.get("range_mode") or "limit").strip().lower()
|
||
if range_mode not in {"all", "limit"}:
|
||
raise ValueError("翻译范围必须是 all 或 limit")
|
||
try:
|
||
limit = int(data.get("limit") or DEFAULT_TRANSLATION_LIMIT)
|
||
except (TypeError, ValueError) as exc:
|
||
raise ValueError("试译段数必须是数字") from exc
|
||
limit = max(1, min(limit, TRANSLATION_RANGE_LIMIT_MAX))
|
||
try:
|
||
temperature = float(data.get("temperature", 0.2))
|
||
except (TypeError, ValueError) as exc:
|
||
raise ValueError("temperature 必须是数字") from exc
|
||
temperature = max(0.0, min(1.0, temperature))
|
||
def prompt_value(name: str) -> Any:
|
||
if data.get(f"use_series_{name}"):
|
||
return None
|
||
return data.get(name) or config.get(name)
|
||
|
||
glossary_value = "" if data.get("use_series_glossary_path") else str(data.get("glossary_path") or config.get("glossary_path") or "").strip()
|
||
return {
|
||
"output_mode": output_mode,
|
||
"range_mode": range_mode,
|
||
"limit": limit,
|
||
"temperature": temperature,
|
||
"glossary_path": normalize_glossary_path_value(glossary_value) if glossary_value else "",
|
||
"translation_prompt": "" if data.get("use_series_translation_prompt") else normalize_gpt_prompt("translation_prompt", prompt_value("translation_prompt")),
|
||
"format_prompt": "" if data.get("use_series_format_prompt") else normalize_gpt_prompt("format_prompt", prompt_value("format_prompt")),
|
||
"character_prompt": "" if data.get("use_series_character_prompt") else normalize_gpt_prompt("character_prompt", prompt_value("character_prompt")),
|
||
}
|
||
|
||
|
||
def html_looks_like_navigation(entry_name: str, text: str) -> bool:
|
||
basename = posixpath.basename(entry_name).lower()
|
||
if basename in {"nav.xhtml", "toc.xhtml", "toc.html", "toc.htm"} or "toc" in basename:
|
||
return True
|
||
heading = extract_heading_from_html(text)
|
||
if re.search(r"(目录|目次|contents?)", heading, flags=re.I):
|
||
return True
|
||
matches = list(P_RE.finditer(text))
|
||
if len(matches) < 4:
|
||
return False
|
||
link_count = sum(1 for match in matches if re.search(r"<a\b[^>]*href=", match.group(2), flags=re.I))
|
||
return link_count / max(1, len(matches)) >= 0.45
|
||
|
||
|
||
def extract_translation_source_items(session_root: Path) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
serial = 0
|
||
for file_order, entry_name in enumerate(html_entries_in_reading_order(session_root), start=1):
|
||
path = extracted_path(session_root, entry_name)
|
||
if not path.exists():
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
if html_looks_like_navigation(entry_name, text):
|
||
continue
|
||
document_title = read_document_title(session_root, entry_name) or Path(entry_name).stem
|
||
file_has_japanese_context = source_file_has_japanese_context(text)
|
||
file_serial = 0
|
||
for block_index, match in enumerate(SOURCE_BLOCK_RE.finditer(text)):
|
||
tag = match.group("tag").lower()
|
||
inner = match.group("inner")
|
||
if not is_translatable_source_block(inner, tag, file_has_japanese_context):
|
||
continue
|
||
source_text = paragraph_prompt_text(inner)
|
||
serial += 1
|
||
file_serial += 1
|
||
items.append(
|
||
{
|
||
"id": f"T{serial:05d}",
|
||
"file": entry_name,
|
||
"file_order": file_order,
|
||
"file_row_index": file_serial,
|
||
"file_label": Path(entry_name).name,
|
||
"document_title": document_title,
|
||
"block_index": block_index,
|
||
"p_index": block_index,
|
||
"tag": tag,
|
||
"open_tag": match.group("open"),
|
||
"close_tag": match.group("close"),
|
||
"p_open": match.group("open"),
|
||
"p_close": match.group("close"),
|
||
"source_html": inner,
|
||
"source_text": source_text,
|
||
}
|
||
)
|
||
return items
|
||
|
||
|
||
def translation_context_lines(items: list[dict[str, Any]], index: int, radius: int = TRANSLATION_CONTEXT_RADIUS) -> list[str]:
|
||
start = max(0, index - radius)
|
||
end = min(len(items), index + radius + 1)
|
||
context: list[str] = []
|
||
for idx in range(start, end):
|
||
if idx == index:
|
||
continue
|
||
item = items[idx]
|
||
direction = "前文" if idx < index else "后文"
|
||
context.append(f"{direction} {item.get('id', '')}: {truncate_prompt_text(item.get('source_text'), 260)}")
|
||
return context
|
||
|
||
|
||
def build_translation_messages(
|
||
item: dict[str, Any],
|
||
items: list[dict[str, Any]],
|
||
index: int,
|
||
settings: dict[str, Any],
|
||
glossary_lines: list[str],
|
||
) -> list[dict[str, str]]:
|
||
glossary_text = "\n".join(f"- {line}" for line in glossary_lines)
|
||
context_lines = translation_context_lines(items, index)
|
||
system_prompt = "\n\n".join(
|
||
[
|
||
"你是日语轻小说到简体中文的整书翻译助手。每次只处理一个源文段落。",
|
||
"只输出该段落的简体中文译文 HTML 片段,不解释,不输出 Markdown。",
|
||
"不得合并、拆分、补写或删除源文信息;意义优先,再改写成自然中文轻小说文风。",
|
||
]
|
||
)
|
||
user_parts = [
|
||
"【翻译内容提示词】",
|
||
str(settings.get("translation_prompt") or DEFAULT_TRANSLATION_PROMPT).strip(),
|
||
"",
|
||
"【目标段落:日文原文 HTML】",
|
||
f"<p>{sanitize_source_html_for_prompt(item.get('source_html', ''))}</p>",
|
||
]
|
||
if glossary_text:
|
||
user_parts.extend(["", "【本段命中术语】", glossary_text])
|
||
if context_lines:
|
||
user_parts.extend(["", "【少量上下文】", "\n".join(f"- {line}" for line in context_lines)])
|
||
user_parts.extend(
|
||
[
|
||
"",
|
||
"【角色状态与口吻提示词】",
|
||
str(settings.get("character_prompt") or DEFAULT_CHARACTER_PROMPT).strip(),
|
||
"",
|
||
"【格式与标点提示词】",
|
||
str(settings.get("format_prompt") or DEFAULT_FORMAT_PROMPT).strip(),
|
||
"",
|
||
"请只输出这个段落的中文译文 HTML 片段,不要包含外层 <p>。",
|
||
]
|
||
)
|
||
return [
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": "\n".join(user_parts)},
|
||
]
|
||
|
||
|
||
def add_gray_style_to_p(open_tag: str) -> str:
|
||
def replace_style(match: re.Match[str]) -> str:
|
||
quote = match.group(1)
|
||
value = match.group(2).strip()
|
||
if re.search(r"\bcolor\s*:", value, flags=re.I):
|
||
value = re.sub(r"\bcolor\s*:\s*[^;]+;?", "color: #808080;", value, count=1, flags=re.I)
|
||
return f"style={quote}{value}{quote}"
|
||
suffix = "" if not value or value.endswith(";") else ";"
|
||
return f'style={quote}{value}{suffix} color: #808080;{quote}'
|
||
|
||
if re.search(r"\sstyle\s*=", open_tag, flags=re.I):
|
||
return re.sub(r"style\s*=\s*(['\"])(.*?)\1", replace_style, open_tag, count=1, flags=re.I | re.S)
|
||
return re.sub(r">\s*$", ' style="color: #808080;">', open_tag)
|
||
|
||
|
||
def replace_translated_paragraphs(
|
||
text: str,
|
||
replacements: dict[int, dict[str, str]],
|
||
output_mode: str,
|
||
) -> str:
|
||
parts: list[str] = []
|
||
last = 0
|
||
for index, match in enumerate(SOURCE_BLOCK_RE.finditer(text)):
|
||
parts.append(text[last:match.start()])
|
||
if index not in replacements:
|
||
parts.append(match.group(0))
|
||
elif output_mode == "bilingual":
|
||
replacement = replacements[index]
|
||
source_open = add_gray_style_to_p(match.group("open"))
|
||
parts.append(source_open)
|
||
parts.append(match.group("inner"))
|
||
parts.append(match.group("close"))
|
||
parts.append(f"\n<{replacement.get('tag') or 'p'}>")
|
||
parts.append(replacement.get("html", ""))
|
||
parts.append(f"</{replacement.get('tag') or 'p'}>")
|
||
else:
|
||
replacement = replacements[index]
|
||
parts.append(match.group("open"))
|
||
parts.append(replacement.get("html", ""))
|
||
parts.append(match.group("close"))
|
||
last = match.end()
|
||
parts.append(text[last:])
|
||
return "".join(parts)
|
||
|
||
|
||
def build_translation_output_epub(
|
||
session_root: Path,
|
||
source_epub: Path,
|
||
source_name: str,
|
||
items: list[dict[str, Any]],
|
||
translations: dict[str, str],
|
||
output_mode: str,
|
||
out_path: Path,
|
||
) -> Path:
|
||
entries = read_json(session_root / "review_state" / "epub_entries.json", [])
|
||
by_file: dict[str, dict[int, dict[str, str]]] = {}
|
||
for item in items:
|
||
translated = translations.get(item["id"])
|
||
if translated is None:
|
||
continue
|
||
tag = str(item.get("tag") or "p").lower()
|
||
translated_tag = tag if tag.startswith("h") else "p"
|
||
by_file.setdefault(item["file"], {})[int(item.get("block_index", item.get("p_index", 0)))] = {
|
||
"html": translated,
|
||
"tag": translated_tag,
|
||
}
|
||
|
||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(out_path, "w") as zf:
|
||
names_written: set[str] = set()
|
||
mimetype = extracted_path(session_root, "mimetype")
|
||
if mimetype.exists():
|
||
zf.write(mimetype, "mimetype", compress_type=zipfile.ZIP_STORED)
|
||
names_written.add("mimetype")
|
||
for entry in entries:
|
||
name = entry["filename"]
|
||
if name in names_written:
|
||
continue
|
||
path = extracted_path(session_root, name)
|
||
if not path.exists():
|
||
continue
|
||
compress = zipfile.ZIP_STORED if name == "mimetype" else zipfile.ZIP_DEFLATED
|
||
if entry.get("is_dir"):
|
||
info = zipfile.ZipInfo(name)
|
||
zf.writestr(info, b"")
|
||
elif name in by_file and name.lower().endswith((".xhtml", ".html", ".htm")):
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
updated = replace_translated_paragraphs(text, by_file[name], output_mode)
|
||
zf.writestr(name, updated.encode("utf-8"), compress_type=compress)
|
||
else:
|
||
zf.write(path, name, compress_type=compress)
|
||
names_written.add(name)
|
||
return out_path
|
||
|
||
|
||
def translation_output_path(review_root: Path, source_name: str, output_mode: str) -> Path:
|
||
suffix = "中日双语" if output_mode == "bilingual" else "中文"
|
||
ts = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
stem = safe_slug(Path(source_name).stem or "translated_epub", max_len=100)
|
||
return review_root / TRANSLATION_OUTPUT_DIR_NAME / f"{stem}_AI翻译_{suffix}_{ts}.epub"
|
||
|
||
|
||
def register_output_session(review_root: Path, out_path: Path) -> str:
|
||
session_root = session_root_for(review_root, out_path)
|
||
ensure_state(session_root, out_path, reset=True)
|
||
write_session_manifest(session_root, out_path, source_name=out_path.name)
|
||
return session_public_id(session_root)
|
||
|
||
|
||
def run_translation_job(review_root: Path, job_id: str) -> None:
|
||
job = read_translation_job(review_root, job_id)
|
||
if not job:
|
||
return
|
||
translations: dict[str, str] = {}
|
||
selected_items: list[dict[str, Any]] = []
|
||
try:
|
||
job["status"] = "running"
|
||
job["started_at"] = now_iso()
|
||
job["error"] = ""
|
||
job["progress"] = {
|
||
"current": 0,
|
||
"total": 0,
|
||
"percent": 0,
|
||
"failed": 0,
|
||
"message": "正在准备源 EPUB……",
|
||
"current_file": "",
|
||
}
|
||
add_translation_job_log(job, "翻译任务已启动。")
|
||
write_translation_job(review_root, job)
|
||
|
||
source_session = session_root_from_id(review_root, str(job.get("source_session_id") or ""))
|
||
if not (source_session / "review_state" / "state.json").exists():
|
||
raise ValueError("源书籍会话不存在或不完整")
|
||
source_epub = session_epub_path(source_session)
|
||
if not source_epub.exists():
|
||
raise ValueError("源 EPUB 文件不存在")
|
||
settings = job.get("settings") if isinstance(job.get("settings"), dict) else {}
|
||
metadata = read_epub_metadata(source_session)
|
||
series_config = read_series_config(review_root, str(metadata.get("series_id") or ""))
|
||
series_overrides: list[str] = []
|
||
for key in ("translation_prompt", "format_prompt", "character_prompt", "glossary_path"):
|
||
if series_config.get(key):
|
||
settings[key] = series_config[key]
|
||
series_overrides.append(key)
|
||
if series_overrides:
|
||
job["settings"] = settings
|
||
add_translation_job_log(job, f"已套用同系列配置:{', '.join(series_overrides)}。")
|
||
config = read_gpt_config(review_root)
|
||
if not config.get("configured"):
|
||
raise ValueError("尚未配置 GPT API Key")
|
||
config = {
|
||
**config,
|
||
"base_url": str(settings.get("base_url") or config.get("base_url") or DEFAULT_GPT_BASE_URL),
|
||
"model": str(settings.get("model") or config.get("model") or DEFAULT_GPT_MODEL),
|
||
}
|
||
glossary_file = Path(str(settings.get("glossary_path") or config.get("glossary_path") or ""))
|
||
glossary = read_glossary_mapping_from_path(glossary_file)
|
||
|
||
all_items = extract_translation_source_items(source_session)
|
||
source_total = len(all_items)
|
||
if settings.get("range_mode") == "all":
|
||
selected_items = all_items
|
||
else:
|
||
selected_items = all_items[: int(settings.get("limit") or DEFAULT_TRANSLATION_LIMIT)]
|
||
if not selected_items:
|
||
raise ValueError("没有找到可翻译的日文正文段落")
|
||
|
||
job["progress"].update(
|
||
{
|
||
"total": len(selected_items),
|
||
"source_total": source_total,
|
||
"message": f"已找到 {source_total} 个日文段落,开始翻译 {len(selected_items)} 段。",
|
||
}
|
||
)
|
||
add_translation_job_log(job, job["progress"]["message"])
|
||
write_translation_job(review_root, job)
|
||
|
||
failed_count = 0
|
||
errors: list[dict[str, Any]] = []
|
||
for index, item in enumerate(selected_items):
|
||
job["progress"].update(
|
||
{
|
||
"current": index,
|
||
"percent": round(index / max(1, len(selected_items)) * 100, 1),
|
||
"message": f"正在翻译 {item['id']}({index + 1}/{len(selected_items)})",
|
||
"current_file": item.get("file", ""),
|
||
"current_item_id": item.get("id", ""),
|
||
}
|
||
)
|
||
write_translation_job(review_root, job)
|
||
glossary_lines = glossary_matches_for_source(glossary, item.get("source_html", ""), item.get("source_text", ""))
|
||
try:
|
||
raw = call_openai_compatible_chat(
|
||
config,
|
||
build_translation_messages(item, selected_items, index, settings, glossary_lines),
|
||
temperature=float(settings.get("temperature", 0.2)),
|
||
timeout=120,
|
||
)
|
||
candidate_html = sanitize_cn_html(
|
||
restore_glossary_ruby_candidates(normalize_candidate_punctuation(raw), glossary_lines)
|
||
)
|
||
if not candidate_html or not strip_tags(candidate_html):
|
||
raise RuntimeError("GPT 返回内容为空或无法作为译文保存")
|
||
translations[item["id"]] = candidate_html
|
||
except Exception as exc:
|
||
failed_count += 1
|
||
error_text = str(exc)[:900]
|
||
errors.append({"item_id": item["id"], "file": item.get("file", ""), "p_index": item.get("p_index"), "error": error_text})
|
||
translations[item["id"]] = f"【本段 AI 翻译失败:{html.escape(error_text)}】"
|
||
add_translation_job_log(job, f"{item['id']} 翻译失败:{error_text}", level="warning")
|
||
if index == 0 or (index + 1) % 5 == 0 or failed_count:
|
||
write_json_atomic(
|
||
translation_job_root(review_root, job_id) / "translations.json",
|
||
{"items": selected_items, "translations": translations, "errors": errors},
|
||
)
|
||
|
||
job["progress"].update(
|
||
{
|
||
"current": len(selected_items),
|
||
"percent": 100,
|
||
"failed": failed_count,
|
||
"message": "正在生成 EPUB……",
|
||
"current_item_id": "",
|
||
}
|
||
)
|
||
write_translation_job(review_root, job)
|
||
source_name = str(job.get("source_name") or source_epub.name)
|
||
out_path = translation_output_path(review_root, source_name, str(settings.get("output_mode") or "bilingual"))
|
||
build_translation_output_epub(
|
||
source_session,
|
||
source_epub,
|
||
source_name,
|
||
selected_items,
|
||
translations,
|
||
str(settings.get("output_mode") or "bilingual"),
|
||
out_path,
|
||
)
|
||
output_session_id = register_output_session(review_root, out_path)
|
||
job["status"] = "completed"
|
||
job["finished_at"] = now_iso()
|
||
job["output_epub"] = str(out_path)
|
||
job["output_session_id"] = output_session_id
|
||
job["errors"] = errors
|
||
job["progress"].update(
|
||
{
|
||
"message": f"翻译完成,已生成 EPUB 并加入书架。失败段落 {failed_count} 个。",
|
||
"failed": failed_count,
|
||
}
|
||
)
|
||
add_translation_job_log(job, job["progress"]["message"])
|
||
write_translation_job(review_root, job)
|
||
append_jsonl(
|
||
source_session / "feedback" / "translation_feedback.jsonl",
|
||
{
|
||
"ts": now_iso(),
|
||
"event": "translation_job_completed",
|
||
"job_id": job_id,
|
||
"output": str(out_path),
|
||
"output_session_id": output_session_id,
|
||
"selected_count": len(selected_items),
|
||
"failed_count": failed_count,
|
||
"glossary_path": str(settings.get("glossary_path") or ""),
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
job = read_translation_job(review_root, job_id) or job
|
||
job["status"] = "failed"
|
||
job["finished_at"] = now_iso()
|
||
job["error"] = str(exc)[:1200]
|
||
job.setdefault("progress", {}).update({"message": f"翻译失败:{job['error']}"})
|
||
add_translation_job_log(job, job["error"], level="error")
|
||
write_translation_job(review_root, job)
|
||
|
||
|
||
def create_app(review_root: Path, initial_epub_path: Path | None = None, reset: bool = False) -> Flask:
|
||
app = Flask(
|
||
__name__,
|
||
root_path=str(Path(__file__).resolve().parent),
|
||
static_folder="static",
|
||
static_url_path="/static",
|
||
)
|
||
review_root.mkdir(parents=True, exist_ok=True)
|
||
translation_jobs_root(review_root).mkdir(parents=True, exist_ok=True)
|
||
app.config["REVIEW_ROOT"] = review_root
|
||
app.config["APP_VERSION"] = version
|
||
app.config["SESSION_ROOT"] = None
|
||
app.config["EPUB_PATH"] = None
|
||
app.config["LOCK"] = threading.Lock()
|
||
app.config["TRANSLATION_THREADS"] = {}
|
||
|
||
def set_active_session(session_root: Path, epub_path: Path | None = None) -> None:
|
||
app.config["SESSION_ROOT"] = session_root
|
||
app.config["EPUB_PATH"] = epub_path or session_epub_path(session_root)
|
||
update_app_manifest(review_root, session_root)
|
||
|
||
def active_session() -> tuple[Path | None, Path | None]:
|
||
session_root = app.config.get("SESSION_ROOT")
|
||
epub_path = app.config.get("EPUB_PATH")
|
||
if isinstance(session_root, Path):
|
||
return session_root, epub_path if isinstance(epub_path, Path) else session_epub_path(session_root)
|
||
active_id = read_json(app_manifest_path(review_root), {}).get("active_session_id", "")
|
||
if active_id:
|
||
try:
|
||
manifest_root = session_root_from_id(review_root, active_id)
|
||
except ValueError:
|
||
return None, None
|
||
if (manifest_root / "review_state" / "state.json").exists():
|
||
set_active_session(manifest_root)
|
||
return app.config["SESSION_ROOT"], app.config["EPUB_PATH"]
|
||
return None, None
|
||
|
||
def no_active_response():
|
||
return jsonify(
|
||
{
|
||
"has_session": False,
|
||
"version": version,
|
||
"version_rules": VERSION_RULES,
|
||
"error": "尚未打开 EPUB。请先上传中日双语 EPUB,或从已有会话中选择一个。",
|
||
"review_root": str(review_root),
|
||
}
|
||
), 409
|
||
|
||
if initial_epub_path is not None:
|
||
session_root = session_root_for(review_root, initial_epub_path)
|
||
ensure_state(session_root, initial_epub_path, reset=reset)
|
||
set_active_session(session_root, initial_epub_path)
|
||
|
||
@app.route("/")
|
||
def index():
|
||
return send_from_directory(app.static_folder, "index.html")
|
||
|
||
@app.route("/api/session")
|
||
def api_session():
|
||
session_root, epub_path = active_session()
|
||
if session_root is None or epub_path is None:
|
||
return jsonify(
|
||
{
|
||
"has_session": False,
|
||
"version": version,
|
||
"version_rules": VERSION_RULES,
|
||
"review_root": str(review_root),
|
||
"sessions": list_review_sessions(review_root),
|
||
}
|
||
)
|
||
rows = merge_rows(session_root)
|
||
touched = [row for row in rows if rowTouched_py(row)]
|
||
manifest = read_json(session_manifest_path(session_root), {})
|
||
return jsonify(
|
||
{
|
||
"has_session": True,
|
||
"version": version,
|
||
"version_rules": VERSION_RULES,
|
||
"id": session_public_id(session_root),
|
||
"source_name": manifest.get("source_name") or epub_path.name,
|
||
"source_epub": str(epub_path),
|
||
"session_root": str(session_root),
|
||
"row_count": len(rows),
|
||
"touched_count": len(touched),
|
||
"marked_count": sum(1 for row in rows if row.get("marked")),
|
||
"feedback_md": str(session_root / "feedback" / "feedback_for_codex.md"),
|
||
"feedback_jsonl": str(session_root / "feedback" / "translation_feedback.jsonl"),
|
||
"reading_position": read_reading_position(session_root),
|
||
}
|
||
)
|
||
|
||
@app.route("/api/version")
|
||
def api_version():
|
||
return jsonify({"version": version, "version_rules": VERSION_RULES})
|
||
|
||
@app.route("/api/sessions")
|
||
def api_sessions():
|
||
session_root, _epub_path = active_session()
|
||
library = build_library_index(review_root)
|
||
return jsonify(
|
||
{
|
||
"review_root": str(review_root),
|
||
"version": version,
|
||
"version_rules": VERSION_RULES,
|
||
"active_session_id": session_public_id(session_root) if session_root else "",
|
||
"sessions": library["sessions"],
|
||
"series": library["series"],
|
||
"library": {
|
||
"book_count": library["book_count"],
|
||
"series_count": library["series_count"],
|
||
"updated_at": library["updated_at"],
|
||
"db_path": str(library_db_path(review_root)),
|
||
},
|
||
}
|
||
)
|
||
|
||
@app.route("/api/library")
|
||
def api_library():
|
||
return jsonify(build_library_index(review_root))
|
||
|
||
@app.route("/api/series/<series_id>/config", methods=["GET", "POST"])
|
||
def api_series_config(series_id: str):
|
||
series_id = safe_slug(series_id, max_len=120)
|
||
if request.method == "GET":
|
||
return jsonify(read_series_config(review_root, series_id))
|
||
data = request.get_json(silent=True) or {}
|
||
try:
|
||
return jsonify(save_series_config(review_root, series_id, data))
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
|
||
@app.route("/api/upload", methods=["POST"])
|
||
def api_upload():
|
||
file_storage = request.files.get("epub")
|
||
if file_storage is None or not file_storage.filename:
|
||
return jsonify({"error": "请选择一个 .epub 文件"}), 400
|
||
reset_upload = str(request.form.get("reset", "")).lower() in {"1", "true", "yes", "on"}
|
||
try:
|
||
epub_path = copy_upload_to_review_root(file_storage, review_root)
|
||
session_root = session_root_for(review_root, epub_path)
|
||
with app.config["LOCK"]:
|
||
if reset_upload and session_root.exists():
|
||
shutil.rmtree(session_root)
|
||
ensure_state(session_root, epub_path, reset=reset_upload)
|
||
write_session_manifest(session_root, epub_path, source_name=file_storage.filename)
|
||
set_active_session(session_root, epub_path)
|
||
summary = summarize_session(session_root)
|
||
summary["has_session"] = True
|
||
return jsonify(summary)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
except zipfile.BadZipFile:
|
||
return jsonify({"error": "文件不是有效的 EPUB/ZIP 包"}), 400
|
||
except Exception as exc:
|
||
return jsonify({"error": f"EPUB 解析失败:{exc}"}), 500
|
||
|
||
@app.route("/api/session/open", methods=["POST"])
|
||
def api_open_session():
|
||
data = request.get_json(silent=True) or {}
|
||
session_id = str(data.get("session_id", ""))
|
||
if not session_id:
|
||
return jsonify({"error": "缺少 session_id"}), 400
|
||
try:
|
||
session_root = session_root_from_id(review_root, session_id)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not (session_root / "review_state" / "state.json").exists():
|
||
return jsonify({"error": "会话不存在或不完整"}), 404
|
||
with app.config["LOCK"]:
|
||
set_active_session(session_root)
|
||
summary = summarize_session(session_root)
|
||
summary["has_session"] = True
|
||
return jsonify(summary)
|
||
|
||
@app.route("/api/rows")
|
||
def api_rows():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
return jsonify({"rows": merge_rows(session_root)})
|
||
|
||
@app.route("/api/structure")
|
||
def api_structure():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
return jsonify(build_structure(session_root))
|
||
|
||
@app.route("/api/reading-position", methods=["GET", "POST"])
|
||
def api_reading_position():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
if request.method == "GET":
|
||
return jsonify({"reading_position": read_reading_position(session_root)})
|
||
data = request.get_json(silent=True) or {}
|
||
if not data and request.data:
|
||
try:
|
||
data = json.loads(request.data.decode("utf-8", errors="replace"))
|
||
except json.JSONDecodeError:
|
||
data = {}
|
||
position = normalize_reading_position(data)
|
||
with app.config["LOCK"]:
|
||
state_path = session_root / "review_state" / "state.json"
|
||
state = read_json(state_path, {"edits": {}})
|
||
state["reading_position"] = position
|
||
state["updated_at"] = now_iso()
|
||
write_json(state_path, state)
|
||
return jsonify({"status": "ok", "reading_position": position})
|
||
|
||
@app.route("/api/asset/<path:entry_name>")
|
||
def api_asset(entry_name: str):
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
entry_name = urllib.parse.unquote(entry_name)
|
||
structure = build_structure(session_root)
|
||
if entry_name not in collect_asset_paths_from_structure(structure):
|
||
return jsonify({"error": "asset not allowed"}), 404
|
||
if not is_raster_image_asset(entry_name):
|
||
return jsonify({"error": "asset type not allowed"}), 415
|
||
try:
|
||
path = extracted_path(session_root, entry_name)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not path.exists() or not path.is_file():
|
||
return jsonify({"error": "asset not found"}), 404
|
||
response = make_response(send_from_directory(path.parent, path.name))
|
||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||
response.headers["Content-Security-Policy"] = "default-src 'none'; img-src 'self' data:; style-src 'none'; script-src 'none'; object-src 'none'"
|
||
return response
|
||
|
||
@app.route("/api/session/<session_id>/asset/<path:entry_name>")
|
||
def api_session_asset(session_id: str, entry_name: str):
|
||
try:
|
||
target_root = session_root_from_id(review_root, session_id)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not (target_root / "review_state" / "state.json").exists():
|
||
return jsonify({"error": "会话不存在或不完整"}), 404
|
||
entry_name = urllib.parse.unquote(entry_name)
|
||
metadata = read_epub_metadata(target_root)
|
||
allowed = {metadata.get("cover_asset_path", "")}
|
||
allowed.update(collect_asset_paths_from_structure(build_structure(target_root)))
|
||
if entry_name not in allowed:
|
||
return jsonify({"error": "asset not allowed"}), 404
|
||
if not is_raster_image_asset(entry_name):
|
||
return jsonify({"error": "asset type not allowed"}), 415
|
||
try:
|
||
path = extracted_path(target_root, entry_name)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not path.exists() or not path.is_file():
|
||
return jsonify({"error": "asset not found"}), 404
|
||
response = make_response(send_from_directory(path.parent, path.name))
|
||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||
response.headers["Content-Security-Policy"] = "default-src 'none'; img-src 'self' data:; style-src 'none'; script-src 'none'; object-src 'none'"
|
||
return response
|
||
|
||
@app.route("/api/gpt/config", methods=["GET", "POST"])
|
||
def api_gpt_config():
|
||
if request.method == "GET":
|
||
return jsonify(public_gpt_config(review_root))
|
||
data = request.get_json(silent=True) or {}
|
||
try:
|
||
if data.get("reset_prompts"):
|
||
current = read_gpt_config(review_root)
|
||
data = {
|
||
**data,
|
||
"base_url": data.get("base_url") or current["base_url"],
|
||
"model": data.get("model") or current["model"],
|
||
"keep_existing": data.get("keep_existing", True),
|
||
**gpt_prompt_defaults(),
|
||
}
|
||
return jsonify(save_gpt_config(review_root, data))
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
|
||
@app.route("/api/glossary", methods=["GET", "POST"])
|
||
def api_glossary():
|
||
if request.method == "GET":
|
||
try:
|
||
return jsonify(public_glossary(review_root))
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
data = request.get_json(silent=True) or {}
|
||
try:
|
||
requested_path = str(data.get("path") or "").strip()
|
||
if requested_path:
|
||
update_gpt_config_fields(review_root, {"glossary_path": normalize_glossary_path_value(requested_path)})
|
||
if "entries" not in data:
|
||
return jsonify(public_glossary(review_root))
|
||
entries = data.get("entries", [])
|
||
if not isinstance(entries, list):
|
||
return jsonify({"error": "entries 必须是数组"}), 400
|
||
return jsonify(save_glossary(review_root, entries))
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
|
||
@app.route("/api/glossary/entry", methods=["POST", "DELETE"])
|
||
def api_glossary_entry():
|
||
data = request.get_json(silent=True) or {}
|
||
source = str(data.get("source") or "").strip()
|
||
if not source:
|
||
return jsonify({"error": "缺少术语原文"}), 400
|
||
try:
|
||
glossary = read_glossary(review_root)
|
||
if request.method == "DELETE":
|
||
if source not in glossary:
|
||
return jsonify({"error": "术语不存在"}), 404
|
||
del glossary[source]
|
||
else:
|
||
target = str(data.get("target") or "").strip()
|
||
if not target:
|
||
return jsonify({"error": "缺少术语译名"}), 400
|
||
old_source = str(data.get("old_source") or source).strip()
|
||
if old_source and old_source != source:
|
||
glossary.pop(old_source, None)
|
||
glossary[source] = target
|
||
return jsonify(save_glossary(review_root, [{"source": key, "target": value} for key, value in glossary.items()]))
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
|
||
@app.route("/api/glossary/search")
|
||
def api_glossary_search():
|
||
term = str(request.args.get("q") or "")
|
||
try:
|
||
matches = search_glossary_entries(read_glossary(review_root), term)
|
||
return jsonify({"query": term, "count": len(matches), "matches": matches})
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
|
||
@app.route("/api/translation/defaults")
|
||
def api_translation_defaults():
|
||
config = public_gpt_config(review_root)
|
||
return jsonify(
|
||
{
|
||
"version": version,
|
||
"gpt": config,
|
||
"defaults": {
|
||
"output_mode": "bilingual",
|
||
"range_mode": "limit",
|
||
"limit": DEFAULT_TRANSLATION_LIMIT,
|
||
"temperature": 0.2,
|
||
"translation_prompt": config.get("translation_prompt") or DEFAULT_TRANSLATION_PROMPT,
|
||
"format_prompt": config.get("format_prompt") or DEFAULT_FORMAT_PROMPT,
|
||
"character_prompt": config.get("character_prompt") or DEFAULT_CHARACTER_PROMPT,
|
||
"glossary_path": config.get("glossary_path", ""),
|
||
},
|
||
"prompt_defaults": gpt_prompt_defaults(),
|
||
}
|
||
)
|
||
|
||
@app.route("/api/session/<session_id>/translation-source")
|
||
def api_translation_source(session_id: str):
|
||
try:
|
||
source_root = session_root_from_id(review_root, session_id)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not (source_root / "review_state" / "state.json").exists():
|
||
return jsonify({"error": "会话不存在或不完整"}), 404
|
||
try:
|
||
items = extract_translation_source_items(source_root)
|
||
stats = translation_source_stats(source_root)
|
||
files: dict[str, int] = {}
|
||
for item in items:
|
||
files[item["file"]] = files.get(item["file"], 0) + 1
|
||
return jsonify(
|
||
{
|
||
"session": summarize_session(source_root),
|
||
"series_config": read_series_config(
|
||
review_root,
|
||
str(read_epub_metadata(source_root).get("series_id") or "uncategorized"),
|
||
),
|
||
"source_count": len(items),
|
||
"stats": {
|
||
**{key: value for key, value in stats.items() if key != "translatable_by_file"},
|
||
"source_count": len(items),
|
||
"note": "段落统计只计算可翻译的日语正文块;空白、纯图片、目录页和不含日文线索的文本不会进入 API 翻译队列,源 EPUB 内容不会被删除。",
|
||
},
|
||
"files": [{"file": key, "count": value} for key, value in files.items()],
|
||
"sample": [
|
||
{
|
||
"id": item["id"],
|
||
"file": item["file"],
|
||
"p_index": item["p_index"],
|
||
"text": truncate_prompt_text(item["source_text"], 140),
|
||
}
|
||
for item in items[:5]
|
||
],
|
||
}
|
||
)
|
||
except Exception as exc:
|
||
return jsonify({"error": f"读取翻译源失败:{exc}"}), 500
|
||
|
||
@app.route("/api/translation/start", methods=["POST"])
|
||
def api_translation_start():
|
||
data = request.get_json(silent=True) or {}
|
||
session_id = str(data.get("session_id") or "").strip()
|
||
if not session_id:
|
||
return jsonify({"error": "缺少 session_id"}), 400
|
||
try:
|
||
source_root = session_root_from_id(review_root, session_id)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not (source_root / "review_state" / "state.json").exists():
|
||
return jsonify({"error": "会话不存在或不完整"}), 404
|
||
try:
|
||
requested_base_url = str(data.get("base_url") or "").strip()
|
||
requested_model = str(data.get("model") or "").strip()
|
||
config = read_gpt_config(review_root)
|
||
if requested_base_url and normalize_gpt_base_url(requested_base_url) != config["base_url"]:
|
||
return jsonify({"error": "整书翻译不能临时覆盖 Base URL。请先保存 API 设置。"}), 400
|
||
if requested_model and requested_model != config["model"]:
|
||
return jsonify({"error": "整书翻译不能临时覆盖模型。请先保存 API 设置。"}), 400
|
||
if not config.get("configured"):
|
||
return jsonify({"error": "尚未配置 GPT API Key。请先保存 API 设置。"}), 400
|
||
settings = normalize_translation_job_settings(data, config)
|
||
manifest = read_json(session_manifest_path(source_root), {})
|
||
source_epub = session_epub_path(source_root)
|
||
job_id = make_translation_job_id(session_id)
|
||
job = {
|
||
"id": job_id,
|
||
"status": "pending",
|
||
"source_session_id": session_id,
|
||
"source_epub": str(source_epub),
|
||
"source_name": manifest.get("source_name") or source_epub.name,
|
||
"created_at": now_iso(),
|
||
"updated_at": now_iso(),
|
||
"started_at": "",
|
||
"finished_at": "",
|
||
"progress": {
|
||
"current": 0,
|
||
"total": 0,
|
||
"percent": 0,
|
||
"failed": 0,
|
||
"message": "任务已创建,等待后台启动。",
|
||
"current_file": "",
|
||
"current_item_id": "",
|
||
},
|
||
"settings": {
|
||
**settings,
|
||
"model": config["model"],
|
||
"base_url": config["base_url"],
|
||
},
|
||
"output_epub": "",
|
||
"output_session_id": "",
|
||
"error": "",
|
||
"logs": [],
|
||
}
|
||
add_translation_job_log(job, "任务已创建。")
|
||
write_translation_job(review_root, job)
|
||
thread = threading.Thread(target=run_translation_job, args=(review_root, job_id), daemon=True)
|
||
app.config["TRANSLATION_THREADS"][job_id] = thread
|
||
thread.start()
|
||
return jsonify({"status": "ok", "job_id": job_id, "job": public_translation_job(job)})
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
except Exception as exc:
|
||
return jsonify({"error": f"创建翻译任务失败:{exc}"}), 500
|
||
|
||
@app.route("/api/translation/jobs/<job_id>")
|
||
def api_translation_job(job_id: str):
|
||
try:
|
||
job = read_translation_job(review_root, job_id)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
if not job:
|
||
return jsonify({"error": "翻译任务不存在"}), 404
|
||
return jsonify({"job": public_translation_job(job)})
|
||
|
||
@app.route("/api/row/<row_id>/retranslate", methods=["POST"])
|
||
def api_retranslate_row(row_id: str):
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
rows = merge_rows(session_root)
|
||
row = next((item for item in rows if item["id"] == row_id), None)
|
||
if row is None:
|
||
return jsonify({"error": "row not found"}), 404
|
||
data = request.get_json(silent=True) or {}
|
||
try:
|
||
config = read_gpt_config(review_root)
|
||
requested_base_url = str(data.get("base_url") or "").strip()
|
||
requested_model = str(data.get("model") or "").strip()
|
||
if requested_base_url and normalize_gpt_base_url(requested_base_url) != config["base_url"]:
|
||
return jsonify({"error": "重翻请求不能临时覆盖 Base URL。请先保存 API 设置,避免把已保存密钥发往未确认端点。"}), 400
|
||
if requested_model and requested_model != config["model"]:
|
||
return jsonify({"error": "重翻请求不能临时覆盖模型。请先保存 API 设置后再重翻。"}), 400
|
||
instruction = str(data.get("instruction") or "")
|
||
glossary_lines = glossary_matches_for_prompt(review_root, row)
|
||
raw = call_openai_compatible_chat(config, build_retranslate_messages(row, rows, config, glossary_lines, instruction))
|
||
candidate_html = sanitize_cn_html(restore_glossary_ruby_candidates(normalize_candidate_punctuation(raw), glossary_lines))
|
||
if not candidate_html:
|
||
return jsonify({"error": "GPT 返回内容为空或无法作为译文保存"}), 502
|
||
candidate_text = strip_tags(candidate_html)
|
||
record = {
|
||
"ts": now_iso(),
|
||
"event": "row_retranslated",
|
||
"row_id": row_id,
|
||
"file": row["file"],
|
||
"model": config["model"],
|
||
"base_url": config["base_url"],
|
||
"prompt_version": version,
|
||
"translation_prompt": config.get("translation_prompt", ""),
|
||
"format_prompt": config.get("format_prompt", ""),
|
||
"character_prompt": config.get("character_prompt", ""),
|
||
"glossary_path": str(glossary_path(review_root)),
|
||
"glossary_matches": glossary_lines,
|
||
"instruction": instruction[:1200],
|
||
"jp_text": row["jp_text"],
|
||
"before_cn_text": strip_tags(row.get("current_html") or row.get("cn_html") or ""),
|
||
"candidate_cn_text": candidate_text,
|
||
}
|
||
append_jsonl(session_root / "feedback" / "translation_feedback.jsonl", record)
|
||
return jsonify(
|
||
{
|
||
"status": "ok",
|
||
"row_id": row_id,
|
||
"candidate_html": candidate_html,
|
||
"candidate_text": candidate_text,
|
||
"model": config["model"],
|
||
"glossary_path": str(glossary_path(review_root)),
|
||
"glossary_matches": glossary_lines,
|
||
"updated_at": record["ts"],
|
||
}
|
||
)
|
||
except ValueError as exc:
|
||
return jsonify({"error": str(exc)}), 400
|
||
except RuntimeError as exc:
|
||
return jsonify({"error": str(exc)}), 502
|
||
|
||
@app.route("/api/row/<row_id>", methods=["POST"])
|
||
def api_save_row(row_id: str):
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
data = request.get_json(silent=True) or {}
|
||
rows = read_json(session_root / "review_state" / "rows.json", [])
|
||
base = next((row for row in rows if row["id"] == row_id), None)
|
||
if base is None:
|
||
return jsonify({"error": "row not found"}), 404
|
||
current_html = sanitize_cn_html(str(data.get("current_html", "")))
|
||
edit = {
|
||
"current_html": current_html,
|
||
"marked": bool(data.get("marked", False)),
|
||
"issue_type": str(data.get("issue_type", ""))[:80],
|
||
"severity": str(data.get("severity", ""))[:40],
|
||
"comment": str(data.get("comment", ""))[:4000],
|
||
"learn_note": str(data.get("learn_note", ""))[:4000],
|
||
"tags": str(data.get("tags", ""))[:500],
|
||
"updated_at": now_iso(),
|
||
}
|
||
with app.config["LOCK"]:
|
||
state_path = session_root / "review_state" / "state.json"
|
||
state = read_json(state_path, {"edits": {}})
|
||
state.setdefault("edits", {})[row_id] = edit
|
||
state["updated_at"] = now_iso()
|
||
write_json(state_path, state)
|
||
append_jsonl(
|
||
session_root / "feedback" / "translation_feedback.jsonl",
|
||
{
|
||
"ts": edit["updated_at"],
|
||
"event": "row_saved",
|
||
"row_id": row_id,
|
||
"file": base["file"],
|
||
"cn_p_index": base["cn_p_index"],
|
||
"edited": current_html != base["cn_html"],
|
||
"marked": edit["marked"],
|
||
"issue_type": edit["issue_type"],
|
||
"severity": edit["severity"],
|
||
"comment": edit["comment"],
|
||
"learn_note": edit["learn_note"],
|
||
"tags": edit["tags"],
|
||
"jp_text": base["jp_text"],
|
||
"before_cn_text": strip_tags(base["cn_html"]),
|
||
"after_cn_text": strip_tags(current_html),
|
||
"before_cn_html": base["cn_html"],
|
||
"after_cn_html": current_html,
|
||
},
|
||
)
|
||
return jsonify({"status": "ok", "row_id": row_id, "updated_at": edit["updated_at"]})
|
||
|
||
@app.route("/api/apply", methods=["POST"])
|
||
def api_apply():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
with app.config["LOCK"]:
|
||
modified = apply_edits_to_xhtml(session_root)
|
||
return jsonify(
|
||
{
|
||
"status": "ok",
|
||
"modified_files": modified,
|
||
"feedback_md": str(session_root / "feedback" / "feedback_for_codex.md"),
|
||
}
|
||
)
|
||
|
||
@app.route("/api/export", methods=["POST"])
|
||
def api_export():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
with app.config["LOCK"]:
|
||
out_path = export_epub(session_root)
|
||
return jsonify(
|
||
{
|
||
"status": "ok",
|
||
"output": str(out_path),
|
||
"download_url": f"/downloads/export/{urllib.parse.quote(out_path.name)}",
|
||
}
|
||
)
|
||
|
||
@app.route("/api/feedback")
|
||
def api_feedback():
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
path = write_feedback_summary(session_root)
|
||
return jsonify(
|
||
{
|
||
"status": "ok",
|
||
"feedback_md": str(path),
|
||
"feedback_jsonl": str(session_root / "feedback" / "translation_feedback.jsonl"),
|
||
"feedback_md_url": "/downloads/feedback/feedback_for_codex.md",
|
||
"feedback_jsonl_url": "/downloads/feedback/translation_feedback.jsonl",
|
||
}
|
||
)
|
||
|
||
@app.route("/downloads/export/<path:filename>")
|
||
def download_export(filename: str):
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
return send_from_directory(session_root / "exports", filename, as_attachment=True)
|
||
|
||
@app.route("/downloads/feedback/<path:filename>")
|
||
def download_feedback(filename: str):
|
||
session_root, _epub_path = active_session()
|
||
if session_root is None:
|
||
return no_active_response()
|
||
if filename not in {"feedback_for_codex.md", "translation_feedback.jsonl"}:
|
||
return jsonify({"error": "unknown feedback file"}), 404
|
||
return send_from_directory(session_root / "feedback", filename, as_attachment=True)
|
||
|
||
@app.route("/api/shutdown", methods=["POST"])
|
||
def api_shutdown():
|
||
def stop() -> None:
|
||
time.sleep(0.3)
|
||
os._exit(0)
|
||
|
||
threading.Thread(target=stop, daemon=True).start()
|
||
return jsonify({"status": "ok"})
|
||
|
||
return app
|
||
|
||
|
||
def wait_until_ready(url: str, proc: subprocess.Popen, timeout: int = 15) -> bool:
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
if proc.poll() is not None:
|
||
return False
|
||
try:
|
||
with urllib.request.urlopen(f"{url}/api/session", timeout=1) as response:
|
||
if response.status == 200:
|
||
return True
|
||
except OSError:
|
||
time.sleep(0.25)
|
||
return False
|
||
|
||
|
||
def run_daemon(args: argparse.Namespace) -> int:
|
||
port = find_free_port(args.port, args.host)
|
||
review_root = Path(args.review_root).resolve()
|
||
review_root.mkdir(parents=True, exist_ok=True)
|
||
if args.epub:
|
||
epub_path = Path(args.epub).resolve()
|
||
if not epub_path.exists():
|
||
print(f"EPUB not found: {epub_path}", file=sys.stderr)
|
||
return 1
|
||
try:
|
||
validate_epub_file(epub_path)
|
||
except ValueError as exc:
|
||
print(str(exc), file=sys.stderr)
|
||
return 1
|
||
session_root = session_root_for(review_root, epub_path)
|
||
if args.reset and session_root.exists():
|
||
shutil.rmtree(session_root)
|
||
session_root.mkdir(parents=True, exist_ok=True)
|
||
log_path = session_root / "server.log"
|
||
else:
|
||
epub_path = None
|
||
log_path = review_root / "server.log"
|
||
cmd = [
|
||
sys.executable,
|
||
str(Path(__file__).resolve()),
|
||
"--review-root",
|
||
str(review_root),
|
||
"--host",
|
||
args.host,
|
||
"--port",
|
||
str(port),
|
||
]
|
||
if epub_path is not None:
|
||
cmd.insert(2, str(epub_path))
|
||
cmd.append("--no-browser")
|
||
if args.reset:
|
||
cmd.append("--reset")
|
||
creationflags = 0
|
||
popen_kwargs: dict[str, Any] = {}
|
||
if os.name == "nt":
|
||
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS
|
||
else:
|
||
popen_kwargs["start_new_session"] = True
|
||
with log_path.open("a", encoding="utf-8") as log:
|
||
proc = subprocess.Popen(
|
||
cmd,
|
||
stdin=subprocess.DEVNULL,
|
||
stdout=log,
|
||
stderr=subprocess.STDOUT,
|
||
creationflags=creationflags,
|
||
**popen_kwargs,
|
||
)
|
||
url = f"http://{display_host(args.host)}:{port}"
|
||
if not wait_until_ready(url, proc):
|
||
print(f"review editor failed to start; see {log_path}", file=sys.stderr)
|
||
return 1
|
||
if not args.no_browser:
|
||
try:
|
||
webbrowser.open(url)
|
||
except Exception:
|
||
pass
|
||
print(url)
|
||
if epub_path is not None:
|
||
print(f"session: {session_root}")
|
||
else:
|
||
print(f"review root: {review_root}")
|
||
print(f"log: {log_path}")
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description="Open a bilingual EPUB in a browser review/editor.")
|
||
parser.add_argument("epub", nargs="?", help="Optional path to a bilingual EPUB. Omit it to upload EPUBs in the browser.")
|
||
parser.add_argument("--review-root", default=DEFAULT_REVIEW_ROOT, help="Directory for review sessions.")
|
||
parser.add_argument("--host", default="127.0.0.1", help="Host to bind. Use 0.0.0.0 for trusted LAN/server access.")
|
||
parser.add_argument("--port", type=int, default=5177, help="Port, auto-advances if busy.")
|
||
parser.add_argument("--daemon", action="store_true", help="Start in background and print URL.")
|
||
parser.add_argument("--no-browser", action="store_true", help="Do not auto-open the browser.")
|
||
parser.add_argument("--reset", action="store_true", help="Re-extract the EPUB and discard prior review state.")
|
||
return parser
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = build_parser().parse_args(argv)
|
||
epub_path = Path(args.epub).resolve() if args.epub else None
|
||
if epub_path is not None:
|
||
if not epub_path.exists():
|
||
print(f"EPUB not found: {epub_path}", file=sys.stderr)
|
||
return 1
|
||
try:
|
||
validate_epub_file(epub_path)
|
||
except ValueError as exc:
|
||
print(str(exc), file=sys.stderr)
|
||
return 1
|
||
if args.daemon:
|
||
return run_daemon(args)
|
||
|
||
review_root = Path(args.review_root).resolve()
|
||
app = create_app(review_root, epub_path, reset=args.reset)
|
||
port = find_free_port(args.port, args.host)
|
||
url = f"http://{display_host(args.host)}:{port}"
|
||
if not args.no_browser:
|
||
try:
|
||
webbrowser.open(url)
|
||
except Exception:
|
||
pass
|
||
|
||
def on_sigterm(_signum: int, _frame: Any) -> None:
|
||
sys.exit(0)
|
||
|
||
try:
|
||
signal.signal(signal.SIGTERM, on_sigterm)
|
||
except Exception:
|
||
pass
|
||
print(f"EPUB review editor: {url}")
|
||
print(f"Review root: {review_root}")
|
||
session_root = app.config.get("SESSION_ROOT")
|
||
if isinstance(session_root, Path):
|
||
print(f"Session: {session_root}")
|
||
print(f"Rows: {len(read_json(session_root / 'review_state' / 'rows.json', []))}")
|
||
else:
|
||
print("No EPUB loaded yet. Upload one in the browser.")
|
||
app.run(host=args.host, port=port, debug=False, threaded=True)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|