diff --git a/README.md b/README.md index 18a1c90..b50134b 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,17 @@ the same mount, otherwise moving turns into copy + delete. | `WORDARR_CONFIG_DIR` | `/config` | location of the SQLite database | | `WORDARR_AUDIBLE_REGIONS` | `de,com` | Audible marketplaces, first ranks first | | `WORDARR_MUSICBRAINZ_UA` | see `metadata/musicbrainz.py` | user agent MusicBrainz requires | +| `WORDARR_HARDCOVER_TOKEN` | – | Hardcover API token, the best ebook source; without it Hardcover is skipped | +| `WORDARR_GOOGLE_BOOKS_KEY` | – | optional, only needed when the keyless Google Books quota runs dry | ## Usage 1. **Libraries** – create one per target folder with a media type, a naming scheme (`{Author}` `{Authors}` `{Narrator}` `{Narrators}` `{Title}` `{Year}` `{Series}` `{Volume}`) and optionally a language that filters the search. -2. **Search** – look up a title (ebooks: Open Library, audiobooks: Audible plus - MusicBrainz, manga: AniList) and request it, or add it by hand. For a series, +2. **Search** – look up a title (ebooks: Hardcover, Google Books and Open + Library, audiobooks: Audible plus MusicBrainz, manga: AniList) and request + it, or add it by hand. The language filter narrows ebook and audiobook hits. For a series, *Ganze Serie…* requests every episode at once. 3. **Import** – scan the download folder. wordarr suggests file→request matches, which you confirm or correct; entries can be merged, split, or turned into diff --git a/docker-compose.yml b/docker-compose.yml index caa24be..4c246cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,4 +11,7 @@ services: - /mnt/library:/library environment: - WORDARR_DOWNLOAD_DIR=/mnt/downloads + # Token unter hardcover.app/account/api erzeugen — ohne ihn fällt die + # Ebook-Suche auf Google Books und Open Library zurück. + - WORDARR_HARDCOVER_TOKEN=${WORDARR_HARDCOVER_TOKEN:-} restart: unless-stopped diff --git a/static/app.js b/static/app.js index ffd14ed..42eb4f1 100644 --- a/static/app.js +++ b/static/app.js @@ -3,7 +3,12 @@ const $$ = (sel) => document.querySelectorAll(sel); let libraries = []; const LANGUAGE_NAMES = { german: "Deutsch", english: "Englisch" }; -const SOURCE_NAMES = { musicbrainz: "MusicBrainz" }; +const SOURCE_NAMES = { + musicbrainz: "MusicBrainz", + hardcover: "Hardcover", + googlebooks: "Google Books", + openlibrary: "Open Library", +}; async function api(path, opts = {}) { const resp = await fetch(path, { @@ -188,7 +193,7 @@ $("#lib-form").addEventListener("submit", async (e) => { // ---- search & request ---- const EMPTY_HINTS = { - ebook: "Keine Treffer. Tipp: ISBN ohne Bindestriche oder Titel + Autor versuchen.", + ebook: "Keine Treffer. Tipp: ISBN ohne Bindestriche oder Titel + Autor versuchen — und ggf. „Alle Sprachen“ wählen.", audiobook: "Keine Treffer. Tipp: EAN/ISBN funktioniert bei Audible nicht — nach Titel/Autor suchen.", comic: "Keine Treffer bei AniList (Manga). Westliche Comics per „Manuell anlegen“ erfassen.", }; diff --git a/static/index.html b/static/index.html index 33a6c2a..97fb100 100644 --- a/static/index.html +++ b/static/index.html @@ -30,7 +30,7 @@ - @@ -329,7 +329,7 @@ name="file_template" placeholder="Datei-Schema (optional)" class="wide" /> - diff --git a/tests/test_ebooks.py b/tests/test_ebooks.py new file mode 100644 index 0000000..ace9c67 --- /dev/null +++ b/tests/test_ebooks.py @@ -0,0 +1,168 @@ +"""The three ebook providers and the search that merges them.""" +import asyncio + +import httpx +import pytest + +from wordarr import metadata +from wordarr.metadata import googlebooks, hardcover, openlibrary + +HARDCOVER_HIT = { + "document": { + "id": "42", + "title": "The Hobbit", + "author_names": ["J. R. R. Tolkien"], + "release_year": 1937, + "isbns": ["9780261102217"], + "image": {"url": "https://example.invalid/hobbit.jpg"}, + "series_names": [], + } +} +GERMAN_EDITION = { + "book_id": 42, + "title": "Der Hobbit", + "isbn_13": "9783608939828", + "isbn_10": None, + "release_date": "1957-03-01", + "cached_image": "https://example.invalid/hobbit-de.jpg", +} + + +def mock_client(monkeypatch, module, handler): + real = httpx.AsyncClient + monkeypatch.setattr(module.httpx, "AsyncClient", + lambda **kw: real(transport=httpx.MockTransport(handler), **kw)) + + +@pytest.fixture +def hardcover_calls(monkeypatch): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + import json + body = json.loads(request.content) + seen.append((body["variables"], request.headers.get("authorization"))) + if "editions" in body["query"]: + return httpx.Response(200, json={"data": {"editions": [GERMAN_EDITION]}}) + return httpx.Response(200, json={"data": {"search": {"results": {"hits": [HARDCOVER_HIT]}}}}) + + monkeypatch.setattr(hardcover, "TOKEN", "test-token") + mock_client(monkeypatch, hardcover, handler) + return seen + + +def test_hardcover_needs_no_network_without_a_token(monkeypatch): + monkeypatch.setattr(hardcover, "TOKEN", "") + assert asyncio.run(hardcover.search("hobbit")) == [] + + +def test_hardcover_parses_a_hit(hardcover_calls): + hit = asyncio.run(hardcover.search("hobbit"))[0] + assert (hit.title, hit.authors, hit.year) == ("The Hobbit", "J. R. R. Tolkien", 1937) + assert hit.external_id == "9780261102217" + assert hit.source == "hardcover" + assert hardcover_calls[0][1] == "Bearer test-token" + + +def test_hardcover_swaps_in_the_german_edition(hardcover_calls): + hit = asyncio.run(hardcover.search("hobbit", "german"))[0] + assert (hit.title, hit.year, hit.language) == ("Der Hobbit", 1957, "german") + assert hit.external_id == "9783608939828" + assert hardcover_calls[1][0] == {"ids": [42], "lang": "de"} + + +def test_hardcover_drops_books_without_an_edition_in_that_language(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + import json + if "editions" in json.loads(request.content)["query"]: + return httpx.Response(200, json={"data": {"editions": []}}) + return httpx.Response(200, json={"data": {"search": {"results": {"hits": [HARDCOVER_HIT]}}}}) + + monkeypatch.setattr(hardcover, "TOKEN", "test-token") + mock_client(monkeypatch, hardcover, handler) + assert asyncio.run(hardcover.search("hobbit", "german")) == [] + + +def test_google_books_restricts_the_language(monkeypatch): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(dict(request.url.params)) + return httpx.Response(200, json={"items": [{"volumeInfo": { + "title": "Der Hobbit", + "authors": ["J. R. R. Tolkien"], + "publishedDate": "1957", + "language": "de", + "industryIdentifiers": [{"type": "ISBN_13", "identifier": "9783608939828"}], + "imageLinks": {"thumbnail": "http://example.invalid/de.jpg"}, + }}]}) + + mock_client(monkeypatch, googlebooks, handler) + hit = asyncio.run(googlebooks.search("hobbit", "german"))[0] + assert seen[0]["langRestrict"] == "de" + assert (hit.title, hit.language, hit.year) == ("Der Hobbit", "german", 1957) + assert hit.cover_url.startswith("https://") # the API still hands out http + + +def test_google_books_searches_an_isbn_as_an_isbn(monkeypatch): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(dict(request.url.params)) + return httpx.Response(200, json={"items": []}) + + mock_client(monkeypatch, googlebooks, handler) + asyncio.run(googlebooks.search("978-3-608-93982-8")) + assert seen[0]["q"] == "isbn:9783608939828" + + +def test_open_library_filters_and_labels_the_language(monkeypatch): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(dict(request.url.params)) + return httpx.Response(200, json={"docs": [{ + "title": "Der Hobbit", + "author_name": ["J. R. R. Tolkien"], + "first_publish_year": 1957, + "language": ["ger", "eng"], + "isbn": ["9783608939828"], + }]}) + + mock_client(monkeypatch, openlibrary, handler) + hit = asyncio.run(openlibrary.search("hobbit", "german"))[0] + assert seen[0]["q"] == "hobbit language:ger" + assert hit.language == "german" + assert hit.source == "openlibrary" + + +def test_ebook_search_merges_the_providers_and_survives_a_broken_one(monkeypatch): + async def hits(source, **fields): + return [metadata.MetadataResult(media_type="ebook", source=source, **fields)] + + monkeypatch.setattr(hardcover, "TOKEN", "test-token") + monkeypatch.setattr(hardcover, "search", lambda q, lang="", fb=True: hits( + "hardcover", title="Der Hobbit", external_id="9783608939828")) + monkeypatch.setattr(googlebooks, "search", lambda q, lang="", fb=True: hits( + "googlebooks", title="Der Hobbit", external_id="9783608939828")) # same edition + + async def boom(q, lang="", fb=True): + raise httpx.ConnectError("open library is down") + + monkeypatch.setattr(openlibrary, "search", boom) + results = asyncio.run(metadata.ebook_search("hobbit", "german")) + assert [r.source for r in results] == ["hardcover"] + + +def test_automatic_search_asks_only_the_best_provider(monkeypatch): + called = [] + + async def record(name, q, lang): + called.append(name) + return [] + + monkeypatch.setattr(hardcover, "TOKEN", "") + monkeypatch.setattr(googlebooks, "search", lambda q, lang="", fb=True: record("gb", q, lang)) + monkeypatch.setattr(openlibrary, "search", lambda q, lang="", fb=True: record("ol", q, lang)) + asyncio.run(metadata.ebook_search("hobbit", "german", fallback=False)) + assert called == ["gb"] # no token, so Google Books stands in for Hardcover diff --git a/wordarr/metadata/__init__.py b/wordarr/metadata/__init__.py index 8b02c83..1b4a152 100644 --- a/wordarr/metadata/__init__.py +++ b/wordarr/metadata/__init__.py @@ -1,12 +1,63 @@ import asyncio import logging -from . import anilist, audible, musicbrainz, openlibrary +from . import anilist, audible, googlebooks, hardcover, musicbrainz, openlibrary from .base import MetadataResult log = logging.getLogger(__name__) +async def _gather(query: str, sources: list) -> list[list[MetadataResult]]: + """Ask every source in parallel; a broken one costs its hits, not the search.""" + results = await asyncio.gather( + *(module.search(query, *args) for module, *args in sources), + return_exceptions=True, + ) + out = [] + for (module, *_), hits in zip(sources, results): + if isinstance(hits, BaseException): + log.warning("%s lookup failed for %r: %s", module.__name__, query, hits) + hits = [] + out.append(hits) + return out + + +def _merge(groups: list[list[MetadataResult]], key) -> list[MetadataResult]: + merged: list[MetadataResult] = [] + seen = set() + for hits in groups: + for hit in hits: + if (marker := key(hit)) in seen: + continue + seen.add(marker) + merged.append(hit) + return merged + + +def _ebook_key(r: MetadataResult): + # editions share a title but not an ISBN, so the ISBN only merges true duplicates + return r.external_id or (r.title.strip().lower(), r.authors.strip().lower()) + + +async def ebook_search(query: str, language: str = "", + fallback: bool = True) -> list[MetadataResult]: + """Hardcover first - it is the only one with reliable German editions - then + Google Books, then Open Library, whose language tags are the weakest. + + Hardcover needs a token and drops out silently without one. Callers that + search automatically (the import dialog when it opens) pass fallback=False + and get the single best source instead of three round trips. + """ + primary = hardcover if hardcover.TOKEN else googlebooks + if not fallback: + return await primary.search(query, language) + + groups = await _gather(query, [ + (hardcover, language), (googlebooks, language), (openlibrary, language), + ]) + return _merge(groups, _ebook_key) + + async def audiobook_search(query: str, language: str = "", fallback: bool = True) -> list[MetadataResult]: """Audible first, MusicBrainz alongside it. @@ -35,7 +86,7 @@ async def audiobook_search(query: str, language: str = "", PROVIDERS = { - "ebook": openlibrary.search, + "ebook": ebook_search, "audiobook": audiobook_search, "comic": anilist.search, } diff --git a/wordarr/metadata/googlebooks.py b/wordarr/metadata/googlebooks.py new file mode 100644 index 0000000..e5fa703 --- /dev/null +++ b/wordarr/metadata/googlebooks.py @@ -0,0 +1,68 @@ +"""Google Books, free and the only ebook source that filters by language well. + +No key needed; the anonymous quota is per client IP and can run dry on a busy +one, so WORDARR_GOOGLE_BOOKS_KEY may be set to use a project quota instead. +""" + +import os +import re + +import httpx + +from .base import MetadataResult + +BASE = "https://www.googleapis.com/books/v1/volumes" +API_KEY = os.environ.get("WORDARR_GOOGLE_BOOKS_KEY", "").strip() +LANGUAGES = {"german": "de", "english": "en"} +RESULT_LIMIT = 10 + +_ISBN = re.compile(r"^(97[89])?\d{9}[\dXx]$") + + +def _isbn(info: dict) -> str: + ids = {i.get("type"): i.get("identifier", "") for i in info.get("industryIdentifiers") or []} + return ids.get("ISBN_13") or ids.get("ISBN_10") or "" + + +async def search(query: str, language: str = "", + fallback: bool = True) -> list[MetadataResult]: + query = query.strip() + isbn = query.replace("-", "").replace(" ", "") + params = { + "q": f"isbn:{isbn}" if _ISBN.match(isbn) else query, + "maxResults": RESULT_LIMIT, + "printType": "books", + } + if language in LANGUAGES: + params["langRestrict"] = LANGUAGES[language] + if API_KEY: + params["key"] = API_KEY + + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.get(BASE, params=params) + resp.raise_for_status() + items = resp.json().get("items", []) + + by_code = {code: name for name, code in LANGUAGES.items()} + results = [] + for item in items: + info = item.get("volumeInfo") or {} + if not info.get("title"): + continue + published = str(info.get("publishedDate") or "") + images = info.get("imageLinks") or {} + cover = images.get("thumbnail") or images.get("smallThumbnail") or "" + results.append( + MetadataResult( + media_type="ebook", + source="googlebooks", + title=info["title"], + authors=", ".join(info.get("authors") or []), + external_id=_isbn(info), + year=int(published[:4]) if published[:4].isdigit() else None, + # only the two languages wordarr knows, the rest stays unlabelled + language=by_code.get(info.get("language", ""), ""), + cover_url=cover.replace("http://", "https://"), + ) + ) + return results diff --git a/wordarr/metadata/hardcover.py b/wordarr/metadata/hardcover.py new file mode 100644 index 0000000..f8c0487 --- /dev/null +++ b/wordarr/metadata/hardcover.py @@ -0,0 +1,154 @@ +"""Hardcover, the ebook provider with usable German metadata. + +Needs an API token (Settings → API in your Hardcover account, put it into +WORDARR_HARDCOVER_TOKEN); without one the provider stays silent instead of +breaking the combined ebook search. + +The GraphQL search is Typesense backed and returns its hits as untyped JSON, so +every field is read defensively. A book row describes the work, not one edition, +so when a language is requested the matching editions are fetched in a second +query and their title/ISBN/cover replace the ones of the original edition. +""" + +import logging +import os + +import httpx + +from .base import MetadataResult + +ENDPOINT = "https://api.hardcover.app/v1/graphql" +TOKEN = os.environ.get("WORDARR_HARDCOVER_TOKEN", "").strip() +LANGUAGES = {"german": "de", "english": "en"} +RESULT_LIMIT = 10 + +log = logging.getLogger(__name__) + +_SEARCH = """ +query Search($q: String!, $per: Int!) { + search(query: $q, query_type: "Book", per_page: $per, page: 1) { + results + } +} +""" + +_EDITIONS = """ +query Editions($ids: [Int!]!, $lang: String!) { + editions( + where: {book_id: {_in: $ids}, language: {code2: {_eq: $lang}}} + order_by: {users_count: desc_nulls_last} + ) { + book_id + title + isbn_13 + isbn_10 + release_date + cached_image + } +} +""" + + +async def _post(client: httpx.AsyncClient, query: str, variables: dict) -> dict: + resp = await client.post( + ENDPOINT, + json={"query": query, "variables": variables}, + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + resp.raise_for_status() + payload = resp.json() + if payload.get("errors"): + raise RuntimeError(payload["errors"][0].get("message", "hardcover error")) + return payload.get("data") or {} + + +def _image_url(value) -> str: + # covers arrive as {"url": …} on books and as a json string or dict on editions + if isinstance(value, dict): + return value.get("url") or "" + return value if isinstance(value, str) and value.startswith("http") else "" + + +def _authors(doc: dict) -> str: + names = doc.get("author_names") + if not names: + names = [ + ((c or {}).get("author") or {}).get("name", "") + for c in doc.get("contributions") or [] + ] + return ", ".join(n for n in names if n) + + +def _isbn(*candidates) -> str: + for value in candidates: + if isinstance(value, list): + value = next((v for v in value if v), "") + if value: + return str(value) + return "" + + +def _to_result(doc: dict, language: str) -> MetadataResult: + series = doc.get("series_names") or [] + year = doc.get("release_year") + return MetadataResult( + media_type="ebook", + source="hardcover", + title=doc.get("title", ""), + authors=_authors(doc), + external_id=_isbn(doc.get("isbns")), + year=int(year) if str(year).isdigit() else None, + language=language, + series=series[0] if series else "", + cover_url=_image_url(doc.get("image")), + ) + + +async def _localize(client: httpx.AsyncClient, results: dict[int, MetadataResult], + language: str) -> list[MetadataResult]: + """Replace each hit with its edition in the wanted language, drop the rest.""" + data = await _post(client, _EDITIONS, + {"ids": list(results), "lang": LANGUAGES[language]}) + localized: dict[int, MetadataResult] = {} + for ed in data.get("editions") or []: + book_id = ed.get("book_id") + base = results.get(book_id) + if base is None or book_id in localized: + continue # the first edition is the most read one + release = str(ed.get("release_date") or "") + localized[book_id] = base.model_copy(update={ + "title": ed.get("title") or base.title, + "external_id": _isbn(ed.get("isbn_13"), ed.get("isbn_10")) or base.external_id, + "year": int(release[:4]) if release[:4].isdigit() else base.year, + "language": language, + "cover_url": _image_url(ed.get("cached_image")) or base.cover_url, + }) + return [localized[book_id] for book_id in results if book_id in localized] + + +async def search(query: str, language: str = "", + fallback: bool = True) -> list[MetadataResult]: + if not TOKEN: + return [] + async with httpx.AsyncClient(timeout=15) as client: + data = await _post(client, _SEARCH, {"q": query.strip(), "per": RESULT_LIMIT}) + hits = ((data.get("search") or {}).get("results") or {}).get("hits") or [] + + results: dict[int, MetadataResult] = {} + for hit in hits: + doc = hit.get("document") or {} + book_id = doc.get("id") + try: + book_id = int(book_id) + except (TypeError, ValueError): + continue + if doc.get("title"): + results.setdefault(book_id, _to_result(doc, "")) + + if language in LANGUAGES and results: + try: + return await _localize(client, results, language) + except (httpx.HTTPError, RuntimeError) as exc: + # without the edition lookup the hits are still worth showing + log.warning("hardcover edition lookup failed for %r: %s", query, exc) + return list(results.values()) diff --git a/wordarr/metadata/openlibrary.py b/wordarr/metadata/openlibrary.py index fa955f4..5f59e25 100644 --- a/wordarr/metadata/openlibrary.py +++ b/wordarr/metadata/openlibrary.py @@ -4,21 +4,36 @@ import httpx from .base import MetadataResult +# Open Library tags editions with MARC codes, not ISO ones +LANGUAGES = {"german": "ger", "english": "eng"} +RESULT_LIMIT = 10 + _ISBN = re.compile(r"^(97[89])?\d{9}[\dXx]$") +def _language(codes: list[str], wanted: str) -> str: + """A work lists every language it was ever published in, so a single code is + the only one that says something; otherwise trust the filtered query.""" + if LANGUAGES.get(wanted) in codes: + return wanted + if len(codes) == 1: + return {code: name for name, code in LANGUAGES.items()}.get(codes[0], "") + return "" + + async def search(query: str, language: str = "", fallback: bool = True) -> list[MetadataResult]: - # language is an Audible concept; Open Library results carry no usable language tag query = query.strip() isbn = query.replace("-", "").replace(" ", "") + q = f"isbn:{isbn}" if _ISBN.match(isbn) else query + if language in LANGUAGES: + q = f"{q} language:{LANGUAGES[language]}" async with httpx.AsyncClient(timeout=15) as client: - if _ISBN.match(isbn): - params = {"q": f"isbn:{isbn}", "limit": 10} - else: - params = {"q": query, "limit": 10} - params["fields"] = "title,author_name,first_publish_year,isbn,cover_i,key" - resp = await client.get("https://openlibrary.org/search.json", params=params) + resp = await client.get("https://openlibrary.org/search.json", params={ + "q": q, + "limit": RESULT_LIMIT, + "fields": "title,author_name,first_publish_year,isbn,cover_i,key,language", + }) resp.raise_for_status() docs = resp.json().get("docs", []) @@ -29,10 +44,12 @@ async def search(query: str, language: str = "", results.append( MetadataResult( media_type="ebook", + source="openlibrary", title=doc.get("title", ""), authors=", ".join(doc.get("author_name") or []), external_id=isbn if _ISBN.match(isbn) else (isbns[0] if isbns else ""), year=doc.get("first_publish_year"), + language=_language(doc.get("language") or [], language), cover_url=f"https://covers.openlibrary.org/b/id/{cover}-M.jpg" if cover else "", ) )