add(metadata): hardcover and google books as ebook providers

README.md

@@ -22,14 +22,17 @@ the same mount, otherwise moving turns into copy + delete.
 | `WORDARR_CONFIG_DIR` | `/config` | location of the SQLite database |
 | `WORDARR_AUDIBLE_REGIONS` | `de,com` | Audible marketplaces, first ranks first |
 | `WORDARR_MUSICBRAINZ_UA` | see `metadata/musicbrainz.py` | user agent MusicBrainz requires |
+| `WORDARR_HARDCOVER_TOKEN` | – | Hardcover API token, the best ebook source; without it Hardcover is skipped |
+| `WORDARR_GOOGLE_BOOKS_KEY` | – | optional, only needed when the keyless Google Books quota runs dry |
 
 ## Usage
 
 1. **Libraries** – create one per target folder with a media type, a naming
    scheme (`{Author}` `{Authors}` `{Narrator}` `{Narrators}` `{Title}` `{Year}`
    `{Series}` `{Volume}`) and optionally a language that filters the search.
-2. **Search** – look up a title (ebooks: Open Library, audiobooks: Audible plus
+2. **Search** – look up a title (ebooks: Hardcover, Google Books and Open
-   MusicBrainz, manga: AniList) and request it, or add it by hand. For a series,
+   Library, audiobooks: Audible plus MusicBrainz, manga: AniList) and request
+   it, or add it by hand. The language filter narrows ebook and audiobook hits. For a series,
    *Ganze Serie…* requests every episode at once.
 3. **Import** – scan the download folder. wordarr suggests file→request matches,
    which you confirm or correct; entries can be merged, split, or turned into

docker-compose.yml

@@ -11,4 +11,7 @@ services:
       - /mnt/library:/library
     environment:
       - WORDARR_DOWNLOAD_DIR=/mnt/downloads
+      # Token unter hardcover.app/account/api erzeugen — ohne ihn fällt die
+      # Ebook-Suche auf Google Books und Open Library zurück.
+      - WORDARR_HARDCOVER_TOKEN=${WORDARR_HARDCOVER_TOKEN:-}
     restart: unless-stopped

static/app.js

@@ -3,7 +3,12 @@ const $$ = (sel) => document.querySelectorAll(sel);
 
 let libraries = [];
 const LANGUAGE_NAMES = { german: "Deutsch", english: "Englisch" };
-const SOURCE_NAMES = { musicbrainz: "MusicBrainz" };
+const SOURCE_NAMES = {
+  musicbrainz: "MusicBrainz",
+  hardcover: "Hardcover",
+  googlebooks: "Google Books",
+  openlibrary: "Open Library",
+};
 
 async function api(path, opts = {}) {
   const resp = await fetch(path, {
@@ -188,7 +193,7 @@ $("#lib-form").addEventListener("submit", async (e) => {
 
 // ---- search & request ----
 const EMPTY_HINTS = {
-  ebook: "Keine Treffer. Tipp: ISBN ohne Bindestriche oder Titel + Autor versuchen.",
+  ebook: "Keine Treffer. Tipp: ISBN ohne Bindestriche oder Titel + Autor versuchen — und ggf. „Alle Sprachen“ wählen.",
   audiobook: "Keine Treffer. Tipp: EAN/ISBN funktioniert bei Audible nicht — nach Titel/Autor suchen.",
   comic: "Keine Treffer bei AniList (Manga). Westliche Comics per „Manuell anlegen“ erfassen.",
 };

static/index.html

@@ -30,7 +30,7 @@
             <option value="audiobook">Audiobook</option>
             <option value="comic">Comic/Manga</option>
           </select>
-          <select id="search-language" title="Nur Audiobooks: Sprache der Ausgabe">
+          <select id="search-language" title="Sprache der Ausgabe — filtert Ebook- und Audiobook-Suche">
             <option value="">Alle Sprachen</option>
             <option value="german">Deutsch</option>
             <option value="english">Englisch</option>
@@ -329,7 +329,7 @@
             name="file_template"
             placeholder="Datei-Schema (optional)"
             class="wide" />
-          <select name="language" title="Sprache der Ausgaben in dieser Library — filtert die Audible-Suche">
+          <select name="language" title="Sprache der Ausgaben in dieser Library — filtert die Metadaten-Suche">
             <option value="">Sprache: egal</option>
             <option value="german">Deutsch</option>
             <option value="english">Englisch</option>

tests/test_ebooks.py

@@ -0,0 +1,168 @@
+"""The three ebook providers and the search that merges them."""
+import asyncio
+
+import httpx
+import pytest
+
+from wordarr import metadata
+from wordarr.metadata import googlebooks, hardcover, openlibrary
+
+HARDCOVER_HIT = {
+    "document": {
+        "id": "42",
+        "title": "The Hobbit",
+        "author_names": ["J. R. R. Tolkien"],
+        "release_year": 1937,
+        "isbns": ["9780261102217"],
+        "image": {"url": "https://example.invalid/hobbit.jpg"},
+        "series_names": [],
+    }
+}
+GERMAN_EDITION = {
+    "book_id": 42,
+    "title": "Der Hobbit",
+    "isbn_13": "9783608939828",
+    "isbn_10": None,
+    "release_date": "1957-03-01",
+    "cached_image": "https://example.invalid/hobbit-de.jpg",
+}
+
+
+def mock_client(monkeypatch, module, handler):
+    real = httpx.AsyncClient
+    monkeypatch.setattr(module.httpx, "AsyncClient",
+                        lambda **kw: real(transport=httpx.MockTransport(handler), **kw))
+
+
+@pytest.fixture
+def hardcover_calls(monkeypatch):
+    seen = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        import json
+        body = json.loads(request.content)
+        seen.append((body["variables"], request.headers.get("authorization")))
+        if "editions" in body["query"]:
+            return httpx.Response(200, json={"data": {"editions": [GERMAN_EDITION]}})
+        return httpx.Response(200, json={"data": {"search": {"results": {"hits": [HARDCOVER_HIT]}}}})
+
+    monkeypatch.setattr(hardcover, "TOKEN", "test-token")
+    mock_client(monkeypatch, hardcover, handler)
+    return seen
+
+
+def test_hardcover_needs_no_network_without_a_token(monkeypatch):
+    monkeypatch.setattr(hardcover, "TOKEN", "")
+    assert asyncio.run(hardcover.search("hobbit")) == []
+
+
+def test_hardcover_parses_a_hit(hardcover_calls):
+    hit = asyncio.run(hardcover.search("hobbit"))[0]
+    assert (hit.title, hit.authors, hit.year) == ("The Hobbit", "J. R. R. Tolkien", 1937)
+    assert hit.external_id == "9780261102217"
+    assert hit.source == "hardcover"
+    assert hardcover_calls[0][1] == "Bearer test-token"
+
+
+def test_hardcover_swaps_in_the_german_edition(hardcover_calls):
+    hit = asyncio.run(hardcover.search("hobbit", "german"))[0]
+    assert (hit.title, hit.year, hit.language) == ("Der Hobbit", 1957, "german")
+    assert hit.external_id == "9783608939828"
+    assert hardcover_calls[1][0] == {"ids": [42], "lang": "de"}
+
+
+def test_hardcover_drops_books_without_an_edition_in_that_language(monkeypatch):
+    def handler(request: httpx.Request) -> httpx.Response:
+        import json
+        if "editions" in json.loads(request.content)["query"]:
+            return httpx.Response(200, json={"data": {"editions": []}})
+        return httpx.Response(200, json={"data": {"search": {"results": {"hits": [HARDCOVER_HIT]}}}})
+
+    monkeypatch.setattr(hardcover, "TOKEN", "test-token")
+    mock_client(monkeypatch, hardcover, handler)
+    assert asyncio.run(hardcover.search("hobbit", "german")) == []
+
+
+def test_google_books_restricts_the_language(monkeypatch):
+    seen = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(dict(request.url.params))
+        return httpx.Response(200, json={"items": [{"volumeInfo": {
+            "title": "Der Hobbit",
+            "authors": ["J. R. R. Tolkien"],
+            "publishedDate": "1957",
+            "language": "de",
+            "industryIdentifiers": [{"type": "ISBN_13", "identifier": "9783608939828"}],
+            "imageLinks": {"thumbnail": "http://example.invalid/de.jpg"},
+        }}]})
+
+    mock_client(monkeypatch, googlebooks, handler)
+    hit = asyncio.run(googlebooks.search("hobbit", "german"))[0]
+    assert seen[0]["langRestrict"] == "de"
+    assert (hit.title, hit.language, hit.year) == ("Der Hobbit", "german", 1957)
+    assert hit.cover_url.startswith("https://")  # the API still hands out http
+
+
+def test_google_books_searches_an_isbn_as_an_isbn(monkeypatch):
+    seen = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(dict(request.url.params))
+        return httpx.Response(200, json={"items": []})
+
+    mock_client(monkeypatch, googlebooks, handler)
+    asyncio.run(googlebooks.search("978-3-608-93982-8"))
+    assert seen[0]["q"] == "isbn:9783608939828"
+
+
+def test_open_library_filters_and_labels_the_language(monkeypatch):
+    seen = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(dict(request.url.params))
+        return httpx.Response(200, json={"docs": [{
+            "title": "Der Hobbit",
+            "author_name": ["J. R. R. Tolkien"],
+            "first_publish_year": 1957,
+            "language": ["ger", "eng"],
+            "isbn": ["9783608939828"],
+        }]})
+
+    mock_client(monkeypatch, openlibrary, handler)
+    hit = asyncio.run(openlibrary.search("hobbit", "german"))[0]
+    assert seen[0]["q"] == "hobbit language:ger"
+    assert hit.language == "german"
+    assert hit.source == "openlibrary"
+
+
+def test_ebook_search_merges_the_providers_and_survives_a_broken_one(monkeypatch):
+    async def hits(source, **fields):
+        return [metadata.MetadataResult(media_type="ebook", source=source, **fields)]
+
+    monkeypatch.setattr(hardcover, "TOKEN", "test-token")
+    monkeypatch.setattr(hardcover, "search", lambda q, lang="", fb=True: hits(
+        "hardcover", title="Der Hobbit", external_id="9783608939828"))
+    monkeypatch.setattr(googlebooks, "search", lambda q, lang="", fb=True: hits(
+        "googlebooks", title="Der Hobbit", external_id="9783608939828"))  # same edition
+
+    async def boom(q, lang="", fb=True):
+        raise httpx.ConnectError("open library is down")
+
+    monkeypatch.setattr(openlibrary, "search", boom)
+    results = asyncio.run(metadata.ebook_search("hobbit", "german"))
+    assert [r.source for r in results] == ["hardcover"]
+
+
+def test_automatic_search_asks_only_the_best_provider(monkeypatch):
+    called = []
+
+    async def record(name, q, lang):
+        called.append(name)
+        return []
+
+    monkeypatch.setattr(hardcover, "TOKEN", "")
+    monkeypatch.setattr(googlebooks, "search", lambda q, lang="", fb=True: record("gb", q, lang))
+    monkeypatch.setattr(openlibrary, "search", lambda q, lang="", fb=True: record("ol", q, lang))
+    asyncio.run(metadata.ebook_search("hobbit", "german", fallback=False))
+    assert called == ["gb"]  # no token, so Google Books stands in for Hardcover

wordarr/metadata/__init__.py

@@ -1,12 +1,63 @@
 import asyncio
 import logging
 
-from . import anilist, audible, musicbrainz, openlibrary
+from . import anilist, audible, googlebooks, hardcover, musicbrainz, openlibrary
 from .base import MetadataResult
 
 log = logging.getLogger(__name__)
 
 
+async def _gather(query: str, sources: list) -> list[list[MetadataResult]]:
+    """Ask every source in parallel; a broken one costs its hits, not the search."""
+    results = await asyncio.gather(
+        *(module.search(query, *args) for module, *args in sources),
+        return_exceptions=True,
+    )
+    out = []
+    for (module, *_), hits in zip(sources, results):
+        if isinstance(hits, BaseException):
+            log.warning("%s lookup failed for %r: %s", module.__name__, query, hits)
+            hits = []
+        out.append(hits)
+    return out
+
+
+def _merge(groups: list[list[MetadataResult]], key) -> list[MetadataResult]:
+    merged: list[MetadataResult] = []
+    seen = set()
+    for hits in groups:
+        for hit in hits:
+            if (marker := key(hit)) in seen:
+                continue
+            seen.add(marker)
+            merged.append(hit)
+    return merged
+
+
+def _ebook_key(r: MetadataResult):
+    # editions share a title but not an ISBN, so the ISBN only merges true duplicates
+    return r.external_id or (r.title.strip().lower(), r.authors.strip().lower())
+
+
+async def ebook_search(query: str, language: str = "",
+                       fallback: bool = True) -> list[MetadataResult]:
+    """Hardcover first - it is the only one with reliable German editions - then
+    Google Books, then Open Library, whose language tags are the weakest.
+
+    Hardcover needs a token and drops out silently without one. Callers that
+    search automatically (the import dialog when it opens) pass fallback=False
+    and get the single best source instead of three round trips.
+    """
+    primary = hardcover if hardcover.TOKEN else googlebooks
+    if not fallback:
+        return await primary.search(query, language)
+
+    groups = await _gather(query, [
+        (hardcover, language), (googlebooks, language), (openlibrary, language),
+    ])
+    return _merge(groups, _ebook_key)
+
+
 async def audiobook_search(query: str, language: str = "",
                            fallback: bool = True) -> list[MetadataResult]:
     """Audible first, MusicBrainz alongside it.
@@ -35,7 +86,7 @@ async def audiobook_search(query: str, language: str = "",
 
 
 PROVIDERS = {
-    "ebook": openlibrary.search,
+    "ebook": ebook_search,
     "audiobook": audiobook_search,
     "comic": anilist.search,
 }

wordarr/metadata/googlebooks.py

@@ -0,0 +1,68 @@
+"""Google Books, free and the only ebook source that filters by language well.
+
+No key needed; the anonymous quota is per client IP and can run dry on a busy
+one, so WORDARR_GOOGLE_BOOKS_KEY may be set to use a project quota instead.
+"""
+
+import os
+import re
+
+import httpx
+
+from .base import MetadataResult
+
+BASE = "https://www.googleapis.com/books/v1/volumes"
+API_KEY = os.environ.get("WORDARR_GOOGLE_BOOKS_KEY", "").strip()
+LANGUAGES = {"german": "de", "english": "en"}
+RESULT_LIMIT = 10
+
+_ISBN = re.compile(r"^(97[89])?\d{9}[\dXx]$")
+
+
+def _isbn(info: dict) -> str:
+    ids = {i.get("type"): i.get("identifier", "") for i in info.get("industryIdentifiers") or []}
+    return ids.get("ISBN_13") or ids.get("ISBN_10") or ""
+
+
+async def search(query: str, language: str = "",
+                 fallback: bool = True) -> list[MetadataResult]:
+    query = query.strip()
+    isbn = query.replace("-", "").replace(" ", "")
+    params = {
+        "q": f"isbn:{isbn}" if _ISBN.match(isbn) else query,
+        "maxResults": RESULT_LIMIT,
+        "printType": "books",
+    }
+    if language in LANGUAGES:
+        params["langRestrict"] = LANGUAGES[language]
+    if API_KEY:
+        params["key"] = API_KEY
+
+    async with httpx.AsyncClient(timeout=15) as client:
+        resp = await client.get(BASE, params=params)
+        resp.raise_for_status()
+        items = resp.json().get("items", [])
+
+    by_code = {code: name for name, code in LANGUAGES.items()}
+    results = []
+    for item in items:
+        info = item.get("volumeInfo") or {}
+        if not info.get("title"):
+            continue
+        published = str(info.get("publishedDate") or "")
+        images = info.get("imageLinks") or {}
+        cover = images.get("thumbnail") or images.get("smallThumbnail") or ""
+        results.append(
+            MetadataResult(
+                media_type="ebook",
+                source="googlebooks",
+                title=info["title"],
+                authors=", ".join(info.get("authors") or []),
+                external_id=_isbn(info),
+                year=int(published[:4]) if published[:4].isdigit() else None,
+                # only the two languages wordarr knows, the rest stays unlabelled
+                language=by_code.get(info.get("language", ""), ""),
+                cover_url=cover.replace("http://", "https://"),
+            )
+        )
+    return results

wordarr/metadata/hardcover.py

@@ -0,0 +1,154 @@
+"""Hardcover, the ebook provider with usable German metadata.
+
+Needs an API token (Settings → API in your Hardcover account, put it into
+WORDARR_HARDCOVER_TOKEN); without one the provider stays silent instead of
+breaking the combined ebook search.
+
+The GraphQL search is Typesense backed and returns its hits as untyped JSON, so
+every field is read defensively. A book row describes the work, not one edition,
+so when a language is requested the matching editions are fetched in a second
+query and their title/ISBN/cover replace the ones of the original edition.
+"""
+
+import logging
+import os
+
+import httpx
+
+from .base import MetadataResult
+
+ENDPOINT = "https://api.hardcover.app/v1/graphql"
+TOKEN = os.environ.get("WORDARR_HARDCOVER_TOKEN", "").strip()
+LANGUAGES = {"german": "de", "english": "en"}
+RESULT_LIMIT = 10
+
+log = logging.getLogger(__name__)
+
+_SEARCH = """
+query Search($q: String!, $per: Int!) {
+  search(query: $q, query_type: "Book", per_page: $per, page: 1) {
+    results
+  }
+}
+"""
+
+_EDITIONS = """
+query Editions($ids: [Int!]!, $lang: String!) {
+  editions(
+    where: {book_id: {_in: $ids}, language: {code2: {_eq: $lang}}}
+    order_by: {users_count: desc_nulls_last}
+  ) {
+    book_id
+    title
+    isbn_13
+    isbn_10
+    release_date
+    cached_image
+  }
+}
+"""
+
+
+async def _post(client: httpx.AsyncClient, query: str, variables: dict) -> dict:
+    resp = await client.post(
+        ENDPOINT,
+        json={"query": query, "variables": variables},
+        headers={"Authorization": f"Bearer {TOKEN}"},
+    )
+    resp.raise_for_status()
+    payload = resp.json()
+    if payload.get("errors"):
+        raise RuntimeError(payload["errors"][0].get("message", "hardcover error"))
+    return payload.get("data") or {}
+
+
+def _image_url(value) -> str:
+    # covers arrive as {"url": …} on books and as a json string or dict on editions
+    if isinstance(value, dict):
+        return value.get("url") or ""
+    return value if isinstance(value, str) and value.startswith("http") else ""
+
+
+def _authors(doc: dict) -> str:
+    names = doc.get("author_names")
+    if not names:
+        names = [
+            ((c or {}).get("author") or {}).get("name", "")
+            for c in doc.get("contributions") or []
+        ]
+    return ", ".join(n for n in names if n)
+
+
+def _isbn(*candidates) -> str:
+    for value in candidates:
+        if isinstance(value, list):
+            value = next((v for v in value if v), "")
+        if value:
+            return str(value)
+    return ""
+
+
+def _to_result(doc: dict, language: str) -> MetadataResult:
+    series = doc.get("series_names") or []
+    year = doc.get("release_year")
+    return MetadataResult(
+        media_type="ebook",
+        source="hardcover",
+        title=doc.get("title", ""),
+        authors=_authors(doc),
+        external_id=_isbn(doc.get("isbns")),
+        year=int(year) if str(year).isdigit() else None,
+        language=language,
+        series=series[0] if series else "",
+        cover_url=_image_url(doc.get("image")),
+    )
+
+
+async def _localize(client: httpx.AsyncClient, results: dict[int, MetadataResult],
+                    language: str) -> list[MetadataResult]:
+    """Replace each hit with its edition in the wanted language, drop the rest."""
+    data = await _post(client, _EDITIONS,
+                       {"ids": list(results), "lang": LANGUAGES[language]})
+    localized: dict[int, MetadataResult] = {}
+    for ed in data.get("editions") or []:
+        book_id = ed.get("book_id")
+        base = results.get(book_id)
+        if base is None or book_id in localized:
+            continue  # the first edition is the most read one
+        release = str(ed.get("release_date") or "")
+        localized[book_id] = base.model_copy(update={
+            "title": ed.get("title") or base.title,
+            "external_id": _isbn(ed.get("isbn_13"), ed.get("isbn_10")) or base.external_id,
+            "year": int(release[:4]) if release[:4].isdigit() else base.year,
+            "language": language,
+            "cover_url": _image_url(ed.get("cached_image")) or base.cover_url,
+        })
+    return [localized[book_id] for book_id in results if book_id in localized]
+
+
+async def search(query: str, language: str = "",
+                 fallback: bool = True) -> list[MetadataResult]:
+    if not TOKEN:
+        return []
+    async with httpx.AsyncClient(timeout=15) as client:
+        data = await _post(client, _SEARCH, {"q": query.strip(), "per": RESULT_LIMIT})
+        hits = ((data.get("search") or {}).get("results") or {}).get("hits") or []
+
+        results: dict[int, MetadataResult] = {}
+        for hit in hits:
+            doc = hit.get("document") or {}
+            book_id = doc.get("id")
+            try:
+                book_id = int(book_id)
+            except (TypeError, ValueError):
+                continue
+            if doc.get("title"):
+                results.setdefault(book_id, _to_result(doc, ""))
+
+        if language in LANGUAGES and results:
+            try:
+                return await _localize(client, results, language)
+            except (httpx.HTTPError, RuntimeError) as exc:
+                # without the edition lookup the hits are still worth showing
+                log.warning("hardcover edition lookup failed for %r: %s", query, exc)
+    return list(results.values())

wordarr/metadata/openlibrary.py

@@ -4,21 +4,36 @@ import httpx
 
 from .base import MetadataResult
 
+# Open Library tags editions with MARC codes, not ISO ones
+LANGUAGES = {"german": "ger", "english": "eng"}
+RESULT_LIMIT = 10
+
 _ISBN = re.compile(r"^(97[89])?\d{9}[\dXx]$")
 
 
+def _language(codes: list[str], wanted: str) -> str:
+    """A work lists every language it was ever published in, so a single code is
+    the only one that says something; otherwise trust the filtered query."""
+    if LANGUAGES.get(wanted) in codes:
+        return wanted
+    if len(codes) == 1:
+        return {code: name for name, code in LANGUAGES.items()}.get(codes[0], "")
+    return ""
+
+
 async def search(query: str, language: str = "",
                  fallback: bool = True) -> list[MetadataResult]:
-    # language is an Audible concept; Open Library results carry no usable language tag
     query = query.strip()
     isbn = query.replace("-", "").replace(" ", "")
+    q = f"isbn:{isbn}" if _ISBN.match(isbn) else query
+    if language in LANGUAGES:
+        q = f"{q} language:{LANGUAGES[language]}"
     async with httpx.AsyncClient(timeout=15) as client:
-        if _ISBN.match(isbn):
+        resp = await client.get("https://openlibrary.org/search.json", params={
-            params = {"q": f"isbn:{isbn}", "limit": 10}
+            "q": q,
-        else:
+            "limit": RESULT_LIMIT,
-            params = {"q": query, "limit": 10}
+            "fields": "title,author_name,first_publish_year,isbn,cover_i,key,language",
-        params["fields"] = "title,author_name,first_publish_year,isbn,cover_i,key"
+        })
-        resp = await client.get("https://openlibrary.org/search.json", params=params)
         resp.raise_for_status()
         docs = resp.json().get("docs", [])
 
@@ -29,10 +44,12 @@ async def search(query: str, language: str = "",
         results.append(
             MetadataResult(
                 media_type="ebook",
+                source="openlibrary",
                 title=doc.get("title", ""),
                 authors=", ".join(doc.get("author_name") or []),
                 external_id=isbn if _ISBN.match(isbn) else (isbns[0] if isbns else ""),
                 year=doc.get("first_publish_year"),
+                language=_language(doc.get("language") or [], language),
                 cover_url=f"https://covers.openlibrary.org/b/id/{cover}-M.jpg" if cover else "",
             )
         )