Merge pull request 'fix(connect): queue new files with the audiobookshelf watcher instead of a full scan' (#10) from bugfix/abs-targeted-scan into main

Reviewed-on: #10

static/app.js

@@ -81,6 +81,11 @@ function libOptions(mediaType) {
     .join("");
 }
 
+// the server keeps a copy; the url in the query makes a changed cover a new address
+function coverSrc(r) {
+  return `/api/requests/${r.id}/cover?u=${encodeURIComponent(r.cover_url)}`;
+}
+
 // ---- missing badge ----
 async function refreshMissingBadge() {
   try {
@@ -609,7 +614,7 @@ function renderRequests(status) {
   box.innerHTML = reqs
     .map(
       (r) => `<div class="card clickable" data-detail="${r.id}" tabindex="0">
-        ${r.cover_url ? `<img src="${esc(r.cover_url)}" alt="" loading="lazy">` : '<div class="nocover">?</div>'}
+        ${r.cover_url ? `<img src="${esc(coverSrc(r))}" alt="" loading="lazy">` : '<div class="nocover">?</div>'}
         <div class="card-body">
           <strong>${esc(r.title)}${r.volume != null ? ` · ${t("common.volume")} ${r.volume}` : ""}</strong>
           <span>${esc(r.authors)}</span>
@@ -888,7 +893,7 @@ function openDetail(id, status) {
   const img = $("#detail-cover");
   img.hidden = !r.cover_url;
   $("#detail-nocover").hidden = !!r.cover_url;
-  if (r.cover_url) img.src = r.cover_url;
+  if (r.cover_url) img.src = coverSrc(r);
   $("#detail-retag").hidden = !(r.status === "imported" && r.media_type === "audiobook");
   $("#detail-relocate").hidden = r.status !== "imported";
   $("#detail-dialog").showModal();

tests/test_import_flow.py

@@ -1,4 +1,5 @@
 import importlib
+import json
 import os
 import shutil
 from pathlib import Path
@@ -1228,12 +1229,13 @@ def test_connections_hear_about_committed_rows(client, monkeypatch):
     from wordarr import connect
 
     calls = []
+    watcher = {"disableWatcher": False}
 
     def handler(request):
-        calls.append((request.method, str(request.url)))
+        calls.append((request.method, str(request.url), request.content))
         if request.url.path == "/api/libraries":
             return httpx.Response(200, json={"libraries": [
-                {"id": "lib-audio", "name": "Audio",
+                {"id": "lib-audio", "name": "Audio", "settings": watcher,
                  "folders": [{"fullPath": str(client.tmp_path / "library")}]},
                 {"id": "lib-pod", "name": "Pod", "folders": [{"fullPath": "/podcasts"}]},
             ]})
@@ -1261,16 +1263,29 @@ def test_connections_hear_about_committed_rows(client, monkeypatch):
         "path": str(src), "is_dir": False, "files": [str(src)], "request_id": req_id,
     }]}).json()["results"][0]["ok"]
 
-    assert calls == [
+    # no full scan while the watcher runs: the two raced and added the book twice
+    dest = str(root / "Momo" / "Momo.epub")
+    assert [c[:2] for c in calls] == [
+        ("POST", "http://hook/in"),
+        ("GET", "http://abs/api/libraries"),
+        ("POST", "http://abs/api/watcher/update"),
+    ]
+    assert json.loads(calls[2][2]) == {
+        "libraryId": "lib-audio", "path": dest, "type": "add"}
+
+    calls.clear()
+    client.delete(f"/api/requests/{req_id}")
+    assert [c[:2] for c in calls] == [("POST", "http://hook/in")]  # nothing new on disk
+
+    calls.clear()
+    watcher["disableWatcher"] = True
+    connect.notify([{"event": "imported", "dest": dest}])
+    assert [c[:2] for c in calls] == [
         ("POST", "http://hook/in"),
         ("GET", "http://abs/api/libraries"),
         ("POST", "http://abs/api/libraries/lib-audio/scan"),
     ]
 
-    calls.clear()
-    client.delete(f"/api/requests/{req_id}")
-    assert calls == [("POST", "http://hook/in")]  # nothing new on disk, no scan
-
 
 def test_bulk_move_delete_and_count(client):
     def lib(name, media_type):
@@ -1424,3 +1439,38 @@ def test_watched_series_suggests_only_new_parts(client, monkeypatch):
 
     assert client.delete(f"/api/series/{w['id']}").status_code == 200
     assert client.get("/api/series").json() == []
+
+
+def test_covers_are_fetched_once_and_kept(client, monkeypatch):
+    import httpx
+    calls = []
+
+    async def fake_get(self, url, **kwargs):
+        calls.append(url)
+        if "dead" in url:
+            raise httpx.ConnectError("gone")
+        ctype = "text/html" if "page" in url else "image/png"
+        return httpx.Response(200, headers={"content-type": ctype}, content=b"PNG",
+                              request=httpx.Request("GET", url))
+
+    monkeypatch.setattr(httpx.AsyncClient, "get", fake_get)
+    from wordarr import config
+    monkeypatch.setattr(config, "CONFIG_DIR", client.tmp_path / "config")
+    lib = client.post("/api/libraries", json={
+        "name": "Covers", "media_type": "ebook", "root_path": str(client.tmp_path)}).json()
+
+    def request(cover_url):
+        return client.post("/api/requests?allow_duplicate=true", json={
+            "library_id": lib["id"], "title": "Buch", "cover_url": cover_url}).json()["id"]
+
+    good = request("https://covers.example/a.png")
+    for _ in range(2):
+        resp = client.get(f"/api/requests/{good}/cover")
+        assert resp.status_code == 200
+        assert resp.content == b"PNG"
+        assert resp.headers["content-type"] == "image/png"
+    assert calls == ["https://covers.example/a.png"]  # the second came from disk
+    assert [p.suffix for p in (client.tmp_path / "config" / "covers").iterdir()] == [".png"]
+
+    for url in ("https://covers.example/page", "https://covers.example/dead", "", "file:///etc/passwd"):
+        assert client.get(f"/api/requests/{request(url)}/cover").status_code == 404

wordarr/api/requests.py

@@ -1,11 +1,12 @@
 from pathlib import Path
 
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
+from fastapi.responses import FileResponse
 from pydantic import BaseModel, ConfigDict
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
-from .. import config, connect
+from .. import config, connect, covers
 from ..db import BookRequest, Library, get_session, record
 from ..importer import mover, tagger
 
@@ -237,6 +238,15 @@ def update_request(request_id: int, data: RequestIn, session: Session = Depends(
     return _out(req)
 
 
+@router.get("/{request_id}/cover")
+async def request_cover(request_id: int, session: Session = Depends(get_session)):
+    req = session.get(BookRequest, request_id)
+    if not req or not (path := await covers.cached(req.cover_url)):
+        raise HTTPException(404, "no cover")
+    # the UI puts the cover url into the query, so a changed cover is a new url
+    return FileResponse(path, headers={"Cache-Control": "max-age=31536000, immutable"})
+
+
 @router.post("/{request_id}/retag")
 def retag_request(request_id: int, background: BackgroundTasks,
                   session: Session = Depends(get_session)):

wordarr/connect.py

@@ -1,9 +1,10 @@
 """Tell other services about history rows once they are committed: a generic
-webhook gets every row, Audiobookshelf a scan of the libraries holding new
-files. Runs as a background task, so a slow or dead service never fails the
+webhook gets every row, Audiobookshelf the new files of the libraries holding
+them. Runs as a background task, so a slow or dead service never fails the
 request that caused it; errors only reach the log."""
 import logging
 import os
+from pathlib import Path
 
 import httpx
 from fastapi import BackgroundTasks
@@ -46,15 +47,39 @@ def notify(events: list[dict]) -> None:
 
 
 def _scan_audiobookshelf(client: httpx.Client, dests: list[str]) -> None:
-    """Scan only the libraries whose folders contain a destination. That needs
-    both containers to mount the media under the same path."""
+    """Tell the libraries whose folders contain a destination about the new
+    files. That needs both containers to mount the media under the same path."""
     headers = {"Authorization": f"Bearer {ABS_TOKEN}"}
     resp = client.get(f"{ABS_URL}/api/libraries", headers=headers)
     resp.raise_for_status()
     for lib in resp.json().get("libraries") or []:
         folders = [str(f.get("fullPath") or "").rstrip("/")
                    for f in lib.get("folders") or []]
-        if any(f and (d == f or d.startswith(f + "/")) for f in folders for d in dests):
+        mine = [d for d in dests
+                if any(f and (d == f or d.startswith(f + "/")) for f in folders)]
+        if not mine:
+            continue
+        if (lib.get("settings") or {}).get("disableWatcher"):
+            # the watcher endpoint drops paths of unwatched libraries
             client.post(f"{ABS_URL}/api/libraries/{lib.get('id')}/scan",
                         headers=headers).raise_for_status()
             log.info("audiobookshelf scans library %s", lib.get("name"))
+            continue
+        # a full scan racing the watcher adds the same folder twice, and it
+        # walks the whole library; this queues the files with the watcher
+        for path in _files(mine):
+            client.post(f"{ABS_URL}/api/watcher/update", headers=headers,
+                        json={"libraryId": lib.get("id"), "path": path,
+                              "type": "add"}).raise_for_status()
+        log.info("audiobookshelf watches new files in library %s", lib.get("name"))
+
+
+def _files(dests: list[str]) -> list[str]:
+    """Imports are flat: a destination is a file or a folder of files."""
+    out = []
+    for d in map(Path, dests):
+        if d.is_file():
+            out.append(str(d))
+        elif d.is_dir():
+            out += sorted(str(f) for f in d.iterdir() if f.is_file())
+    return out

wordarr/covers.py

@@ -0,0 +1,44 @@
+"""Covers are fetched once and kept under the config dir, so a dead link or a
+slow host does not take the picture with it."""
+import hashlib
+import logging
+import mimetypes
+import tempfile
+from pathlib import Path
+
+import httpx
+
+from . import config
+
+MAX_BYTES = 5 * 1024 * 1024
+log = logging.getLogger(__name__)
+
+
+# shortcut: covers of deleted requests stay on disk, add a sweep if the dir grows
+async def cached(url: str) -> Path | None:
+    """The file for this url, downloaded on first use; None if it cannot be had."""
+    if not url.startswith(("http://", "https://")):
+        return None
+    folder = config.CONFIG_DIR / "covers"
+    name = hashlib.sha256(url.encode()).hexdigest()
+    if hit := next(folder.glob(name + ".*"), None):
+        return hit
+    try:
+        async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client:
+            resp = await client.get(url)
+            resp.raise_for_status()
+    except httpx.HTTPError as exc:
+        log.warning("cover %s failed: %s", url, exc)
+        return None
+    ctype = resp.headers.get("content-type", "").split(";")[0].strip()
+    if not ctype.startswith("image/") or len(resp.content) > MAX_BYTES:
+        log.warning("cover %s is no usable image (%s, %d bytes)", url, ctype, len(resp.content))
+        return None
+    folder.mkdir(parents=True, exist_ok=True)
+    # written aside and renamed, so a parallel request never serves half a file
+    with tempfile.NamedTemporaryFile(dir=folder, suffix=".tmp", delete=False) as tmp:
+        tmp.write(resp.content)
+    path = folder / (name + (mimetypes.guess_extension(ctype) or ".img"))
+    Path(tmp.name).chmod(0o644)  # NamedTemporaryFile creates 0600
+    Path(tmp.name).replace(path)
+    return path