fix(import): survive file names that are not valid utf-8

tests/test_fsnames.py

@@ -0,0 +1,37 @@
+"""Names that are not valid UTF-8 must not break the API."""
+import json
+import os
+from pathlib import Path
+
+from wordarr.fsnames import decode_path, encode_path, is_broken, readable
+
+# a Latin-1 "ß" in an otherwise normal name, as it comes from older rips
+BROKEN = os.fsdecode(b"Die drei ??? - Die Fu\xdfball-Falle")
+
+
+def test_detects_and_cleans_broken_names():
+    assert is_broken(BROKEN)
+    assert not is_broken("Die drei ??? - Die Fußball-Falle")
+    shown = readable(BROKEN)
+    assert shown.startswith("Die drei ??? - Die Fu") and "�" in shown
+    # the cleaned name survives the same encoding that used to raise
+    json.dumps({"name": shown}, ensure_ascii=False).encode("utf-8")
+
+
+def test_encoded_paths_round_trip(tmp_path):
+    folder = tmp_path / BROKEN
+    folder.mkdir()
+    (folder / "01.mp3").write_bytes(b"x")
+
+    encoded = encode_path(str(folder))
+    json.dumps({"path": encoded}, ensure_ascii=False).encode("utf-8")  # transportable
+    assert not is_broken(encoded)
+    assert Path(decode_path(encoded)) == folder
+    assert (Path(decode_path(encoded)) / "01.mp3").read_bytes() == b"x"
+
+
+def test_plain_names_are_left_alone():
+    plain = "/downloads/Die drei ??? - Die Fußball-Falle"
+    assert encode_path(plain) == plain
+    assert decode_path(plain) == plain
+    assert readable(plain) == plain

tests/test_import_flow.py

@@ -1,4 +1,5 @@
 import importlib
+import os
 import shutil
 from pathlib import Path
 
@@ -928,3 +929,39 @@ def test_two_titles_rendering_to_the_same_name_do_not_overwrite(client):
     assert b"Teil 1 Datei 1" in content and b"Teil 2" not in content
     assert sorted(p.name for p in (client.downloads / "teil2").iterdir()) == \
         ["01.mp3", "02.mp3"]
+
+
+def test_scan_and_import_survive_a_name_that_is_not_utf8(client):
+    """A single Latin-1 byte in a folder name used to fail the whole scan with a
+    500 (UnicodeEncodeError: surrogates not allowed)."""
+    broken = os.fsdecode(b"Die drei ??? - Die Fu\xdfball-Falle")
+    folder = client.downloads / broken
+    folder.mkdir()
+    for i in (1, 2):
+        (folder / f"{i:02d}.mp3").write_bytes(b"")
+
+    root = client.tmp_path / "library" / "kids"
+    lib_id = client.post("/api/libraries", json={
+        "name": "kids", "media_type": "audiobook", "root_path": str(root),
+        "folder_template": "{Title}", "file_template": "{Title}",
+    }).json()["id"]
+    req_id = client.post("/api/requests", json={
+        "library_id": lib_id, "title": "Die drei ??? und die Fußball-Falle",
+    }).json()["id"]
+
+    resp = client.get("/api/import/scan")
+    assert resp.status_code == 200
+    item = resp.json()["items"][0]
+    assert "�" in item["name"]          # shown with a replacement character
+    assert item["path"].startswith("b64:")   # but transported losslessly
+
+    res = client.post("/api/import", json={"items": [{
+        "path": item["path"], "is_dir": True, "files": item["files"],
+        "request_id": req_id,
+    }]}).json()["results"][0]
+    assert res["ok"], res
+    assert sorted(p.name for p in Path(res["dest"]).iterdir()) == [
+        "Die drei ??? und die Fußball-Falle - Part 01.mp3",
+        "Die drei ??? und die Fußball-Falle - Part 02.mp3",
+    ]
+    assert not folder.exists()