From 7461fddb7adde3b838051e4fca2977e744370328 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 19 Aug 2026 20:25:32 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=205=20=E2=80=93=20PDF/Bild-Beleg-?= =?UTF-8?q?Import=20(lokal,=20Tesseract-OCR)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - services/ocr: PDF-Text via PyMuPDF, OCR (Tesseract) für Scans/Bilder, heuristischer Parser (Datum + Betrag je Zeile, CH/DE/US-Beträge). - routers/documents: /documents/preview (Upload -> Text -> Kandidaten -> gemeinsame Prüf-/Buchungslogik des CSV-Imports). csv_import refaktoriert: detect_rows() von CSV und Beleg gemeinsam genutzt. - requirements: PyMuPDF, pytesseract, pillow; install.sh installiert tesseract-ocr (+ deu). pytest: Heuristik-Parser und PDF-Vorschau. Frontend: - Import-Seite: CSV/PDF-Umschalter; PDF-Modus mit Beleg-Auslesen, extrahiertem Text zur Kontrolle und Best-Effort-Hinweis; gleiche Prüfansicht + Buchen. Co-Authored-By: Claude Opus 4.8 --- backend/app/main.py | 2 + backend/app/routers/documents.py | 34 ++++++++ backend/app/services/csv_import.py | 77 ++++++++++------- backend/app/services/ocr.py | 127 +++++++++++++++++++++++++++++ backend/requirements.txt | 5 ++ backend/tests/test_documents.py | 55 +++++++++++++ deploy/install.sh | 3 +- docs/ROADMAP.md | 5 +- web/src/api.js | 3 + web/src/finance.css | 11 +++ web/src/locales/de.json | 9 +- web/src/locales/en.json | 9 +- web/src/pages/Import.jsx | 54 ++++++++++-- 13 files changed, 353 insertions(+), 41 deletions(-) create mode 100644 backend/app/routers/documents.py create mode 100644 backend/app/services/ocr.py create mode 100644 backend/tests/test_documents.py diff --git a/backend/app/main.py b/backend/app/main.py index 44f8a9b..2555ecf 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -24,6 +24,7 @@ from .routers import ( budgets, categories, dashboard, + documents, imports, rates, settings as settings_router, @@ -105,6 +106,7 @@ api.include_router(transactions.router) api.include_router(budgets.router) api.include_router(dashboard.router) api.include_router(imports.router) +api.include_router(documents.router) api.include_router(rates.router) app.include_router(api) diff --git a/backend/app/routers/documents.py b/backend/app/routers/documents.py new file mode 100644 index 0000000..152b872 --- /dev/null +++ b/backend/app/routers/documents.py @@ -0,0 +1,34 @@ +"""PDF-/Bild-Beleg-Import (Phase 5, lokal). Nutzt die Prüf-/Buchungslogik des CSV-Imports.""" + +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from sqlalchemy.orm import Session + +from ..database import get_db +from ..deps import get_current_user, get_owned +from ..models import Account, User +from ..services import csv_import, ocr + +router = APIRouter(prefix="/documents", tags=["documents"]) + + +@router.post("/preview") +async def preview( + file: UploadFile = File(...), + account_id: int = Form(...), + db: Session = Depends(get_db), + user: User = Depends(get_current_user), +) -> dict: + account = get_owned(db, Account, account_id, user) + data = await file.read() + try: + text = ocr.extract_text(data, file.filename or "", file.content_type or "") + except ocr.ExtractionError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + candidates = ocr.parse_transactions(text) + rows = csv_import.detect_rows(db, user, account, candidates) + return { + "rows": rows, + "text": text[:8000], + "count": len(rows), + "filename": file.filename or "", + } diff --git a/backend/app/services/csv_import.py b/backend/app/services/csv_import.py index cb1198c..9269831 100644 --- a/backend/app/services/csv_import.py +++ b/backend/app/services/csv_import.py @@ -153,16 +153,45 @@ def _apply_rules(rules: list[CategorizationRule], payee: str, purpose: str) -> i return None -def build_preview(db: Session, user: User, account: Account, template: ImportTemplate, data: bytes) -> list[dict]: +def parse_file_to_candidates(data: bytes, template: ImportTemplate) -> list[dict]: + """Zerlegt die Datei anhand des Templates in rohe Kandidaten-Zeilen.""" cmap = json.loads(template.column_map or "{}") def col(field: str) -> int | None: idx = cmap.get(field) return idx if isinstance(idx, int) and idx >= 0 else None - rows = read_rows(data, template) + out: list[dict] = [] + for r in read_rows(data, template): + def cell(field: str) -> str: + c = col(field) + return r[c].strip() if c is not None and c < len(r) else "" - # Identifikatoren aller Konten des Nutzers -> Konto-ID (für Transfer-Erkennung). + d = parse_date(cell("date"), template.date_format) + if template.amount_mode == AmountMode.debit_credit: + debit = parse_amount(cell("debit"), template.decimal_sep) + credit = parse_amount(cell("credit"), template.decimal_sep) + amount = (credit or 0) - (abs(debit) if debit else 0) + elif template.amount_mode == AmountMode.abs_expense: + a = parse_amount(cell("amount"), template.decimal_sep) + amount = -abs(a) if a is not None else None + else: + amount = parse_amount(cell("amount"), template.decimal_sep) + + out.append({ + "date": d.isoformat() if d else None, + "amount_cents": amount, + "payee": cell("payee") or None, + "purpose": cell("purpose") or None, + "counterparty_iban": normalize_iban(cell("counterparty_iban")) or None, + }) + return out + + +def detect_rows(db: Session, user: User, account: Account, candidates: list[dict]) -> list[dict]: + """Reichert Kandidaten an: Typ, Auto-Kategorie, interner Transfer, Duplikat. + + Wird von CSV- UND Beleg-Import genutzt (gleiche Prüf-/Buchungslogik).""" idents: dict[str, int] = {} for value_norm, acc_id in ( db.query(AccountIdentifier.value_norm, AccountIdentifier.account_id) @@ -177,39 +206,25 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem .order_by(CategorizationRule.priority.desc(), CategorizationRule.hits.desc()) .all() ) - existing = { h[0] for h in db.query(Transaction.import_hash).filter( Transaction.owner_id == user.id, Transaction.import_hash.isnot(None) ) } - seen_in_file: set[str] = set() + seen: set[str] = set() out: list[dict] = [] - for i, r in enumerate(rows): - def cell(field: str) -> str: - c = col(field) - return r[c].strip() if c is not None and c < len(r) else "" - - d = parse_date(cell("date"), template.date_format) - - if template.amount_mode == AmountMode.debit_credit: - debit = parse_amount(cell("debit"), template.decimal_sep) - credit = parse_amount(cell("credit"), template.decimal_sep) - amount = (credit or 0) - (abs(debit) if debit else 0) - elif template.amount_mode == AmountMode.abs_expense: - a = parse_amount(cell("amount"), template.decimal_sep) - amount = -abs(a) if a is not None else None - else: - amount = parse_amount(cell("amount"), template.decimal_sep) - - payee = cell("payee") - purpose = cell("purpose") - iban = normalize_iban(cell("counterparty_iban")) + for i, cand in enumerate(candidates): + d_iso = cand.get("date") + amount = cand.get("amount_cents") + payee = cand.get("payee") + purpose = cand.get("purpose") + iban = cand.get("counterparty_iban") + d = date.fromisoformat(d_iso) if d_iso else None valid = d is not None and amount is not None and amount != 0 - ttype = "transfer" if False else ("expense" if (amount or 0) < 0 else "income") + ttype = "expense" if (amount or 0) < 0 else "income" category_id: int | None = None to_account_id: int | None = None @@ -218,11 +233,11 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem ttype = "transfer" to_account_id = other else: - category_id = _apply_rules(rules, payee, purpose) + category_id = _apply_rules(rules, payee or "", purpose or "") h = import_hash(account.id, d, amount, payee) - is_dup = h in existing or h in seen_in_file - seen_in_file.add(h) + is_dup = h in existing or h in seen + seen.add(h) out.append({ "index": i, @@ -242,6 +257,10 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem return out +def build_preview(db: Session, user: User, account: Account, template: ImportTemplate, data: bytes) -> list[dict]: + return detect_rows(db, user, account, parse_file_to_candidates(data, template)) + + # --- Verbuchen ------------------------------------------------------------- diff --git a/backend/app/services/ocr.py b/backend/app/services/ocr.py new file mode 100644 index 0000000..8e9c573 --- /dev/null +++ b/backend/app/services/ocr.py @@ -0,0 +1,127 @@ +"""PDF-/Bild-Belege lokal auslesen (kein Cloud-Dienst). + +- Digitale PDFs: Text direkt via PyMuPDF. +- Gescannte PDFs / Bilder: OCR via Tesseract (pytesseract). +Danach ein heuristischer Parser, der Zeilen mit Datum + Betrag findet. Das ist +bewusst Best-Effort – das Ergebnis landet in derselben Prüfansicht wie der +CSV-Import und wird dort korrigiert. +""" + +import io +import re +from datetime import datetime +from decimal import Decimal, InvalidOperation + +try: # PyMuPDF (Text + Rendern) + import fitz # type: ignore +except Exception: # noqa: BLE001 + fitz = None + +try: # OCR + import pytesseract # type: ignore + from PIL import Image # type: ignore +except Exception: # noqa: BLE001 + pytesseract = None + Image = None + + +class ExtractionError(RuntimeError): + pass + + +def _image_to_text(data: bytes) -> str: + if pytesseract is None or Image is None: + raise ExtractionError("OCR nicht verfügbar (pytesseract/Pillow/Tesseract fehlen).") + img = Image.open(io.BytesIO(data)) + return pytesseract.image_to_string(img, lang="deu+eng") + + +def _pdf_to_text(data: bytes) -> str: + if fitz is None: + raise ExtractionError("PDF-Verarbeitung nicht verfügbar (PyMuPDF fehlt).") + parts: list[str] = [] + with fitz.open(stream=data, filetype="pdf") as doc: + for page in doc: + txt = page.get_text() + if txt and txt.strip(): + parts.append(txt) + elif pytesseract is not None and Image is not None: + pix = page.get_pixmap(dpi=200) + img = Image.open(io.BytesIO(pix.tobytes("png"))) + parts.append(pytesseract.image_to_string(img, lang="deu+eng")) + return "\n".join(parts) + + +def extract_text(data: bytes, filename: str, content_type: str) -> str: + name = (filename or "").lower() + if name.endswith(".pdf") or content_type == "application/pdf": + return _pdf_to_text(data) + return _image_to_text(data) + + +# --- Heuristischer Zeilen-Parser ------------------------------------------- + +DATE_RE = re.compile(r"\b(\d{4}-\d{2}-\d{2}|\d{1,2}[./]\d{1,2}[./]\d{2,4})\b") +# Betrag: entweder mit Tausender-Gruppen (1'234.56 / 1.234,56) oder schlicht (3000.00). +AMOUNT_RE = re.compile(r"[-+]?(?:\d{1,3}(?:['.,\s]\d{3})+|\d+)[.,]\d{2}(?!\d)") +_DATE_FORMATS = ["%Y-%m-%d", "%d.%m.%Y", "%d.%m.%y", "%d/%m/%Y", "%d/%m/%y"] + + +def _norm_date(token: str): + for fmt in _DATE_FORMATS: + try: + return datetime.strptime(token, fmt).date() + except ValueError: + continue + return None + + +def _parse_amount_auto(s: str) -> int | None: + """Betrag mit automatisch erkanntem Dezimaltrenner (CH/DE/US) -> Cent.""" + s = s.strip() + neg = s.startswith("-") or (s.startswith("(") and s.endswith(")")) + m = re.search(r"[.,](\d{2})$", s) + if not m: + return None + int_digits = re.sub(r"\D", "", s[: m.start()]) + frac = m.group(1) + try: + val = Decimal(f"{int_digits or '0'}.{frac}") + except InvalidOperation: + return None + cents = int(val * 100) + return -cents if neg else cents + + +def parse_transactions(text: str) -> list[dict]: + """Findet Zeilen mit Datum + Betrag und macht daraus Kandidaten-Buchungen.""" + rows: list[dict] = [] + for line in text.splitlines(): + line = line.strip() + if len(line) < 6: + continue + dm = DATE_RE.search(line) + if not dm: + continue + d = _norm_date(dm.group(0)) + if d is None: + continue + rest = f"{line[: dm.start()]} {line[dm.end():]}" + amounts = AMOUNT_RE.findall(rest) + if not amounts: + continue + amount = _parse_amount_auto(amounts[0]) + if amount is None or amount == 0: + continue + desc = rest + for a in amounts: + desc = desc.replace(a, " ") + desc = re.sub(r"\s+", " ", desc).strip(" -–|;,.") + rows.append({ + "date": d.isoformat(), + "amount_cents": amount, + "payee": desc[:120] or None, + "purpose": None, + "counterparty_iban": None, + }) + return rows diff --git a/backend/requirements.txt b/backend/requirements.txt index 45e01f7..f707896 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,4 +8,9 @@ argon2-cffi==23.1.0 python-multipart==0.0.20 httpx==0.28.1 python-dateutil==2.9.0.post0 +# PDF/Bild-Import (Phase 5): Text aus PDFs (PyMuPDF) und OCR (Tesseract) für Scans/Bilder. +PyMuPDF==1.25.1 +pytesseract==0.3.13 +pillow==11.1.0 pytest==8.3.4 + diff --git a/backend/tests/test_documents.py b/backend/tests/test_documents.py new file mode 100644 index 0000000..ebead87 --- /dev/null +++ b/backend/tests/test_documents.py @@ -0,0 +1,55 @@ +import io + +import pytest + +fitz = pytest.importorskip("fitz") + + +def _account(admin): + r = admin.post("/api/accounts", json={"name": "Beleg", "type": "girokonto", "currency": "EUR"}) + assert r.status_code == 201 + return r.json() + + +def _make_pdf() -> bytes: + doc = fitz.open() + page = doc.new_page() + page.insert_text((50, 72), "15.03.2026 REWE Supermarkt Zuerich -50.00") + page.insert_text((50, 96), "25.03.2026 Gehalt Firma AG 3000.00") + page.insert_text((50, 120), "Kein Datum hier, nur Text 12.34") + return doc.tobytes() + + +def test_parse_transactions_heuristic(): + from app.services import ocr + + text = ( + "Kontoauszug März 2026\n" + "15.03.2026 REWE Supermarkt -50,00\n" + "2026-03-25 Gehalt 3'000.00\n" + "Zwischensumme\n" + ) + rows = ocr.parse_transactions(text) + assert len(rows) == 2 + assert rows[0]["date"] == "2026-03-15" + assert rows[0]["amount_cents"] == -5000 + assert "REWE" in rows[0]["payee"] + assert rows[1]["date"] == "2026-03-25" + assert rows[1]["amount_cents"] == 300000 # Schweizer Tausender-Apostroph + + +def test_pdf_document_preview(admin): + acc = _account(admin) + pdf = _make_pdf() + r = admin.post( + "/api/documents/preview", + files={"file": ("beleg.pdf", io.BytesIO(pdf), "application/pdf")}, + data={"account_id": str(acc["id"])}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["count"] == 2 + rows = body["rows"] + kinds = {(row["type"], row["amount_cents"]) for row in rows} + assert ("expense", -5000) in kinds + assert ("income", 300000) in kinds diff --git a/deploy/install.sh b/deploy/install.sh index d5a76ca..214e1f1 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -48,7 +48,8 @@ fi info "Installiere Systempakete (Python, Build-Tools, git, curl) …" export DEBIAN_FRONTEND=noninteractive apt-get update -y -apt-get install -y python3 python3-venv python3-dev build-essential curl ca-certificates git +apt-get install -y python3 python3-venv python3-dev build-essential curl ca-certificates git \ + tesseract-ocr tesseract-ocr-deu # --- Python-Version prüfen (3.11+ erforderlich) ------------------------------ PY_VER="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null || echo 0.0)" diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d5f465a..5ff33f8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -28,8 +28,9 @@ Bank-Templates (Trennzeichen/Dezimal/Datum/Spaltenmapping/Vorzeichen), lernende Auto-Kategorisierung, Prüfansicht, Duplikat-Erkennung, interne-Transfer-Erkennung via IBAN, Batch-Undo. Vorgefertigte Bank-Templates folgen aus Nutzer-Beispielen. -## Phase 5 – PDF/Bild-Import (Stretch) -Lokal (pdfplumber/pypdf + Tesseract-OCR), speist dieselbe Prüfansicht. +## Phase 5 – PDF/Bild-Import (Stretch) ✅ +Lokal: PyMuPDF (PDF-Text) + Tesseract-OCR (Scans/Bilder), heuristischer +Zeilen-Parser (Datum + Betrag), speist dieselbe Prüfansicht. Best-Effort. ## Querschnitt – Backup/Export DB-Backup-Download + Export je Entität (CSV/JSON); dokumentierte LXC-Sicherung. diff --git a/web/src/api.js b/web/src/api.js index 96736cd..7bd175b 100644 --- a/web/src/api.js +++ b/web/src/api.js @@ -198,4 +198,7 @@ export const api = { importCommit: (body) => request("/imports/commit", { method: "POST", body }), importBatches: () => request("/imports/batches"), undoImportBatch: (id) => request(`/imports/batches/${id}`, { method: "DELETE" }), + + // PDF/Bild-Beleg + documentPreview: (formData) => request("/documents/preview", { method: "POST", formData }), }; diff --git a/web/src/finance.css b/web/src/finance.css index 6951189..ff74fcd 100644 --- a/web/src/finance.css +++ b/web/src/finance.css @@ -560,3 +560,14 @@ tr.row-invalid { tr.row-dup { opacity: 0.55; } +.doc-text { + max-height: 240px; + overflow: auto; + font-size: 0.78rem; + white-space: pre-wrap; + color: var(--muted); + background: var(--surface-2); + padding: var(--sp-3); + border-radius: var(--radius-sm); + margin-top: var(--sp-2); +} diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 5b395d4..c3d68ec 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -197,13 +197,18 @@ "targetAccount": "Zielkonto" }, "import": { - "title": "CSV-Import", + "title": "Import", + "modeCsv": "CSV", + "modeDoc": "PDF · Bild", + "readDoc": "Beleg auslesen", + "docHint": "PDF-/Bild-Erkennung ist Best-Effort — bitte jede Zeile prüfen und ggf. korrigieren.", + "extractedText": "Extrahierter Text (zur Kontrolle)", "step1": "1 · Konto & Datei", "step2": "2 · Format", "step3": "3 · Spalten zuordnen", "step4": "4 · Prüfen & buchen", "account": "Zielkonto", - "file": "CSV-Datei", + "file": "Datei", "preset": "Bank-Vorlage (vorgefertigt)", "template": "Eigene Vorlage", "templateManual": "Manuell einstellen", diff --git a/web/src/locales/en.json b/web/src/locales/en.json index 27620be..cee946f 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -197,13 +197,18 @@ "targetAccount": "Target account" }, "import": { - "title": "CSV import", + "title": "Import", + "modeCsv": "CSV", + "modeDoc": "PDF · Image", + "readDoc": "Read document", + "docHint": "PDF/image recognition is best-effort — please check and fix every row.", + "extractedText": "Extracted text (for reference)", "step1": "1 · Account & file", "step2": "2 · Format", "step3": "3 · Map columns", "step4": "4 · Review & import", "account": "Target account", - "file": "CSV file", + "file": "File", "preset": "Bank preset", "template": "My template", "templateManual": "Configure manually", diff --git a/web/src/pages/Import.jsx b/web/src/pages/Import.jsx index 4a8ebb5..e480a8c 100644 --- a/web/src/pages/Import.jsx +++ b/web/src/pages/Import.jsx @@ -41,11 +41,13 @@ export default function Import() { const [presets, setPresets] = useState([]); const [batches, setBatches] = useState([]); + const [mode, setMode] = useState("csv"); // "csv" | "doc" const [accountId, setAccountId] = useState(""); const [file, setFile] = useState(null); const [config, setConfig] = useState(defaultConfig()); const [raw, setRaw] = useState(null); const [rows, setRows] = useState(null); + const [docText, setDocText] = useState(""); const [busy, setBusy] = useState(false); const [saveName, setSaveName] = useState(""); @@ -117,6 +119,19 @@ export default function Import() { } catch (e) { toast(e.message, "error"); } finally { setBusy(false); } } + async function loadDoc() { + if (!file || !accountId) { toast(t("import.needFileAccount"), "error"); return; } + setBusy(true); + try { + const fd = new FormData(); + fd.append("file", file); + fd.append("account_id", accountId); + const res = await api.documentPreview(fd); + setRows(res.rows.map((r) => ({ ...r }))); + setDocText(res.text || ""); + } catch (e) { toast(e.message, "error"); } finally { setBusy(false); } + } + async function commit() { setBusy(true); try { @@ -168,7 +183,15 @@ export default function Import() { return (
-

{t("import.title")}

+
+

{t("import.title")}

+
+ + +
+
{/* Schritt 1: Konto + Datei */}
@@ -182,11 +205,20 @@ export default function Import() {
-
+ {mode === "doc" && ( + <> + +

{t("import.docHint")}

+ + )} +
{presets.length > 0 && (
- {/* Schritt 2: Format */} + {/* Schritt 2: Format (nur CSV) */} + {mode === "csv" && (

{t("import.step2")}

@@ -266,6 +299,17 @@ export default function Import() { {t("import.readFile")}
+ )} + + {/* Extrahierter Text (PDF/Bild) */} + {mode === "doc" && docText && ( +
+
+ {t("import.extractedText")} + 
{docText}
+
+
+ )} {/* Schritt 3: Spaltenzuordnung */} {raw && (