feat: Phase 5 – PDF/Bild-Beleg-Import (lokal, Tesseract-OCR)
Backend: - services/ocr: PDF-Text via PyMuPDF, OCR (Tesseract) für Scans/Bilder, heuristischer Parser (Datum + Betrag je Zeile, CH/DE/US-Beträge). - routers/documents: /documents/preview (Upload -> Text -> Kandidaten -> gemeinsame Prüf-/Buchungslogik des CSV-Imports). csv_import refaktoriert: detect_rows() von CSV und Beleg gemeinsam genutzt. - requirements: PyMuPDF, pytesseract, pillow; install.sh installiert tesseract-ocr (+ deu). pytest: Heuristik-Parser und PDF-Vorschau. Frontend: - Import-Seite: CSV/PDF-Umschalter; PDF-Modus mit Beleg-Auslesen, extrahiertem Text zur Kontrolle und Best-Effort-Hinweis; gleiche Prüfansicht + Buchen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Guido
Claude Opus 4.8
@@ -24,6 +24,7 @@ from .routers import (
|
||||
budgets,
|
||||
categories,
|
||||
dashboard,
|
||||
documents,
|
||||
imports,
|
||||
rates,
|
||||
settings as settings_router,
|
||||
@@ -105,6 +106,7 @@ api.include_router(transactions.router)
|
||||
api.include_router(budgets.router)
|
||||
api.include_router(dashboard.router)
|
||||
api.include_router(imports.router)
|
||||
api.include_router(documents.router)
|
||||
api.include_router(rates.router)
|
||||
|
||||
app.include_router(api)
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""PDF-/Bild-Beleg-Import (Phase 5, lokal). Nutzt die Prüf-/Buchungslogik des CSV-Imports."""
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..deps import get_current_user, get_owned
|
||||
from ..models import Account, User
|
||||
from ..services import csv_import, ocr
|
||||
|
||||
router = APIRouter(prefix="/documents", tags=["documents"])
|
||||
|
||||
|
||||
@router.post("/preview")
|
||||
async def preview(
|
||||
file: UploadFile = File(...),
|
||||
account_id: int = Form(...),
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
) -> dict:
|
||||
account = get_owned(db, Account, account_id, user)
|
||||
data = await file.read()
|
||||
try:
|
||||
text = ocr.extract_text(data, file.filename or "", file.content_type or "")
|
||||
except ocr.ExtractionError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
candidates = ocr.parse_transactions(text)
|
||||
rows = csv_import.detect_rows(db, user, account, candidates)
|
||||
return {
|
||||
"rows": rows,
|
||||
"text": text[:8000],
|
||||
"count": len(rows),
|
||||
"filename": file.filename or "",
|
||||
}
|
||||
@@ -153,16 +153,45 @@ def _apply_rules(rules: list[CategorizationRule], payee: str, purpose: str) -> i
|
||||
return None
|
||||
|
||||
|
||||
def build_preview(db: Session, user: User, account: Account, template: ImportTemplate, data: bytes) -> list[dict]:
|
||||
def parse_file_to_candidates(data: bytes, template: ImportTemplate) -> list[dict]:
|
||||
"""Zerlegt die Datei anhand des Templates in rohe Kandidaten-Zeilen."""
|
||||
cmap = json.loads(template.column_map or "{}")
|
||||
|
||||
def col(field: str) -> int | None:
|
||||
idx = cmap.get(field)
|
||||
return idx if isinstance(idx, int) and idx >= 0 else None
|
||||
|
||||
rows = read_rows(data, template)
|
||||
out: list[dict] = []
|
||||
for r in read_rows(data, template):
|
||||
def cell(field: str) -> str:
|
||||
c = col(field)
|
||||
return r[c].strip() if c is not None and c < len(r) else ""
|
||||
|
||||
# Identifikatoren aller Konten des Nutzers -> Konto-ID (für Transfer-Erkennung).
|
||||
d = parse_date(cell("date"), template.date_format)
|
||||
if template.amount_mode == AmountMode.debit_credit:
|
||||
debit = parse_amount(cell("debit"), template.decimal_sep)
|
||||
credit = parse_amount(cell("credit"), template.decimal_sep)
|
||||
amount = (credit or 0) - (abs(debit) if debit else 0)
|
||||
elif template.amount_mode == AmountMode.abs_expense:
|
||||
a = parse_amount(cell("amount"), template.decimal_sep)
|
||||
amount = -abs(a) if a is not None else None
|
||||
else:
|
||||
amount = parse_amount(cell("amount"), template.decimal_sep)
|
||||
|
||||
out.append({
|
||||
"date": d.isoformat() if d else None,
|
||||
"amount_cents": amount,
|
||||
"payee": cell("payee") or None,
|
||||
"purpose": cell("purpose") or None,
|
||||
"counterparty_iban": normalize_iban(cell("counterparty_iban")) or None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detect_rows(db: Session, user: User, account: Account, candidates: list[dict]) -> list[dict]:
|
||||
"""Reichert Kandidaten an: Typ, Auto-Kategorie, interner Transfer, Duplikat.
|
||||
|
||||
Wird von CSV- UND Beleg-Import genutzt (gleiche Prüf-/Buchungslogik)."""
|
||||
idents: dict[str, int] = {}
|
||||
for value_norm, acc_id in (
|
||||
db.query(AccountIdentifier.value_norm, AccountIdentifier.account_id)
|
||||
@@ -177,39 +206,25 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem
|
||||
.order_by(CategorizationRule.priority.desc(), CategorizationRule.hits.desc())
|
||||
.all()
|
||||
)
|
||||
|
||||
existing = {
|
||||
h[0]
|
||||
for h in db.query(Transaction.import_hash).filter(
|
||||
Transaction.owner_id == user.id, Transaction.import_hash.isnot(None)
|
||||
)
|
||||
}
|
||||
seen_in_file: set[str] = set()
|
||||
seen: set[str] = set()
|
||||
|
||||
out: list[dict] = []
|
||||
for i, r in enumerate(rows):
|
||||
def cell(field: str) -> str:
|
||||
c = col(field)
|
||||
return r[c].strip() if c is not None and c < len(r) else ""
|
||||
|
||||
d = parse_date(cell("date"), template.date_format)
|
||||
|
||||
if template.amount_mode == AmountMode.debit_credit:
|
||||
debit = parse_amount(cell("debit"), template.decimal_sep)
|
||||
credit = parse_amount(cell("credit"), template.decimal_sep)
|
||||
amount = (credit or 0) - (abs(debit) if debit else 0)
|
||||
elif template.amount_mode == AmountMode.abs_expense:
|
||||
a = parse_amount(cell("amount"), template.decimal_sep)
|
||||
amount = -abs(a) if a is not None else None
|
||||
else:
|
||||
amount = parse_amount(cell("amount"), template.decimal_sep)
|
||||
|
||||
payee = cell("payee")
|
||||
purpose = cell("purpose")
|
||||
iban = normalize_iban(cell("counterparty_iban"))
|
||||
for i, cand in enumerate(candidates):
|
||||
d_iso = cand.get("date")
|
||||
amount = cand.get("amount_cents")
|
||||
payee = cand.get("payee")
|
||||
purpose = cand.get("purpose")
|
||||
iban = cand.get("counterparty_iban")
|
||||
d = date.fromisoformat(d_iso) if d_iso else None
|
||||
|
||||
valid = d is not None and amount is not None and amount != 0
|
||||
ttype = "transfer" if False else ("expense" if (amount or 0) < 0 else "income")
|
||||
ttype = "expense" if (amount or 0) < 0 else "income"
|
||||
category_id: int | None = None
|
||||
to_account_id: int | None = None
|
||||
|
||||
@@ -218,11 +233,11 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem
|
||||
ttype = "transfer"
|
||||
to_account_id = other
|
||||
else:
|
||||
category_id = _apply_rules(rules, payee, purpose)
|
||||
category_id = _apply_rules(rules, payee or "", purpose or "")
|
||||
|
||||
h = import_hash(account.id, d, amount, payee)
|
||||
is_dup = h in existing or h in seen_in_file
|
||||
seen_in_file.add(h)
|
||||
is_dup = h in existing or h in seen
|
||||
seen.add(h)
|
||||
|
||||
out.append({
|
||||
"index": i,
|
||||
@@ -242,6 +257,10 @@ def build_preview(db: Session, user: User, account: Account, template: ImportTem
|
||||
return out
|
||||
|
||||
|
||||
def build_preview(db: Session, user: User, account: Account, template: ImportTemplate, data: bytes) -> list[dict]:
|
||||
return detect_rows(db, user, account, parse_file_to_candidates(data, template))
|
||||
|
||||
|
||||
# --- Verbuchen -------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""PDF-/Bild-Belege lokal auslesen (kein Cloud-Dienst).
|
||||
|
||||
- Digitale PDFs: Text direkt via PyMuPDF.
|
||||
- Gescannte PDFs / Bilder: OCR via Tesseract (pytesseract).
|
||||
Danach ein heuristischer Parser, der Zeilen mit Datum + Betrag findet. Das ist
|
||||
bewusst Best-Effort – das Ergebnis landet in derselben Prüfansicht wie der
|
||||
CSV-Import und wird dort korrigiert.
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from datetime import datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
try: # PyMuPDF (Text + Rendern)
|
||||
import fitz # type: ignore
|
||||
except Exception: # noqa: BLE001
|
||||
fitz = None
|
||||
|
||||
try: # OCR
|
||||
import pytesseract # type: ignore
|
||||
from PIL import Image # type: ignore
|
||||
except Exception: # noqa: BLE001
|
||||
pytesseract = None
|
||||
Image = None
|
||||
|
||||
|
||||
class ExtractionError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _image_to_text(data: bytes) -> str:
|
||||
if pytesseract is None or Image is None:
|
||||
raise ExtractionError("OCR nicht verfügbar (pytesseract/Pillow/Tesseract fehlen).")
|
||||
img = Image.open(io.BytesIO(data))
|
||||
return pytesseract.image_to_string(img, lang="deu+eng")
|
||||
|
||||
|
||||
def _pdf_to_text(data: bytes) -> str:
|
||||
if fitz is None:
|
||||
raise ExtractionError("PDF-Verarbeitung nicht verfügbar (PyMuPDF fehlt).")
|
||||
parts: list[str] = []
|
||||
with fitz.open(stream=data, filetype="pdf") as doc:
|
||||
for page in doc:
|
||||
txt = page.get_text()
|
||||
if txt and txt.strip():
|
||||
parts.append(txt)
|
||||
elif pytesseract is not None and Image is not None:
|
||||
pix = page.get_pixmap(dpi=200)
|
||||
img = Image.open(io.BytesIO(pix.tobytes("png")))
|
||||
parts.append(pytesseract.image_to_string(img, lang="deu+eng"))
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def extract_text(data: bytes, filename: str, content_type: str) -> str:
|
||||
name = (filename or "").lower()
|
||||
if name.endswith(".pdf") or content_type == "application/pdf":
|
||||
return _pdf_to_text(data)
|
||||
return _image_to_text(data)
|
||||
|
||||
|
||||
# --- Heuristischer Zeilen-Parser -------------------------------------------
|
||||
|
||||
DATE_RE = re.compile(r"\b(\d{4}-\d{2}-\d{2}|\d{1,2}[./]\d{1,2}[./]\d{2,4})\b")
|
||||
# Betrag: entweder mit Tausender-Gruppen (1'234.56 / 1.234,56) oder schlicht (3000.00).
|
||||
AMOUNT_RE = re.compile(r"[-+]?(?:\d{1,3}(?:['.,\s]\d{3})+|\d+)[.,]\d{2}(?!\d)")
|
||||
_DATE_FORMATS = ["%Y-%m-%d", "%d.%m.%Y", "%d.%m.%y", "%d/%m/%Y", "%d/%m/%y"]
|
||||
|
||||
|
||||
def _norm_date(token: str):
|
||||
for fmt in _DATE_FORMATS:
|
||||
try:
|
||||
return datetime.strptime(token, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _parse_amount_auto(s: str) -> int | None:
|
||||
"""Betrag mit automatisch erkanntem Dezimaltrenner (CH/DE/US) -> Cent."""
|
||||
s = s.strip()
|
||||
neg = s.startswith("-") or (s.startswith("(") and s.endswith(")"))
|
||||
m = re.search(r"[.,](\d{2})$", s)
|
||||
if not m:
|
||||
return None
|
||||
int_digits = re.sub(r"\D", "", s[: m.start()])
|
||||
frac = m.group(1)
|
||||
try:
|
||||
val = Decimal(f"{int_digits or '0'}.{frac}")
|
||||
except InvalidOperation:
|
||||
return None
|
||||
cents = int(val * 100)
|
||||
return -cents if neg else cents
|
||||
|
||||
|
||||
def parse_transactions(text: str) -> list[dict]:
|
||||
"""Findet Zeilen mit Datum + Betrag und macht daraus Kandidaten-Buchungen."""
|
||||
rows: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if len(line) < 6:
|
||||
continue
|
||||
dm = DATE_RE.search(line)
|
||||
if not dm:
|
||||
continue
|
||||
d = _norm_date(dm.group(0))
|
||||
if d is None:
|
||||
continue
|
||||
rest = f"{line[: dm.start()]} {line[dm.end():]}"
|
||||
amounts = AMOUNT_RE.findall(rest)
|
||||
if not amounts:
|
||||
continue
|
||||
amount = _parse_amount_auto(amounts[0])
|
||||
if amount is None or amount == 0:
|
||||
continue
|
||||
desc = rest
|
||||
for a in amounts:
|
||||
desc = desc.replace(a, " ")
|
||||
desc = re.sub(r"\s+", " ", desc).strip(" -–|;,.")
|
||||
rows.append({
|
||||
"date": d.isoformat(),
|
||||
"amount_cents": amount,
|
||||
"payee": desc[:120] or None,
|
||||
"purpose": None,
|
||||
"counterparty_iban": None,
|
||||
})
|
||||
return rows
|
||||
@@ -8,4 +8,9 @@ argon2-cffi==23.1.0
|
||||
python-multipart==0.0.20
|
||||
httpx==0.28.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# PDF/Bild-Import (Phase 5): Text aus PDFs (PyMuPDF) und OCR (Tesseract) für Scans/Bilder.
|
||||
PyMuPDF==1.25.1
|
||||
pytesseract==0.3.13
|
||||
pillow==11.1.0
|
||||
pytest==8.3.4
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import io
|
||||
|
||||
import pytest
|
||||
|
||||
fitz = pytest.importorskip("fitz")
|
||||
|
||||
|
||||
def _account(admin):
|
||||
r = admin.post("/api/accounts", json={"name": "Beleg", "type": "girokonto", "currency": "EUR"})
|
||||
assert r.status_code == 201
|
||||
return r.json()
|
||||
|
||||
|
||||
def _make_pdf() -> bytes:
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
page.insert_text((50, 72), "15.03.2026 REWE Supermarkt Zuerich -50.00")
|
||||
page.insert_text((50, 96), "25.03.2026 Gehalt Firma AG 3000.00")
|
||||
page.insert_text((50, 120), "Kein Datum hier, nur Text 12.34")
|
||||
return doc.tobytes()
|
||||
|
||||
|
||||
def test_parse_transactions_heuristic():
|
||||
from app.services import ocr
|
||||
|
||||
text = (
|
||||
"Kontoauszug März 2026\n"
|
||||
"15.03.2026 REWE Supermarkt -50,00\n"
|
||||
"2026-03-25 Gehalt 3'000.00\n"
|
||||
"Zwischensumme\n"
|
||||
)
|
||||
rows = ocr.parse_transactions(text)
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["date"] == "2026-03-15"
|
||||
assert rows[0]["amount_cents"] == -5000
|
||||
assert "REWE" in rows[0]["payee"]
|
||||
assert rows[1]["date"] == "2026-03-25"
|
||||
assert rows[1]["amount_cents"] == 300000 # Schweizer Tausender-Apostroph
|
||||
|
||||
|
||||
def test_pdf_document_preview(admin):
|
||||
acc = _account(admin)
|
||||
pdf = _make_pdf()
|
||||
r = admin.post(
|
||||
"/api/documents/preview",
|
||||
files={"file": ("beleg.pdf", io.BytesIO(pdf), "application/pdf")},
|
||||
data={"account_id": str(acc["id"])},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["count"] == 2
|
||||
rows = body["rows"]
|
||||
kinds = {(row["type"], row["amount_cents"]) for row in rows}
|
||||
assert ("expense", -5000) in kinds
|
||||
assert ("income", 300000) in kinds
|
||||
Reference in New Issue
Block a user