Files
certain/backend/app/routers/imports.py
T
GuidoandClaude Opus 5 e6801b0474 feat: CSV-Format automatisch erkennen
Bisher musste man bei jedem Import Trennzeichen, Datumsformat und
Spaltenzuordnung von Hand eintragen – die haeufigste Fehlerquelle ueberhaupt:
Ein falsches Datumsformat liess alle Zeilen durchfallen, ein falscher
Dezimaltrenner verzehnfachte Betraege.

Zuerst wird gegen die Bank-Vorlagen geprueft (Kopfzeilen-Signatur); alle vier
echten Ausaege des Nutzers werden so sicher erkannt. Sonst inhaltsbasiert:
welche Spalte enthaelt Daten, welche Betraege, welche den laengsten Text.

Zwei Fallen, die dabei aufgefallen sind und getestet werden:
- Deutsche Datumsangaben stecken voller Punkte, und viele Auszuege fuehren
  Buchungs- UND Valutadatum. Ohne sie herauszurechnen kaeme "." als
  Dezimalzeichen heraus und aus 1.204,70 wuerden 1.20.
- parse_date ist absichtlich nachsichtig; beim Erkennen darf es das nicht
  sein, sonst gewinnt ein Format mit Uhrzeit gegen das richtige ohne.

Ganzzahlen ohne Vorzeichen zaehlen nur halb, damit Referenznummern und Kurse
nicht als Betragsspalte durchgehen.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-22 21:17:37 +02:00

231 lines
8.7 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CSV-Import: Templates verwalten, Datei-Vorschau, verbuchen, rückgängig machen."""
import json
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
from sqlalchemy.orm import Session
from ..database import get_db
from ..deps import get_current_user, get_owned
from fastapi.responses import Response
from ..models import PendingUpload, Account, AmountMode, ImportBatch, ImportStatus, ImportTemplate, User
from ..schemas import (
ImportBatchOut,
ImportCommitIn,
ImportTemplateCreate,
ImportTemplateOut,
ImportTemplateUpdate,
)
from ..services import csv_import
router = APIRouter(prefix="/imports", tags=["imports"])
def _to_out(tpl: ImportTemplate) -> ImportTemplateOut:
return ImportTemplateOut(
id=tpl.id, name=tpl.name, delimiter=tpl.delimiter, decimal_sep=tpl.decimal_sep,
date_format=tpl.date_format, encoding=tpl.encoding, has_header=tpl.has_header,
skip_rows=tpl.skip_rows, amount_mode=tpl.amount_mode,
column_map=json.loads(tpl.column_map or "{}"),
filter_column=tpl.filter_column, filter_value=tpl.filter_value,
)
# --- Templates -------------------------------------------------------------
@router.get("/presets")
def list_presets(_: User = Depends(get_current_user)) -> list[dict]:
"""Vorgefertigte Bank-Vorlagen (füllen die Format-/Spalten-Einstellungen)."""
return csv_import.PRESETS
@router.get("/templates", response_model=list[ImportTemplateOut])
def list_templates(db: Session = Depends(get_db), user: User = Depends(get_current_user)):
rows = db.query(ImportTemplate).filter(ImportTemplate.owner_id == user.id).order_by(ImportTemplate.name).all()
return [_to_out(t) for t in rows]
@router.post("/templates", response_model=ImportTemplateOut, status_code=status.HTTP_201_CREATED)
def create_template(payload: ImportTemplateCreate, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
tpl = ImportTemplate(
owner_id=user.id, name=payload.name.strip(), delimiter=payload.delimiter,
decimal_sep=payload.decimal_sep, date_format=payload.date_format, encoding=payload.encoding,
has_header=payload.has_header, skip_rows=payload.skip_rows, amount_mode=payload.amount_mode,
column_map=json.dumps(payload.column_map),
filter_column=payload.filter_column, filter_value=payload.filter_value,
)
db.add(tpl)
db.commit()
db.refresh(tpl)
return _to_out(tpl)
@router.patch("/templates/{template_id}", response_model=ImportTemplateOut)
def update_template(template_id: int, payload: ImportTemplateUpdate, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
tpl = get_owned(db, ImportTemplate, template_id, user)
data = payload.model_dump(exclude_unset=True)
if "column_map" in data and data["column_map"] is not None:
tpl.column_map = json.dumps(data.pop("column_map"))
for key, value in data.items():
if value is not None:
setattr(tpl, key, value)
db.commit()
db.refresh(tpl)
return _to_out(tpl)
@router.delete("/templates/{template_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_template(template_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
tpl = get_owned(db, ImportTemplate, template_id, user)
db.delete(tpl)
db.commit()
# --- Datei-Vorschau --------------------------------------------------------
def _transient(user_id: int, **fields) -> ImportTemplate:
return ImportTemplate(owner_id=user_id, name="_transient", **fields)
@router.post("/detect")
async def detect_format(
file: UploadFile = File(...),
encoding: str = Form("utf-8"),
_: User = Depends(get_current_user),
) -> dict:
"""Format der Datei erkennen, damit niemand mehr Spalten von Hand zuordnet."""
daten = await file.read()
return csv_import.erkenne_format(daten, encoding)
@router.post("/raw")
async def raw_preview(
file: UploadFile = File(...),
delimiter: str = Form(";"),
encoding: str = Form("utf-8"),
has_header: bool = Form(True),
skip_rows: int = Form(0),
_: User = Depends(get_current_user),
) -> dict:
data = await file.read()
tpl = _transient(0, delimiter=delimiter, encoding=encoding, has_header=has_header, skip_rows=skip_rows)
return csv_import.preview_raw(data, tpl)
@router.post("/preview")
async def preview(
file: UploadFile = File(...),
account_id: int = Form(...),
delimiter: str = Form(";"),
decimal_sep: str = Form(","),
date_format: str = Form("%d.%m.%Y"),
encoding: str = Form("utf-8"),
has_header: bool = Form(True),
skip_rows: int = Form(0),
amount_mode: str = Form("signed"),
column_map: str = Form("{}"),
filter_column: int | None = Form(None),
filter_value: str | None = Form(None),
db: Session = Depends(get_db),
user: User = Depends(get_current_user),
) -> dict:
account = get_owned(db, Account, account_id, user)
data = await file.read()
tpl = _transient(
user.id, delimiter=delimiter, decimal_sep=decimal_sep, date_format=date_format,
encoding=encoding, has_header=has_header, skip_rows=skip_rows,
amount_mode=AmountMode(amount_mode), column_map=column_map,
filter_column=filter_column, filter_value=filter_value,
)
rows = csv_import.build_preview(db, user, account, tpl, data)
return {
"rows": rows,
"filename": file.filename or "",
# Abgleich gegen die Saldo-Spalte der Datei (falls zugeordnet).
"reconciliation": csv_import.reconcile(rows),
}
# --- Verbuchen / Undo ------------------------------------------------------
@router.post("/commit", response_model=ImportBatchOut)
def commit(payload: ImportCommitIn, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
account = get_owned(db, Account, payload.account_id, user)
return csv_import.commit_import(db, user, account, payload.template_id, payload.filename,
payload.rows, preset=payload.preset)
@router.get("/batches", response_model=list[ImportBatchOut])
def list_batches(db: Session = Depends(get_db), user: User = Depends(get_current_user)):
return (
db.query(ImportBatch)
.filter(ImportBatch.owner_id == user.id)
.order_by(ImportBatch.created_at.desc(), ImportBatch.id.desc())
.all()
)
@router.delete("/batches/{batch_id}")
def undo(batch_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)) -> dict:
batch = get_owned(db, ImportBatch, batch_id, user)
deleted = csv_import.undo_batch(db, user, batch)
return {"deleted": deleted}
@router.delete("/batches/{batch_id}/record", status_code=status.HTTP_204_NO_CONTENT)
def remove_batch_record(
batch_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)
):
"""Entfernt einen rückgängig gemachten Eintrag aus dem Verlauf.
Nur für bereits zurückgenommene Importe: Solange ein Import steht, ist der
Eintrag die einzige Möglichkeit, seine Buchungen wieder loszuwerden."""
batch = get_owned(db, ImportBatch, batch_id, user)
if batch.status != ImportStatus.undone:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Nur rückgängig gemachte Importe lassen sich aus dem Verlauf entfernen.",
)
db.delete(batch)
db.commit()
# --- Vom Telefon abgelieferte Dateien ---------------------------------------
@router.get("/uploads")
def list_uploads(db: Session = Depends(get_db), user: User = Depends(get_current_user)) -> list[dict]:
"""Wartende Dateien. Der Inhalt bleibt hier draussen – nur die Eckdaten."""
zeilen = (
db.query(PendingUpload)
.filter(PendingUpload.owner_id == user.id)
.order_by(PendingUpload.created_at.desc(), PendingUpload.id.desc())
.all()
)
return [
{"id": u.id, "filename": u.filename, "account_id": u.account_id,
"size_bytes": u.size_bytes, "created_at": u.created_at.isoformat() if u.created_at else None}
for u in zeilen
]
@router.get("/uploads/{upload_id}/content")
def upload_content(upload_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
"""Die Datei selbst – damit die Import-Ansicht sie wie eine lokale behandelt."""
upload = get_owned(db, PendingUpload, upload_id, user)
return Response(
content=upload.content,
media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="{upload.filename}"'},
)
@router.delete("/uploads/{upload_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_upload(upload_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
db.delete(get_owned(db, PendingUpload, upload_id, user))
db.commit()