Bisher musste man bei jedem Import Trennzeichen, Datumsformat und Spaltenzuordnung von Hand eintragen – die haeufigste Fehlerquelle ueberhaupt: Ein falsches Datumsformat liess alle Zeilen durchfallen, ein falscher Dezimaltrenner verzehnfachte Betraege. Zuerst wird gegen die Bank-Vorlagen geprueft (Kopfzeilen-Signatur); alle vier echten Ausaege des Nutzers werden so sicher erkannt. Sonst inhaltsbasiert: welche Spalte enthaelt Daten, welche Betraege, welche den laengsten Text. Zwei Fallen, die dabei aufgefallen sind und getestet werden: - Deutsche Datumsangaben stecken voller Punkte, und viele Auszuege fuehren Buchungs- UND Valutadatum. Ohne sie herauszurechnen kaeme "." als Dezimalzeichen heraus und aus 1.204,70 wuerden 1.20. - parse_date ist absichtlich nachsichtig; beim Erkennen darf es das nicht sein, sonst gewinnt ein Format mit Uhrzeit gegen das richtige ohne. Ganzzahlen ohne Vorzeichen zaehlen nur halb, damit Referenznummern und Kurse nicht als Betragsspalte durchgehen. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
231 lines
8.7 KiB
Python
231 lines
8.7 KiB
Python
"""CSV-Import: Templates verwalten, Datei-Vorschau, verbuchen, rückgängig machen."""
|
||
|
||
import json
|
||
|
||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
|
||
from sqlalchemy.orm import Session
|
||
|
||
from ..database import get_db
|
||
from ..deps import get_current_user, get_owned
|
||
from fastapi.responses import Response
|
||
|
||
from ..models import PendingUpload, Account, AmountMode, ImportBatch, ImportStatus, ImportTemplate, User
|
||
from ..schemas import (
|
||
ImportBatchOut,
|
||
ImportCommitIn,
|
||
ImportTemplateCreate,
|
||
ImportTemplateOut,
|
||
ImportTemplateUpdate,
|
||
)
|
||
from ..services import csv_import
|
||
|
||
router = APIRouter(prefix="/imports", tags=["imports"])
|
||
|
||
|
||
def _to_out(tpl: ImportTemplate) -> ImportTemplateOut:
|
||
return ImportTemplateOut(
|
||
id=tpl.id, name=tpl.name, delimiter=tpl.delimiter, decimal_sep=tpl.decimal_sep,
|
||
date_format=tpl.date_format, encoding=tpl.encoding, has_header=tpl.has_header,
|
||
skip_rows=tpl.skip_rows, amount_mode=tpl.amount_mode,
|
||
column_map=json.loads(tpl.column_map or "{}"),
|
||
filter_column=tpl.filter_column, filter_value=tpl.filter_value,
|
||
)
|
||
|
||
|
||
# --- Templates -------------------------------------------------------------
|
||
|
||
|
||
@router.get("/presets")
|
||
def list_presets(_: User = Depends(get_current_user)) -> list[dict]:
|
||
"""Vorgefertigte Bank-Vorlagen (füllen die Format-/Spalten-Einstellungen)."""
|
||
return csv_import.PRESETS
|
||
|
||
|
||
@router.get("/templates", response_model=list[ImportTemplateOut])
|
||
def list_templates(db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
rows = db.query(ImportTemplate).filter(ImportTemplate.owner_id == user.id).order_by(ImportTemplate.name).all()
|
||
return [_to_out(t) for t in rows]
|
||
|
||
|
||
@router.post("/templates", response_model=ImportTemplateOut, status_code=status.HTTP_201_CREATED)
|
||
def create_template(payload: ImportTemplateCreate, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
tpl = ImportTemplate(
|
||
owner_id=user.id, name=payload.name.strip(), delimiter=payload.delimiter,
|
||
decimal_sep=payload.decimal_sep, date_format=payload.date_format, encoding=payload.encoding,
|
||
has_header=payload.has_header, skip_rows=payload.skip_rows, amount_mode=payload.amount_mode,
|
||
column_map=json.dumps(payload.column_map),
|
||
filter_column=payload.filter_column, filter_value=payload.filter_value,
|
||
)
|
||
db.add(tpl)
|
||
db.commit()
|
||
db.refresh(tpl)
|
||
return _to_out(tpl)
|
||
|
||
|
||
@router.patch("/templates/{template_id}", response_model=ImportTemplateOut)
|
||
def update_template(template_id: int, payload: ImportTemplateUpdate, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
tpl = get_owned(db, ImportTemplate, template_id, user)
|
||
data = payload.model_dump(exclude_unset=True)
|
||
if "column_map" in data and data["column_map"] is not None:
|
||
tpl.column_map = json.dumps(data.pop("column_map"))
|
||
for key, value in data.items():
|
||
if value is not None:
|
||
setattr(tpl, key, value)
|
||
db.commit()
|
||
db.refresh(tpl)
|
||
return _to_out(tpl)
|
||
|
||
|
||
@router.delete("/templates/{template_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||
def delete_template(template_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
tpl = get_owned(db, ImportTemplate, template_id, user)
|
||
db.delete(tpl)
|
||
db.commit()
|
||
|
||
|
||
# --- Datei-Vorschau --------------------------------------------------------
|
||
|
||
|
||
def _transient(user_id: int, **fields) -> ImportTemplate:
|
||
return ImportTemplate(owner_id=user_id, name="_transient", **fields)
|
||
|
||
|
||
@router.post("/detect")
|
||
async def detect_format(
|
||
file: UploadFile = File(...),
|
||
encoding: str = Form("utf-8"),
|
||
_: User = Depends(get_current_user),
|
||
) -> dict:
|
||
"""Format der Datei erkennen, damit niemand mehr Spalten von Hand zuordnet."""
|
||
daten = await file.read()
|
||
return csv_import.erkenne_format(daten, encoding)
|
||
|
||
|
||
@router.post("/raw")
|
||
async def raw_preview(
|
||
file: UploadFile = File(...),
|
||
delimiter: str = Form(";"),
|
||
encoding: str = Form("utf-8"),
|
||
has_header: bool = Form(True),
|
||
skip_rows: int = Form(0),
|
||
_: User = Depends(get_current_user),
|
||
) -> dict:
|
||
data = await file.read()
|
||
tpl = _transient(0, delimiter=delimiter, encoding=encoding, has_header=has_header, skip_rows=skip_rows)
|
||
return csv_import.preview_raw(data, tpl)
|
||
|
||
|
||
@router.post("/preview")
|
||
async def preview(
|
||
file: UploadFile = File(...),
|
||
account_id: int = Form(...),
|
||
delimiter: str = Form(";"),
|
||
decimal_sep: str = Form(","),
|
||
date_format: str = Form("%d.%m.%Y"),
|
||
encoding: str = Form("utf-8"),
|
||
has_header: bool = Form(True),
|
||
skip_rows: int = Form(0),
|
||
amount_mode: str = Form("signed"),
|
||
column_map: str = Form("{}"),
|
||
filter_column: int | None = Form(None),
|
||
filter_value: str | None = Form(None),
|
||
db: Session = Depends(get_db),
|
||
user: User = Depends(get_current_user),
|
||
) -> dict:
|
||
account = get_owned(db, Account, account_id, user)
|
||
data = await file.read()
|
||
tpl = _transient(
|
||
user.id, delimiter=delimiter, decimal_sep=decimal_sep, date_format=date_format,
|
||
encoding=encoding, has_header=has_header, skip_rows=skip_rows,
|
||
amount_mode=AmountMode(amount_mode), column_map=column_map,
|
||
filter_column=filter_column, filter_value=filter_value,
|
||
)
|
||
rows = csv_import.build_preview(db, user, account, tpl, data)
|
||
return {
|
||
"rows": rows,
|
||
"filename": file.filename or "",
|
||
# Abgleich gegen die Saldo-Spalte der Datei (falls zugeordnet).
|
||
"reconciliation": csv_import.reconcile(rows),
|
||
}
|
||
|
||
|
||
# --- Verbuchen / Undo ------------------------------------------------------
|
||
|
||
|
||
@router.post("/commit", response_model=ImportBatchOut)
|
||
def commit(payload: ImportCommitIn, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
account = get_owned(db, Account, payload.account_id, user)
|
||
return csv_import.commit_import(db, user, account, payload.template_id, payload.filename,
|
||
payload.rows, preset=payload.preset)
|
||
|
||
|
||
@router.get("/batches", response_model=list[ImportBatchOut])
|
||
def list_batches(db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
return (
|
||
db.query(ImportBatch)
|
||
.filter(ImportBatch.owner_id == user.id)
|
||
.order_by(ImportBatch.created_at.desc(), ImportBatch.id.desc())
|
||
.all()
|
||
)
|
||
|
||
|
||
@router.delete("/batches/{batch_id}")
|
||
def undo(batch_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)) -> dict:
|
||
batch = get_owned(db, ImportBatch, batch_id, user)
|
||
deleted = csv_import.undo_batch(db, user, batch)
|
||
return {"deleted": deleted}
|
||
|
||
|
||
@router.delete("/batches/{batch_id}/record", status_code=status.HTTP_204_NO_CONTENT)
|
||
def remove_batch_record(
|
||
batch_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)
|
||
):
|
||
"""Entfernt einen rückgängig gemachten Eintrag aus dem Verlauf.
|
||
|
||
Nur für bereits zurückgenommene Importe: Solange ein Import steht, ist der
|
||
Eintrag die einzige Möglichkeit, seine Buchungen wieder loszuwerden."""
|
||
batch = get_owned(db, ImportBatch, batch_id, user)
|
||
if batch.status != ImportStatus.undone:
|
||
raise HTTPException(
|
||
status_code=status.HTTP_400_BAD_REQUEST,
|
||
detail="Nur rückgängig gemachte Importe lassen sich aus dem Verlauf entfernen.",
|
||
)
|
||
db.delete(batch)
|
||
db.commit()
|
||
|
||
|
||
# --- Vom Telefon abgelieferte Dateien ---------------------------------------
|
||
|
||
|
||
@router.get("/uploads")
|
||
def list_uploads(db: Session = Depends(get_db), user: User = Depends(get_current_user)) -> list[dict]:
|
||
"""Wartende Dateien. Der Inhalt bleibt hier draussen – nur die Eckdaten."""
|
||
zeilen = (
|
||
db.query(PendingUpload)
|
||
.filter(PendingUpload.owner_id == user.id)
|
||
.order_by(PendingUpload.created_at.desc(), PendingUpload.id.desc())
|
||
.all()
|
||
)
|
||
return [
|
||
{"id": u.id, "filename": u.filename, "account_id": u.account_id,
|
||
"size_bytes": u.size_bytes, "created_at": u.created_at.isoformat() if u.created_at else None}
|
||
for u in zeilen
|
||
]
|
||
|
||
|
||
@router.get("/uploads/{upload_id}/content")
|
||
def upload_content(upload_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
"""Die Datei selbst – damit die Import-Ansicht sie wie eine lokale behandelt."""
|
||
upload = get_owned(db, PendingUpload, upload_id, user)
|
||
return Response(
|
||
content=upload.content,
|
||
media_type="text/csv",
|
||
headers={"Content-Disposition": f'attachment; filename="{upload.filename}"'},
|
||
)
|
||
|
||
|
||
@router.delete("/uploads/{upload_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||
def delete_upload(upload_id: int, db: Session = Depends(get_db), user: User = Depends(get_current_user)):
|
||
db.delete(get_owned(db, PendingUpload, upload_id, user))
|
||
db.commit()
|