WIP: Deutschland als zweite Datenquelle (gtfs.de)
- mehrere Quellen (ch, de) mit je eigenem Fahrplan und Echtzeit-Feed - OSM CH+DE, mit osmium auf Schienennetz gefiltert, für pfaedle - Landesgrenze CH gegen doppelte grenzüberschreitende Züge - Import robust gegen andere Spaltenreihenfolge (gtfs.de) - Kacheln für CH+DE (Zoom 14) Noch nicht vollständig getestet (pfaedle auf Deutschland läuft). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X48pE4bzjHej8w9HyZXCjb
This commit is contained in:
Claude
+138
-66
@@ -1,9 +1,10 @@
|
||||
"""Fahrplan-Update: holt den Schweizer GTFS-Fahrplan, filtert auf Bahn und rechnet mit pfaedle
|
||||
die echten Streckenverläufe (shapes) aus OpenStreetMap. Läuft stündlich per systemd-Timer und
|
||||
macht nichts, solange opentransportdata.swiss keinen neuen Fahrplan veröffentlicht hat.
|
||||
"""Fahrplan-Update: holt für jede Quelle (Schweiz, Deutschland) den GTFS-Fahrplan, filtert auf Bahn
|
||||
und rechnet mit pfaedle die echten Streckenverläufe (shapes) aus OpenStreetMap. Läuft stündlich
|
||||
und macht für eine Quelle nichts, solange dort kein neuer Fahrplan veröffentlicht wurde.
|
||||
|
||||
python -m trainpulse.prepare # normaler Lauf
|
||||
python -m trainpulse.prepare --zip X # lokale Datei statt Download (zum Testen)
|
||||
python -m trainpulse.prepare # alle Quellen
|
||||
python -m trainpulse.prepare --source de --force # eine Quelle neu aufbereiten
|
||||
python -m trainpulse.prepare --source ch --zip X # lokale Datei statt Download (zum Testen)
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
@@ -39,10 +40,12 @@ def timetable_year(today: dt.date) -> int:
|
||||
return today.year
|
||||
|
||||
|
||||
def gtfs_url() -> str:
|
||||
# shortcut: in der Nacht des Fahrplanwechsels fehlen bis zum nächsten Stundenlauf die Züge
|
||||
return C.GTFS_URL or (
|
||||
f"https://data.opentransportdata.swiss/dataset/timetable-{timetable_year(dt.date.today())}-gtfs2020/permalink")
|
||||
def gtfs_url(src: str) -> str:
|
||||
url = C.SOURCES[src]["gtfs_url"]
|
||||
if src == "ch" and not url:
|
||||
# shortcut: in der Nacht des Fahrplanwechsels fehlen bis zum nächsten Stundenlauf die Züge
|
||||
url = f"https://data.opentransportdata.swiss/dataset/timetable-{timetable_year(dt.date.today())}-gtfs2020/permalink"
|
||||
return url
|
||||
|
||||
|
||||
def download(url: str, dest: Path) -> None:
|
||||
@@ -56,6 +59,19 @@ def download(url: str, dest: Path) -> None:
|
||||
tmp.rename(dest)
|
||||
|
||||
|
||||
def remote_id(src: str, url: str) -> tuple[str, str | None]:
|
||||
"""Kennung der aktuell veröffentlichten Datei, ohne sie herunterzuladen: (Name, Datum JJJJMMTT oder None)."""
|
||||
if src == "ch":
|
||||
return latest_gtfs_name(url), None
|
||||
r = httpx.head(url, follow_redirects=True, timeout=60, headers={"User-Agent": C.USER_AGENT})
|
||||
r.raise_for_status()
|
||||
lm = r.headers.get("last-modified")
|
||||
if not lm:
|
||||
raise RuntimeError(f"{url}: kein Last-Modified-Header")
|
||||
date = dt.datetime.strptime(lm, "%a, %d %b %Y %H:%M:%S GMT").strftime("%Y%m%d")
|
||||
return f"{src}-{lm}", date
|
||||
|
||||
|
||||
def latest_gtfs_name(url: str) -> str:
|
||||
# Der Permalink leitet auf .../gtfs_fp2026_YYYYMMDD.zip weiter. HEAD geht beim Speicher nicht,
|
||||
# darum nur die erste Weiterleitung lesen – so sehen wir ohne 300-MB-Download, ob es Neues gibt.
|
||||
@@ -83,26 +99,33 @@ def write_csv(path: Path, header, data) -> int:
|
||||
return n
|
||||
|
||||
|
||||
def filter_by_first_col(z: zipfile.ZipFile, name: str, keys: set, out: Path, collect: str | None = None):
|
||||
"""Schnellfilter für die Riesendateien (stop_times.txt hat ~3.7 GB): vergleicht nur das erste Feld
|
||||
als Rohtext, schreibt Treffer unverändert weg. Gibt (Anzahl, Werte der Spalte `collect`) zurück."""
|
||||
def filter_by_col(z: zipfile.ZipFile, name: str, col: str, keys: set, out: Path, collect: str | None = None):
|
||||
"""Filter für die Riesendateien (stop_times.txt hat mehrere GB): behält Zeilen, deren Spalte `col`
|
||||
in `keys` ist, und schreibt sie unverändert weg. Steht `col` vorne, wird nur das erste Feld als
|
||||
Rohtext verglichen (schnell). Gibt (Anzahl, Werte der Spalte `collect`) zurück."""
|
||||
n, seen = 0, set()
|
||||
with z.open(name) as raw, open(out, "w", newline="", encoding="utf-8") as f:
|
||||
lines = io.TextIOWrapper(raw, encoding="utf-8-sig", newline="")
|
||||
header = next(csv.reader([lines.readline()]))
|
||||
assert header[0] in ("trip_id", "service_id"), header
|
||||
f.write(",".join(header) + "\n")
|
||||
ci = header.index(collect) if collect else None
|
||||
ki, ci = header.index(col), header.index(collect) if collect else None
|
||||
for line in lines:
|
||||
if line.split(",", 1)[0].strip('"') in keys:
|
||||
f.write(line)
|
||||
n += 1
|
||||
if ci is not None:
|
||||
seen.add(next(csv.reader([line]))[ci])
|
||||
if ki == 0:
|
||||
if line.split(",", 1)[0].strip('"') not in keys:
|
||||
continue
|
||||
row = next(csv.reader([line])) if ci is not None else None
|
||||
else:
|
||||
row = next(csv.reader([line]))
|
||||
if row[ki] not in keys:
|
||||
continue
|
||||
f.write(line)
|
||||
n += 1
|
||||
if ci is not None:
|
||||
seen.add(row[ci])
|
||||
return n, seen
|
||||
|
||||
|
||||
def filter_rail(zip_path: Path, out: Path) -> dict:
|
||||
def filter_rail(zip_path: Path, out: Path, version: str | None = None) -> dict:
|
||||
out.mkdir(parents=True)
|
||||
z = zipfile.ZipFile(zip_path)
|
||||
|
||||
@@ -119,20 +142,21 @@ def filter_rail(zip_path: Path, out: Path) -> dict:
|
||||
|
||||
if "frequencies.txt" in z.namelist():
|
||||
fh, frd = rows(z, "frequencies.txt")
|
||||
n = sum(1 for r in frd if r[0] in trip_ids)
|
||||
n = sum(1 for r in frd if r[fh.index("trip_id")] in trip_ids)
|
||||
if n:
|
||||
log.warning("%d Bahnfahrten nur über frequencies.txt – werden ignoriert", n)
|
||||
|
||||
n_st, used_stops = filter_by_first_col(z, "stop_times.txt", trip_ids, out / "stop_times.txt", "stop_id")
|
||||
n_st, used_stops = filter_by_col(z, "stop_times.txt", "trip_id", trip_ids, out / "stop_times.txt", "stop_id")
|
||||
|
||||
h, rd = rows(z, "stops.txt")
|
||||
sid, par = h.index("stop_id"), h.index("parent_station")
|
||||
stops = list(rd)
|
||||
parents = {r[h.index("parent_station")] for r in stops if r[0] in used_stops}
|
||||
write_csv(out / "stops.txt", h, (r for r in stops if r[0] in used_stops or r[0] in parents))
|
||||
parents = {r[par] for r in stops if r[sid] in used_stops}
|
||||
write_csv(out / "stops.txt", h, (r for r in stops if r[sid] in used_stops or r[sid] in parents))
|
||||
|
||||
for name in ("calendar.txt", "calendar_dates.txt"):
|
||||
if name in z.namelist():
|
||||
filter_by_first_col(z, name, service_ids, out / name)
|
||||
filter_by_col(z, name, "service_id", service_ids, out / name)
|
||||
for name in ("agency.txt", "feed_info.txt"):
|
||||
if name in z.namelist():
|
||||
h, rd = rows(z, name)
|
||||
@@ -140,17 +164,40 @@ def filter_rail(zip_path: Path, out: Path) -> dict:
|
||||
|
||||
h, rd = rows(z, "feed_info.txt") if "feed_info.txt" in z.namelist() else ([], iter([]))
|
||||
info = dict(zip(h, next(rd, [])))
|
||||
return {"feed_version": info.get("feed_version") or zip_path.stem,
|
||||
fv = info.get("feed_version", "")
|
||||
return {"feed_version": fv if fv.isdigit() else (version or zip_path.stem), # gtfs.de: "latest-free"
|
||||
"routes": len(routes), "trips": len(trips), "stop_times": n_st}
|
||||
|
||||
|
||||
def ensure_osm() -> Path:
|
||||
osm = C.DATA_DIR / "osm" / "region.osm.pbf"
|
||||
if not osm.exists() or time.time() - osm.stat().st_mtime > 30 * 86400:
|
||||
osm.parent.mkdir(parents=True, exist_ok=True)
|
||||
log.info("Lade OSM-Daten %s", C.OSM_URL)
|
||||
download(C.OSM_URL, osm)
|
||||
return osm
|
||||
"""Schienennetz aus OpenStreetMap für alle Länder: herunterladen, mit osmium auf Bahn-Objekte
|
||||
filtern (Deutschland: ~4 GB -> wenige 100 MB) und zusammenführen. Monatlich erneuert."""
|
||||
d = C.DATA_DIR / "osm"
|
||||
rail, urls_file = d / "rail.osm.pbf", d / "urls.txt"
|
||||
wanted = "\n".join(C.OSM_URLS)
|
||||
if rail.exists() and time.time() - rail.stat().st_mtime < 30 * 86400 \
|
||||
and urls_file.exists() and urls_file.read_text() == wanted:
|
||||
return rail
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
parts = []
|
||||
for i, url in enumerate(C.OSM_URLS):
|
||||
raw, part = d / f"download-{i}.osm.pbf", d / f"rail-{i}.osm.pbf"
|
||||
log.info("Lade OSM-Daten %s", url)
|
||||
download(url, raw)
|
||||
log.info("Filtere Schienennetz …")
|
||||
subprocess.run([C.OSMIUM, "tags-filter", str(raw), "nwr/railway", "nwr/public_transport", "r/route",
|
||||
"-o", str(part), "--overwrite"], check=True)
|
||||
raw.unlink() # spart Platz
|
||||
parts.append(str(part))
|
||||
tmp = d / "rail.tmp.osm.pbf"
|
||||
subprocess.run([C.OSMIUM, "merge", *parts, "-o", str(tmp), "--overwrite"], check=True)
|
||||
tmp.rename(rail)
|
||||
urls_file.write_text(wanted)
|
||||
for p in parts:
|
||||
Path(p).unlink()
|
||||
for old in d.glob("region.osm.pbf"): # Datei der alten Version (nur Schweiz)
|
||||
old.unlink()
|
||||
return rail
|
||||
|
||||
|
||||
def run_pfaedle(work: Path, rail: Path) -> Path | None:
|
||||
@@ -202,9 +249,12 @@ def pack_shapes(d: Path) -> None:
|
||||
|
||||
|
||||
def ensure_tiles() -> None:
|
||||
"""Kartenkacheln (OpenStreetMap, von Protomaps) einmalig für die Region herunterladen."""
|
||||
"""Kartenkacheln (OpenStreetMap, von Protomaps) einmalig für die Region herunterladen
|
||||
(erneut, wenn sich der Ausschnitt geändert hat)."""
|
||||
dest = C.DATA_DIR / "tiles" / "region.pmtiles"
|
||||
if dest.exists() and dest.stat().st_size:
|
||||
info = C.DATA_DIR / "tiles" / "region.txt"
|
||||
wanted = f"{C.TILES_BBOX} z{C.TILES_MAXZOOM}"
|
||||
if dest.exists() and dest.stat().st_size and info.exists() and info.read_text() == wanted:
|
||||
return
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(".tmp")
|
||||
@@ -212,55 +262,48 @@ def ensure_tiles() -> None:
|
||||
d = (dt.date.today() - dt.timedelta(days=days)).strftime("%Y%m%d")
|
||||
log.info("Lade Kartenkacheln vom %s …", d)
|
||||
r = subprocess.run([C.PMTILES, "extract", f"https://build.protomaps.com/{d}.pmtiles", str(tmp),
|
||||
f"--bbox={C.TILES_BBOX}", "--maxzoom=15"])
|
||||
f"--bbox={C.TILES_BBOX}", f"--maxzoom={C.TILES_MAXZOOM}"])
|
||||
if r.returncode == 0 and tmp.exists() and tmp.stat().st_size:
|
||||
tmp.rename(dest)
|
||||
info.write_text(wanted)
|
||||
return
|
||||
log.error("Kartenkacheln konnten nicht geladen werden – Karte bleibt ohne Hintergrund")
|
||||
|
||||
|
||||
def publish(src: Path, meta: dict) -> None:
|
||||
def publish(source: str, src: Path, meta: dict) -> None:
|
||||
(src / "meta.json").write_text(json.dumps(meta))
|
||||
dest = C.FEEDS_DIR / meta["feed_version"]
|
||||
base = C.FEEDS_DIR / source
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
dest = base / meta["feed_version"]
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
src.rename(dest) # gleiche Partition -> atomar; der Server sieht nur fertige Ordner
|
||||
for old in sorted(p for p in C.FEEDS_DIR.iterdir() if p.is_dir())[:-3]:
|
||||
for old in sorted(p for p in base.iterdir() if p.is_dir())[:-2]:
|
||||
shutil.rmtree(old)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--zip", type=Path, help="lokale GTFS-Zip statt Download")
|
||||
ap.add_argument("--force", action="store_true")
|
||||
args = ap.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
C.FEEDS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if not args.zip:
|
||||
ensure_tiles()
|
||||
state_file = C.DATA_DIR / "state.json"
|
||||
state = json.loads(state_file.read_text()) if state_file.exists() else {}
|
||||
def prepare(source: str, state: dict, zip_path: Path | None = None, force: bool = False) -> None:
|
||||
work = C.DATA_DIR / "work"
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir()
|
||||
|
||||
if args.zip:
|
||||
zip_path, name = args.zip, args.zip.name
|
||||
version = None
|
||||
if zip_path:
|
||||
name = zip_path.name
|
||||
else:
|
||||
url = gtfs_url()
|
||||
name = latest_gtfs_name(url)
|
||||
if name == state.get("last_file") and not args.force:
|
||||
log.info("Kein neuer Fahrplan (%s)", name)
|
||||
url = gtfs_url(source)
|
||||
name, version = remote_id(source, url)
|
||||
if name == state.get(source) and not force:
|
||||
log.info("[%s] Kein neuer Fahrplan (%s)", source, name)
|
||||
return
|
||||
zip_path = work / name
|
||||
log.info("Lade %s", name)
|
||||
zip_path = work / "gtfs.zip"
|
||||
log.info("[%s] Lade Fahrplan %s", source, name)
|
||||
download(url, zip_path)
|
||||
|
||||
t = time.time()
|
||||
meta = filter_rail(zip_path, work / "rail")
|
||||
log.info("Bahn-Filter: %s in %.0f s", meta, time.time() - t)
|
||||
if not args.zip:
|
||||
meta = filter_rail(zip_path, work / "rail", version)
|
||||
log.info("[%s] Bahn-Filter: %s in %.0f s", source, meta, time.time() - t)
|
||||
if zip_path.parent == work:
|
||||
zip_path.unlink()
|
||||
|
||||
shaped = run_pfaedle(work, work / "rail")
|
||||
@@ -269,12 +312,41 @@ def main() -> None:
|
||||
pack_shapes(shaped)
|
||||
meta["source_file"] = name
|
||||
meta["prepared_at"] = int(time.time())
|
||||
publish(shaped or work / "rail", meta)
|
||||
|
||||
state["last_file"] = name
|
||||
state_file.write_text(json.dumps(state))
|
||||
publish(source, shaped or work / "rail", meta)
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
log.info("Fahrplan %s bereit", meta["feed_version"])
|
||||
state[source] = name
|
||||
log.info("[%s] Fahrplan %s bereit", source, meta["feed_version"])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--source", choices=list(C.SOURCES), help="nur diese Quelle (Standard: alle aktiven)")
|
||||
ap.add_argument("--zip", type=Path, help="lokale GTFS-Zip statt Download (nur mit --source)")
|
||||
ap.add_argument("--force", action="store_true")
|
||||
args = ap.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
if args.zip and not args.source:
|
||||
ap.error("--zip braucht --source")
|
||||
|
||||
C.FEEDS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
for p in C.FEEDS_DIR.iterdir(): # Ordner der alten Version (feeds/<version>) und abgeschaltete Quellen
|
||||
if p.is_dir() and p.name not in C.SOURCES:
|
||||
shutil.rmtree(p)
|
||||
if not args.zip:
|
||||
ensure_tiles()
|
||||
state_file = C.DATA_DIR / "state.json"
|
||||
state = json.loads(state_file.read_text()) if state_file.exists() else {}
|
||||
|
||||
failed = False
|
||||
for source in [args.source] if args.source else C.ENABLED:
|
||||
try:
|
||||
prepare(source, state, args.zip, args.force)
|
||||
except Exception: # eine kaputte Quelle soll die anderen nicht aufhalten
|
||||
log.exception("[%s] Aufbereitung fehlgeschlagen", source)
|
||||
failed = True
|
||||
state_file.write_text(json.dumps(state))
|
||||
if failed:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user