From 2a4962c37989436fbb79f34ef3b761614d03555f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 21:27:54 +0200 Subject: [PATCH] WIP: Deutschland als zweite Datenquelle (gtfs.de) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mehrere Quellen (ch, de) mit je eigenem Fahrplan und Echtzeit-Feed - OSM CH+DE, mit osmium auf Schienennetz gefiltert, für pfaedle - Landesgrenze CH gegen doppelte grenzüberschreitende Züge - Import robust gegen andere Spaltenreihenfolge (gtfs.de) - Kacheln für CH+DE (Zoom 14) Noch nicht vollständig getestet (pfaedle auf Deutschland läuft). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X48pE4bzjHej8w9HyZXCjb --- Dockerfile | 2 +- README.md | 48 +++++---- tests/test_core.py | 37 ++++++- trainpulse/border.py | 27 +++++ trainpulse/ch_border.json | 1 + trainpulse/config.py | 35 +++++-- trainpulse/feed.py | 13 ++- trainpulse/prepare.py | 204 ++++++++++++++++++++++++++------------ trainpulse/realtime.py | 8 +- trainpulse/server.py | 149 +++++++++++++++++----------- web/app.js | 14 +-- 11 files changed, 370 insertions(+), 168 deletions(-) create mode 100644 trainpulse/border.py create mode 100644 trainpulse/ch_border.json diff --git a/Dockerfile b/Dockerfile index fa29468..527cfae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ FROM golang:1.25-trixie AS pmtiles RUN go install github.com/protomaps/go-pmtiles@v1.31.2 FROM python:3.13-slim-trixie -RUN apt-get update && apt-get install -y --no-install-recommends libzip5 zlib1g libbz2-1.0 libgomp1 \ +RUN apt-get update && apt-get install -y --no-install-recommends libzip5 zlib1g libbz2-1.0 libgomp1 osmium-tool \ && rm -rf /var/lib/apt/lists/* COPY --from=pfaedle /usr/local/bin/pfaedle /usr/local/bin/ COPY --from=pfaedle /usr/local/etc/pfaedle /usr/local/etc/pfaedle diff --git a/README.md b/README.md index 465fd4c..6be11b9 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,29 @@ -# TrainPulse – Schweiz +# TrainPulse -Live-Karte aller Züge in der Schweiz: Position entlang der echten Strecke, Linie, Zugnummer, -Ziel, Verspätung und Fahrtverlauf mit Gleis. Hintergrund umschaltbar zwischen Karte -(OpenStreetMap) und Satellit (swisstopo SWISSIMAGE, ausserhalb der Schweiz Sentinel-2). +Live-Karte aller Züge in der Schweiz und in Deutschland: Position entlang der echten Strecke, +Linie, Ziel, Verspätung und Fahrtverlauf mit Gleis. Hintergrund umschaltbar zwischen Karte, +dunkler Karte (OpenStreetMap) und Satellit (swisstopo SWISSIMAGE, ausserhalb der Schweiz Sentinel-2). ## Wie es funktioniert ``` -opentransportdata.swiss ──GTFS (Mo+Do)──▶ updater-Container (stündlich) - ├─ Filter: nur Bahn (route_type 100–199) -OpenStreetMap (Geofabrik) ─────────────────▶├─ pfaedle: echte Streckenverläufe - └─▶ data/feeds// -opentransportdata.swiss ──GTFS-RT (35 s)──▶ web-Container (Port 8080) - ├─ Position = Fahrplan + Verspätung, entlang Strecke - └─ /api/vehicles, /api/trip/…, /api/status, Weboberfläche +opentransportdata.swiss ──GTFS (Mo+Do)──┐ +gtfs.de (DELFI) ─────────GTFS (täglich)──┼▶ updater-Container (stündlich) + │ ├─ Filter: nur Bahn (route_type 2, 100–199) +OpenStreetMap CH + DE (Geofabrik) ───────┴▶ ├─ osmium: nur Schienennetz, pfaedle: Streckenverläufe + └─▶ data/feeds/// +opentransportdata.swiss ──GTFS-RT (35 s)──┐ +gtfs.de ─────────────────GTFS-RT (60 s)───┴▶ web-Container (Port 8080) + ├─ Position = Fahrplan + Verspätung, entlang Strecke + └─ /api/vehicles, /api/trip/…, /api/status, Weboberfläche ``` -Es gibt in der Schweiz **keine offenen GPS-Positionen** der Züge. Die Position wird aus Fahrplan -und gemeldeter Verspätung berechnet und auf den Streckenverlauf gelegt. +Es gibt **keine offenen GPS-Positionen** der Züge. Die Position wird aus Fahrplan und gemeldeter +Verspätung berechnet und auf den Streckenverlauf gelegt. + +Beide Fahrpläne enthalten grenzüberschreitende Züge. Damit sie nicht doppelt erscheinen, zeigt der +Schweizer Fahrplan nur Züge innerhalb der Schweiz und der deutsche nur Züge ausserhalb +(Landesgrenze aus OpenStreetMap, `trainpulse/ch_border.json`). Wechsel des Fahrplans: Neue Fahrpläne erscheinen Montag und Donnerstag zwischen 9 und 10 Uhr, die Echtzeitdaten stellen erst um 15 Uhr um. Der Server lädt deshalb immer den Fahrplan, auf den @@ -57,7 +63,7 @@ hinter deinen bestehenden Reverse Proxy hängen. | Status | `curl localhost:8080/api/status` | | Logs Webserver | `docker compose logs -f web` | | Logs Fahrplan-Update | `docker compose logs -f updater` | -| Fahrplan sofort neu aufbereiten | `docker compose exec updater python -m trainpulse.prepare --force` | +| Fahrplan sofort neu aufbereiten | `docker compose exec updater python -m trainpulse.prepare --source de --force` (oder `ch`) | | Selbsttest | `docker compose exec web python tests/test_core.py` | | Stoppen / Starten | `docker compose down` / `docker compose up -d` | @@ -65,7 +71,9 @@ hinter deinen bestehenden Reverse Proxy hängen. Die Hinweise erscheinen unten rechts auf der Karte. -- Fahrplan & Echtzeit: [opentransportdata.swiss](https://opentransportdata.swiss) (Quellenangabe Pflicht) +- Fahrplan & Echtzeit Schweiz: [opentransportdata.swiss](https://opentransportdata.swiss) (Quellenangabe Pflicht) +- Fahrplan Deutschland: [gtfs.de](https://gtfs.de), Daten von DELFI e.V. (CC BY 4.0) +- Echtzeit Deutschland: [gtfs.de](https://gtfs.de/de/realtime/) (CC BY-SA 4.0, Beta, Abdeckung je nach Region) - Karte: © OpenStreetMap-Mitwirkende (ODbL), Kacheln von [Protomaps](https://protomaps.com) - Streckenverläufe: berechnet mit [pfaedle](https://github.com/ad-freiburg/pfaedle) aus OpenStreetMap-Daten (ODbL) - Satellit Schweiz: © swisstopo (frei, Quellenangabe) @@ -73,9 +81,13 @@ Die Hinweise erscheinen unten rechts auf der Karte. ## Bekannte Grenzen -- Nur Bahn. Busse, Trams und Schiffe sind im Fahrplan, werden aber noch herausgefiltert. -- Der Schweizer Fahrplan enthält auch einige ausländische Züge (z. B. TGV, TER); deren Strecke - ausserhalb der Schweiz ist nur Luftlinie, weil der OSM-Ausschnitt nur die Schweiz abdeckt. +- Nur Bahn. Busse, Trams und U-Bahnen sind im Fahrplan, werden aber noch herausgefiltert. +- Züge des Schweizer Fahrplans im übrigen Ausland (z. B. Tirano, Domodossola, Frankreich) + werden nicht angezeigt, sobald sie die Schweiz verlassen. +- Der kostenlose deutsche Fahrplan enthält keine Zugnummern und kein Fahrtziel; angezeigt + wird als Ziel der letzte Halt. +- Der deutsche Echtzeit-Feed ist ~6 MB gross (komprimiert). Bei einem Abruf pro Minute sind das + rund 9 GB Datenverkehr pro Tag. - Zwischen zwei Abfragen fahren die Züge auf gerader Linie; in engen Kurven schneiden sie für ein paar Sekunden minimal ab. - Zusätzliche Fahrten, die nur in den Echtzeitdaten vorkommen (Extrazüge), fehlen. diff --git a/tests/test_core.py b/tests/test_core.py index 959f2aa..2034c92 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,5 +1,7 @@ """Selbsttest mit einem Mini-Fahrplan: python -m pytest tests (oder: python tests/test_core.py)""" +import csv import datetime as dt +import io import json import sys import tempfile @@ -36,14 +38,29 @@ SHAPE = "shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_travele DAY = dt.date(2026, 10, 21) -def make_feed(tmp: Path, shapes=True) -> Feed: +def german_style(files: dict) -> dict: + """Wie gtfs.de: Spalten in anderer Reihenfolge, kein Fahrtziel/keine Zugnummer, Version 'latest-free'.""" + out = {} + for n, c in files.items(): + rows = list(csv.reader(io.StringIO(c))) + if n == "trips.txt": + rows = [r[:3] for r in rows] # route_id, service_id, trip_id + if n == "feed_info.txt": + rows[1][1] = "latest-free" + rows = [r[::-1] for r in rows] # Spalten rückwärts + out[n] = "".join(",".join(r) + "\n" for r in rows) + return out + + +def make_feed(tmp: Path, shapes=True, files=FILES) -> Feed: tmp.mkdir(parents=True, exist_ok=True) zp = tmp / "in.zip" with zipfile.ZipFile(zp, "w") as z: - for n, c in FILES.items(): - z.writestr(n, "" + c) # wie die echten Schweizer Dateien: mit BOM - meta = prepare.filter_rail(zp, tmp / "rail") - assert meta["feed_version"] == "20261017" and meta["trips"] == 1 and meta["stop_times"] == 3, meta + for n, c in files.items(): + z.writestr(n, "\ufeff" + c) # wie die echten Schweizer Dateien: mit BOM + meta = prepare.filter_rail(zp, tmp / "rail", "20261020") + assert meta["feed_version"] == ("20261017" if files is FILES else "20261020"), meta + assert meta["trips"] == 1 and meta["stop_times"] == 3, meta if shapes: (tmp / "rail" / "shapes.txt").write_text(SHAPE) prepare.pack_shapes(tmp / "rail") @@ -127,6 +144,16 @@ def test_all(): g = make_feed(Path(d) / "noshape", shapes=False) # ohne shapes: Luftlinie assert g.vehicles(at(8, 5), {})[0]["geometry"]["coordinates"] == [8.05, 47.0] + + de = make_feed(Path(d) / "de", shapes=False, files=german_style(FILES)) # Format von gtfs.de + v = de.vehicles(at(8, 5), {}) + assert len(v) == 1 and v[0]["properties"]["to"] == "Gamma" and v[0]["properties"]["nr"] == "" + assert de.vehicles(dt.datetime(2026, 10, 25, 8, 5, tzinfo=TZ).timestamp(), {}) == [] # calendar_dates + + from trainpulse.border import SHOW, in_switzerland + assert in_switzerland(8.540, 47.378) and not in_switzerland(9.177, 47.659) # Zürich HB / Konstanz + assert not in_switzerland(8.690, 47.697) # Büsingen (deutsche Exklave) + assert SHOW["ch"](8.54, 47.378) and not SHOW["de"](8.54, 47.378) print("alle Tests ok") diff --git a/trainpulse/border.py b/trainpulse/border.py new file mode 100644 index 0000000..d2b3e86 --- /dev/null +++ b/trainpulse/border.py @@ -0,0 +1,27 @@ +"""Landesgrenze der Schweiz (aus OpenStreetMap, auf ~400 m vereinfacht, mit den Exklaven Büsingen +und Campione als Löcher). Damit zeigt jeder Fahrplan seine Züge nur im eigenen Land, sonst wären +grenzüberschreitende Züge doppelt auf der Karte.""" +import json +from pathlib import Path + +_B = json.loads((Path(__file__).with_name("ch_border.json")).read_text()) +_BBOX = (min(x for x, _ in _B["outer"]), min(y for _, y in _B["outer"]), + max(x for x, _ in _B["outer"]), max(y for _, y in _B["outer"])) + + +def _in_ring(x: float, y: float, ring) -> bool: + inside = False + for (x1, y1), (x2, y2) in zip(ring, ring[1:] + ring[:1]): + if (y1 > y) != (y2 > y) and x < (x2 - x1) * (y - y1) / (y2 - y1) + x1: + inside = not inside + return inside + + +def in_switzerland(lon: float, lat: float) -> bool: + if not (_BBOX[0] <= lon <= _BBOX[2] and _BBOX[1] <= lat <= _BBOX[3]): + return False + return _in_ring(lon, lat, _B["outer"]) and not any(_in_ring(lon, lat, h) for h in _B["holes"]) + + +# Welche Züge ein Fahrplan zeigen darf, wenn mehrere Quellen aktiv sind +SHOW = {"ch": in_switzerland, "de": lambda lon, lat: not in_switzerland(lon, lat)} diff --git a/trainpulse/ch_border.json b/trainpulse/ch_border.json new file mode 100644 index 0000000..cc7427c --- /dev/null +++ b/trainpulse/ch_border.json @@ -0,0 +1 @@ +{"outer":[[5.9559,46.1324],[5.973,46.1316],[5.9835,46.143],[5.9941,46.1446],[6.0285,46.1404],[6.0354,46.1343],[6.0523,46.1513],[6.0918,46.1519],[6.0985,46.1438],[6.1364,46.1416],[6.1856,46.1641],[6.1871,46.1804],[6.2219,46.2007],[6.2496,46.205],[6.2947,46.2252],[6.3103,46.244],[6.3057,46.2514],[6.3098,46.2562],[6.295,46.2561],[6.2949,46.2646],[6.2787,46.2513],[6.2602,46.2517],[6.2376,46.2776],[6.253,46.2904],[6.249,46.3019],[6.2196,46.3119],[6.253,46.3604],[6.3352,46.4037],[6.4256,46.4158],[6.5191,46.4564],[6.6819,46.4543],[6.8211,46.4272],[6.8019,46.3889],[6.8055,46.3785],[6.7925,46.3675],[6.7755,46.3644],[6.771,46.3538],[6.7843,46.3328],[6.7964,46.333],[6.8004,46.32],[6.8198,46.3156],[6.83,46.3002],[6.8474,46.2899],[6.8578,46.2913],[6.8649,46.2802],[6.8554,46.2552],[6.8212,46.2323],[6.8216,46.2231],[6.8035,46.2046],[6.8126,46.1815],[6.7917,46.1629],[6.7965,46.1387],[6.8151,46.1293],[6.8382,46.1329],[6.8513,46.1267],[6.8999,46.1239],[6.8951,46.1081],[6.8822,46.0952],[6.8919,46.0851],[6.8918,46.0737],[6.8803,46.0693],[6.8717,46.0512],[6.8913,46.0427],[6.9246,46.0652],[6.9373,46.0643],[6.935,46.056],[6.9501,46.0516],[6.9629,46.0304],[6.9813,46.0196],[6.985,46.0046],[7.0105,45.9974],[7.0122,45.9873],[7.0236,45.9791],[7.0085,45.969],[7.0374,45.9549],[7.0357,45.9383],[7.0437,45.9232],[7.0606,45.9134],[7.0642,45.9001],[7.0954,45.8759],[7.0921,45.8721],[7.1011,45.8593],[7.1185,45.8594],[7.1535,45.8793],[7.1641,45.8768],[7.1616,45.8716],[7.1753,45.8629],[7.1965,45.8603],[7.2001,45.8755],[7.2164,45.8888],[7.2495,45.8931],[7.2568,45.8882],[7.2769,45.9007],[7.2944,45.9218],[7.3239,45.9102],[7.3443,45.9152],[7.3612,45.9033],[7.3863,45.8971],[7.4018,45.9115],[7.4108,45.9086],[7.4271,45.9155],[7.4444,45.9316],[7.4747,45.9358],[7.4714,45.9472],[7.4992,45.9624],[7.5427,45.957],[7.5466,45.963],[7.5415,45.9762],[7.5501,45.9866],[7.5786,45.986],[7.5886,45.9705],[7.6643,45.9757],[7.681,45.957],[7.7099,45.9481],[7.7073,45.9349],[7.7205,45.9238],[7.7351,45.924],[7.7478,45.941],[7.7693,45.9371],[7.7995,45.9171],[7.8218,45.927],[7.8641,45.9167],[7.8771,45.9272],[7.8684,45.9367],[7.8818,45.964],[7.8774,45.9739],[7.8938,45.9786],[7.9087,45.9969],[7.9891,45.9959],[8.0011,46.0122],[8.0124,46.012],[8.0108,46.0189],[8.0175,46.0247],[8.0119,46.0312],[8.0354,46.044],[8.0217,46.0694],[8.0348,46.1008],[8.1082,46.1116],[8.1154,46.1303],[8.1444,46.1368],[8.1553,46.1476],[8.1499,46.1625],[8.1655,46.1822],[8.1533,46.1911],[8.1389,46.2261],[8.1153,46.236],[8.1102,46.2495],[8.0826,46.2565],[8.0859,46.2666],[8.1376,46.3018],[8.1614,46.2959],[8.1988,46.3023],[8.212,46.3093],[8.2147,46.3197],[8.2251,46.3264],[8.224,46.3344],[8.2629,46.346],[8.2638,46.364],[8.2843,46.3636],[8.3129,46.3771],[8.3187,46.3861],[8.3157,46.399],[8.2889,46.4081],[8.3075,46.4259],[8.3254,46.4257],[8.3669,46.4519],[8.393,46.4516],[8.4457,46.4637],[8.4659,46.4444],[8.4569,46.4232],[8.4683,46.4117],[8.4631,46.4049],[8.4712,46.3959],[8.4608,46.3867],[8.4702,46.3626],[8.4616,46.3533],[8.4654,46.3338],[8.4538,46.3213],[8.4437,46.3195],[8.4405,46.3041],[8.4277,46.2983],[8.4559,46.2638],[8.4423,46.2542],[8.4445,46.2489],[8.4641,46.2446],[8.4702,46.2323],[8.5324,46.2183],[8.5405,46.1978],[8.5637,46.1839],[8.5736,46.1643],[8.6032,46.1553],[8.5927,46.1429],[8.6097,46.1329],[8.6118,46.1217],[8.6476,46.1235],[8.6577,46.1127],[8.714,46.0972],[8.7423,46.1223],[8.76,46.1016],[8.783,46.0942],[8.7998,46.0947],[8.806,46.1013],[8.8521,46.0756],[8.8546,46.0616],[8.8442,46.0485],[8.8345,46.0514],[8.8285,46.0465],[8.8355,46.0438],[8.8271,46.0323],[8.8059,46.0219],[8.7859,45.9891],[8.8313,45.988],[8.8683,45.9606],[8.8937,45.959],[8.8986,45.9463],[8.8933,45.9312],[8.9143,45.9178],[8.9241,45.9056],[8.9215,45.8958],[8.9316,45.8858],[8.9338,45.8703],[8.9453,45.867],[8.9142,45.8422],[8.9121,45.8304],[8.9562,45.8434],[8.97,45.8324],[8.9867,45.8385],[8.9975,45.8349],[8.9937,45.822],[9.0173,45.818],[9.0345,45.823],[9.0357,45.841],[9.0508,45.8522],[9.0548,45.8736],[9.0664,45.8754],[9.089,45.9007],[9.0769,45.8993],[9.0766,45.9114],[9.0678,45.9106],[9.0599,45.9209],[9.0194,45.9284],[9.0225,45.9371],[9.0122,45.9438],[9.0171,45.949],[9.014,45.9605],[8.989,45.9703],[8.9958,45.9728],[8.9953,45.9805],[9.0073,45.9803],[9.0114,45.9879],[9.0285,45.9936],[9.0225,46.0169],[9.0076,46.0298],[9.0095,46.0373],[9.0227,46.0527],[9.0783,46.0653],[9.0791,46.0759],[9.0896,46.087],[9.0725,46.1181],[9.1215,46.1348],[9.1345,46.1533],[9.1565,46.1621],[9.1612,46.1707],[9.1834,46.1702],[9.1946,46.1792],[9.2022,46.2084],[9.2189,46.2144],[9.221,46.2291],[9.2484,46.2338],[9.2505,46.2665],[9.2849,46.297],[9.2816,46.3094],[9.2994,46.3275],[9.2921,46.3362],[9.2999,46.3436],[9.2963,46.3556],[9.2826,46.358],[9.2761,46.3678],[9.2844,46.3851],[9.2758,46.3949],[9.282,46.4016],[9.2803,46.4142],[9.2482,46.4332],[9.247,46.4468],[9.2663,46.4514],[9.2782,46.4617],[9.2742,46.4837],[9.2822,46.4965],[9.3637,46.5093],[9.3734,46.5039],[9.3685,46.4941],[9.3764,46.4847],[9.3852,46.4824],[9.3898,46.4731],[9.4117,46.4669],[9.4241,46.4761],[9.4247,46.4887],[9.4342,46.4979],[9.4636,46.5089],[9.461,46.4848],[9.4491,46.4844],[9.4497,46.48],[9.4629,46.4805],[9.4656,46.4695],[9.4592,46.4631],[9.4544,46.4208],[9.469,46.3886],[9.4611,46.376],[9.4964,46.3643],[9.5185,46.3271],[9.5496,46.3023],[9.5602,46.3062],[9.5782,46.2954],[9.6104,46.2946],[9.6187,46.2874],[9.635,46.2861],[9.677,46.3031],[9.6874,46.2939],[9.7098,46.2919],[9.726,46.3135],[9.7174,46.3235],[9.7275,46.3327],[9.723,46.3408],[9.7369,46.3509],[9.7769,46.3348],[9.8183,46.3497],[9.8327,46.361],[9.8682,46.3624],[9.9068,46.3809],[9.9252,46.366],[9.9345,46.3748],[9.958,46.3789],[9.9557,46.3736],[9.9645,46.3638],[9.9964,46.3511],[9.996,46.3422],[9.9799,46.3231],[10.0001,46.3132],[9.9962,46.3073],[10.0008,46.3016],[9.9917,46.2963],[9.9959,46.2849],[10.0555,46.2658],[10.0606,46.2483],[10.0438,46.2297],[10.0704,46.2172],[10.0947,46.2286],[10.1323,46.2249],[10.1765,46.2583],[10.1543,46.2928],[10.1166,46.3141],[10.1046,46.3332],[10.106,46.3422],[10.1084,46.3523],[10.1291,46.3609],[10.1277,46.3777],[10.1426,46.3862],[10.162,46.3875],[10.1667,46.4076],[10.1611,46.4154],[10.1477,46.4142],[10.143,46.4283],[10.1291,46.4318],[10.1079,46.4283],[10.1007,46.4213],[10.0802,46.4211],[10.0772,46.4282],[10.0615,46.4274],[10.0563,46.4409],[10.0419,46.4429],[10.0418,46.4522],[10.0545,46.4647],[10.0428,46.48],[10.0491,46.4996],[10.0423,46.51],[10.0515,46.5134],[10.054,46.5227],[10.0535,46.5317],[10.0439,46.5405],[10.0614,46.5457],[10.0823,46.5635],[10.0854,46.5671],[10.0787,46.5751],[10.1007,46.5827],[10.097,46.5935],[10.1024,46.6107],[10.1276,46.6051],[10.1928,46.6259],[10.2153,46.6169],[10.2241,46.6291],[10.2393,46.6354],[10.2591,46.6102],[10.2418,46.5916],[10.2461,46.575],[10.2541,46.5712],[10.2693,46.5777],[10.2874,46.5704],[10.2961,46.55],[10.3114,46.5467],[10.3255,46.5517],[10.3382,46.5427],[10.3519,46.5559],[10.397,46.5439],[10.4182,46.5513],[10.4528,46.5307],[10.4591,46.5413],[10.4715,46.5427],[10.4779,46.5574],[10.4747,46.5667],[10.4878,46.5896],[10.4844,46.6051],[10.4921,46.6154],[10.4461,46.6413],[10.4023,46.6368],[10.392,46.6552],[10.3925,46.6716],[10.3818,46.6843],[10.3935,46.6893],[10.4033,46.7055],[10.416,46.709],[10.4188,46.7191],[10.4008,46.7338],[10.4427,46.7539],[10.4405,46.774],[10.4227,46.7882],[10.4296,46.7968],[10.4476,46.8008],[10.4577,46.816],[10.4566,46.8302],[10.4665,46.8359],[10.4638,46.8412],[10.4718,46.8487],[10.4713,46.873],[10.4647,46.8843],[10.4862,46.9155],[10.4893,46.9378],[10.4555,46.9529],[10.4264,46.9573],[10.422,46.9608],[10.4266,46.9757],[10.3983,46.9971],[10.3846,47.0],[10.3721,46.9907],[10.3465,46.9896],[10.3278,46.954],[10.3094,46.9502],[10.3061,46.9403],[10.3172,46.931],[10.3161,46.9252],[10.2969,46.92],[10.2406,46.9314],[10.242,46.9188],[10.2257,46.8959],[10.2348,46.8847],[10.2327,46.8663],[10.1937,46.8664],[10.1691,46.8507],[10.1231,46.8485],[10.1054,46.8409],[10.0874,46.8612],[10.0512,46.8639],[10.0519,46.8767],[10.0178,46.9016],[9.9901,46.9024],[9.9779,46.9159],[9.9372,46.9133],[9.8761,46.9346],[9.8787,46.9571],[9.8707,46.9632],[9.8923,46.9904],[9.8888,47.0007],[9.8707,47.0097],[9.8804,47.0172],[9.8763,47.0212],[9.8598,47.0232],[9.8361,47.0125],[9.7849,47.0385],[9.7481,47.0369],[9.7419,47.0427],[9.7182,47.0434],[9.707,47.048],[9.7076,47.0538],[9.6882,47.0552],[9.6818,47.0621],[9.6451,47.0598],[9.6327,47.0513],[9.6035,47.0618],[9.5589,47.0484],[9.5398,47.0651],[9.4913,47.0562],[9.4858,47.0491],[9.4823,47.0561],[9.4758,47.0518],[9.4724,47.065],[9.5135,47.0856],[9.5193,47.0966],[9.5118,47.132],[9.4858,47.1814],[9.531,47.271],[9.5462,47.28],[9.5564,47.2984],[9.5882,47.3174],[9.6058,47.3519],[9.6245,47.3661],[9.6604,47.3701],[9.6738,47.3839],[9.6725,47.3926],[9.6516,47.4046],[9.6444,47.4341],[9.659,47.4522],[9.6224,47.4575],[9.6092,47.4705],[9.603,47.4621],[9.5951,47.4634],[9.5819,47.4831],[9.562,47.4963],[9.5587,47.5419],[9.4956,47.5515],[9.3947,47.6203],[9.2566,47.6587],[9.176,47.654],[9.1508,47.6676],[9.1392,47.6642],[9.0946,47.6793],[9.0237,47.6869],[8.9416,47.6563],[8.8937,47.6484],[8.8754,47.6549],[8.8679,47.6759],[8.8506,47.6812],[8.8577,47.6926],[8.8515,47.6976],[8.8734,47.694],[8.8727,47.7041],[8.8483,47.7034],[8.8432,47.7122],[8.8191,47.7128],[8.8263,47.7179],[8.8048,47.7245],[8.8122,47.7304],[8.8067,47.7383],[8.7966,47.7345],[8.7992,47.7271],[8.7848,47.7273],[8.7698,47.7184],[8.7693,47.7068],[8.794,47.7042],[8.8101,47.6934],[8.8001,47.6918],[8.7961,47.6753],[8.7883,47.6755],[8.7578,47.6894],[8.728,47.6928],[8.7363,47.7163],[8.7179,47.7216],[8.7112,47.7301],[8.7236,47.7456],[8.7414,47.7482],[8.7273,47.7623],[8.7141,47.7654],[8.6951,47.7557],[8.6887,47.7584],[8.681,47.7866],[8.6574,47.7917],[8.6569,47.8004],[8.6457,47.7907],[8.6481,47.7741],[8.653,47.7734],[8.6317,47.7582],[8.6194,47.7677],[8.6234,47.7773],[8.6151,47.784],[8.6222,47.7949],[8.6186,47.7983],[8.6009,47.803],[8.5754,47.7996],[8.568,47.8085],[8.562,47.7926],[8.575,47.7896],[8.5773,47.7817],[8.5629,47.7779],[8.5529,47.7846],[8.5265,47.778],[8.5202,47.7705],[8.5103,47.7762],[8.4841,47.7709],[8.4508,47.7402],[8.4549,47.7225],[8.4171,47.7102],[8.4046,47.698],[8.4208,47.6839],[8.4056,47.6742],[8.4451,47.6539],[8.466,47.6572],[8.4637,47.6485],[8.4735,47.6384],[8.478,47.6398],[8.471,47.6424],[8.4777,47.6447],[8.4764,47.6501],[8.4924,47.6421],[8.4933,47.6469],[8.5318,47.6457],[8.5269,47.6605],[8.5335,47.6631],[8.5416,47.6568],[8.54,47.6651],[8.5609,47.6703],[8.5777,47.6615],[8.6066,47.6721],[8.6081,47.6641],[8.6287,47.6491],[8.6156,47.6377],[8.6034,47.6388],[8.614,47.6444],[8.6085,47.6522],[8.596,47.6433],[8.605,47.6135],[8.5826,47.5961],[8.5622,47.6003],[8.5664,47.6099],[8.5745,47.612],[8.5579,47.6245],[8.5157,47.6338],[8.5083,47.6175],[8.4792,47.6152],[8.4685,47.6039],[8.4567,47.6023],[8.4669,47.5842],[8.4887,47.5878],[8.4947,47.581],[8.4641,47.5722],[8.4305,47.5666],[8.3972,47.5772],[8.3825,47.5657],[8.3297,47.5709],[8.2978,47.5892],[8.2977,47.6052],[8.2925,47.6089],[8.2649,47.6093],[8.2573,47.6153],[8.2265,47.605],[8.2197,47.6176],[8.2056,47.6211],[8.1649,47.5942],[8.1491,47.5958],[8.1359,47.5837],[8.1098,47.5826],[8.0995,47.5628],[8.0893,47.5577],[8.0673,47.5645],[8.0214,47.5503],[7.9599,47.5581],[7.9441,47.5439],[7.912,47.5508],[7.9072,47.5599],[7.9106,47.5714],[7.8921,47.5874],[7.8426,47.5821],[7.8228,47.588],[7.7957,47.5574],[7.6925,47.5324],[7.6687,47.5356],[7.6339,47.5612],[7.6773,47.5637],[7.6835,47.5713],[7.6859,47.5656],[7.6897,47.5714],[7.6717,47.5873],[7.6938,47.6008],[7.6752,47.592],[7.6457,47.597],[7.6191,47.5769],[7.6049,47.5779],[7.6048,47.5849],[7.589,47.5899],[7.5847,47.5755],[7.5665,47.5777],[7.5569,47.5725],[7.5573,47.565],[7.4992,47.5401],[7.5023,47.5284],[7.5194,47.5347],[7.5312,47.5282],[7.5222,47.5142],[7.5023,47.5149],[7.4979,47.5213],[7.5103,47.506],[7.5111,47.497],[7.5025,47.4908],[7.472,47.4804],[7.4624,47.4891],[7.4331,47.4972],[7.4208,47.4804],[7.4294,47.4829],[7.4558,47.4707],[7.43,47.4595],[7.4206,47.4456],[7.4031,47.4355],[7.3548,47.4339],[7.3303,47.4418],[7.2819,47.4348],[7.2466,47.4203],[7.2334,47.4386],[7.1958,47.4354],[7.1702,47.443],[7.1775,47.4681],[7.201,47.4943],[7.1623,47.49],[7.128,47.5039],[7.1125,47.4951],[7.0749,47.4882],[7.0215,47.5045],[6.983,47.4946],[6.9886,47.4869],[6.9864,47.4763],[6.9917,47.4664],[7.0005,47.4672],[6.9975,47.4564],[7.0022,47.4542],[6.9703,47.4471],[6.9637,47.4358],[6.9403,47.4333],[6.9384,47.4061],[6.9133,47.4046],[6.9178,47.4032],[6.9093,47.3961],[6.9135,47.3881],[6.8836,47.3729],[6.8793,47.3526],[6.9028,47.3596],[6.9241,47.3552],[6.9746,47.3601],[7.0156,47.3729],[7.0499,47.3614],[7.0499,47.3477],[7.0624,47.3441],[7.0529,47.3367],[7.057,47.3344],[7.0456,47.3265],[7.01,47.3245],[7.0158,47.3133],[6.9963,47.2958],[6.941,47.2867],[6.952,47.2699],[6.9473,47.2507],[6.9557,47.2431],[6.8807,47.2005],[6.8739,47.1853],[6.8411,47.1714],[6.8584,47.1654],[6.8499,47.1564],[6.806,47.1307],[6.7405,47.1096],[6.7467,47.0978],[6.7416,47.0909],[6.7259,47.0919],[6.7027,47.0821],[6.7062,47.0742],[6.6915,47.0667],[6.7192,47.0512],[6.6965,47.0373],[6.6592,47.0273],[6.6404,47.0028],[6.618,46.9916],[6.5931,46.9918],[6.5048,46.9657],[6.4959,46.9739],[6.4327,46.9287],[6.4646,46.8904],[6.46,46.8513],[6.4432,46.8327],[6.4408,46.8158],[6.431,46.8124],[6.4348,46.8015],[6.4584,46.7888],[6.4523,46.7742],[6.4303,46.7568],[6.3952,46.7485],[6.3849,46.7314],[6.3678,46.7292],[6.3717,46.7241],[6.282,46.6907],[6.179,46.6158],[6.1105,46.5764],[6.1567,46.5453],[6.1437,46.5277],[6.138,46.531],[6.1126,46.5096],[6.0969,46.4813],[6.0728,46.4654],[6.0744,46.4535],[6.0864,46.4439],[6.0639,46.4164],[6.0976,46.409],[6.1057,46.3994],[6.1497,46.3775],[6.1593,46.3797],[6.1699,46.3661],[6.1494,46.3427],[6.1379,46.3387],[6.1361,46.3285],[6.1198,46.3126],[6.1196,46.2943],[6.1024,46.2849],[6.1245,46.2513],[6.1015,46.2376],[6.0881,46.2471],[6.073,46.2413],[6.0633,46.2458],[6.0461,46.2314],[6.0337,46.2386],[6.0015,46.2205],[5.9902,46.2217],[5.9928,46.2152],[5.9747,46.215],[5.9693,46.2067],[5.9734,46.2026],[5.9637,46.197],[5.9949,46.1818],[5.988,46.1708],[5.9819,46.1732],[5.9643,46.1448],[5.9663,46.1379],[5.9559,46.1324]],"holes":[[[8.6586,47.6913],[8.6684,47.6921],[8.6681,47.6932],[8.6736,47.6951],[8.6741,47.6979],[8.6763,47.6976],[8.6736,47.7049],[8.666,47.7079],[8.6647,47.7134],[8.6714,47.711],[8.6742,47.7127],[8.6813,47.7124],[8.6799,47.7106],[8.6863,47.7086],[8.6903,47.7094],[8.6897,47.7117],[8.7013,47.7152],[8.7046,47.7134],[8.7089,47.7041],[8.7118,47.7017],[8.7114,47.6963],[8.7176,47.6949],[8.7179,47.6907],[8.7074,47.6896],[8.6901,47.6954],[8.6784,47.6875],[8.6696,47.6848],[8.6634,47.6858],[8.6641,47.6881],[8.6597,47.6895],[8.6586,47.6913]],[[8.9585,45.9648],[8.9671,45.9841],[8.9774,45.9829],[8.9763,45.9759],[8.9772,45.9719],[8.9794,45.971],[8.9756,45.9616],[8.9685,45.9619],[8.9682,45.963],[8.9585,45.9648]]]} \ No newline at end of file diff --git a/trainpulse/config.py b/trainpulse/config.py index 8a6c753..b194de9 100644 --- a/trainpulse/config.py +++ b/trainpulse/config.py @@ -2,17 +2,36 @@ import os from pathlib import Path DATA_DIR = Path(os.environ.get("TRAINPULSE_DATA", "/var/lib/trainpulse")) -FEEDS_DIR = DATA_DIR / "feeds" # fertig aufbereitete Fahrpläne, ein Ordner pro feed_version +FEEDS_DIR = DATA_DIR / "feeds" # fertig aufbereitete Fahrpläne: feeds/// TZ = "Europe/Zurich" -# Schweiz: offizieller GTFS-Fahrplan + GTFS-RT (Trip Updates) von opentransportdata.swiss -GTFS_URL = os.environ.get("TRAINPULSE_GTFS_URL", "") # leer = automatisch das aktuelle Fahrplanjahr -RT_URL = os.environ.get("TRAINPULSE_RT_URL", "https://api.opentransportdata.swiss/la/gtfs-rt") -RT_KEY = os.environ.get("TRAINPULSE_RT_KEY", "") -RT_INTERVAL = 35 # Sekunden; Limit sind 2 Abfragen/Minute (Sliding Window) +# Datenquellen. Jede hat einen GTFS-Fahrplan und einen passenden GTFS-RT-Feed (Trip Updates). +SOURCES = { + "ch": { # opentransportdata.swiss; GTFS-URL leer = automatisch das aktuelle Fahrplanjahr + "gtfs_url": os.environ.get("TRAINPULSE_GTFS_URL", ""), + "rt_url": os.environ.get("TRAINPULSE_RT_URL", "https://api.opentransportdata.swiss/la/gtfs-rt"), + "rt_key": os.environ.get("TRAINPULSE_RT_KEY", ""), + "needs_key": True, + "rt_interval": 35, # Limit sind 2 Abfragen/Minute (Sliding Window) + }, + "de": { # gtfs.de (Daten von DELFI e.V.), Fahrplan CC BY 4.0, Echtzeit CC BY-SA 4.0 + "gtfs_url": os.environ.get("TRAINPULSE_DE_GTFS_URL", "https://download.gtfs.de/germany/free/latest.zip"), + "rt_url": os.environ.get("TRAINPULSE_DE_RT_URL", "https://realtime.gtfs.de/realtime-free.pb"), + "rt_key": "", + "needs_key": False, + "rt_interval": 60, # ~6 MB pro Abruf, öfter lohnt sich nicht + }, +} +ENABLED = [s for s in os.environ.get("TRAINPULSE_SOURCES", "ch,de").split(",") if s in SOURCES] -OSM_URL = os.environ.get("TRAINPULSE_OSM_URL", "https://download.geofabrik.de/europe/switzerland-latest.osm.pbf") +# OpenStreetMap: daraus werden das Schienennetz (für pfaedle) gefiltert +OSM_URLS = os.environ.get("TRAINPULSE_OSM_URLS", " ".join([ + "https://download.geofabrik.de/europe/switzerland-latest.osm.pbf", + "https://download.geofabrik.de/europe/germany-latest.osm.pbf", +])).split() PFAEDLE = os.environ.get("TRAINPULSE_PFAEDLE", "pfaedle") +OSMIUM = os.environ.get("TRAINPULSE_OSMIUM", "osmium") PMTILES = os.environ.get("TRAINPULSE_PMTILES", "pmtiles") -TILES_BBOX = "5.8,45.7,10.6,47.9" # Schweiz mit etwas Rand +TILES_BBOX = "5.5,45.7,15.3,55.2" # Schweiz + Deutschland mit etwas Rand +TILES_MAXZOOM = 14 # die Karte vergrössert Vektorkacheln darüber hinaus selbst; spart viel Platz USER_AGENT = "trainpulse/0.1 (+https://scarriffle.com)" diff --git a/trainpulse/feed.py b/trainpulse/feed.py index 6f596ab..9c1addf 100644 --- a/trainpulse/feed.py +++ b/trainpulse/feed.py @@ -111,6 +111,9 @@ class Feed: self.st_dep.append(d) self.st_dist.append(m) self.off[n] = len(self.st_stop) + for ti in range(n): # gtfs.de liefert kein Fahrtziel: dann der letzte Halt + if not self.trip_headsign[ti] and self.off[ti + 1] > self.off[ti]: + self.trip_headsign[ti] = self.stop_name[self.st_stop[self.off[ti + 1] - 1]] self.shape_index, self.shape_data = {}, None if (path / "shapes.bin").exists() and (path / "shapes.bin").stat().st_size: @@ -131,12 +134,12 @@ class Feed: cd = path / "calendar_dates.txt" if cd.exists(): with open(cd, encoding="utf-8-sig") as f: - assert next(f).strip().replace('"', "").startswith("service_id,date,exception_type") + h = next(f).strip().replace('"', "").split(",") # Spaltenreihenfolge ist je Quelle anders + si, di, ti_ = h.index("service_id"), h.index("date"), h.index("exception_type") for line in f: - sid, date, typ = line.rstrip("\r\n").rsplit(",", 2) - date = date.strip('"') - if date in window: - self.exceptions.setdefault(date, {})[sid.strip('"')] = int(typ.strip('"')) + v = line.rstrip("\r\n").replace('"', "").split(",") # kurze Datei ohne Kommas in Werten + if v[di] in window: + self.exceptions.setdefault(v[di], {})[v[si]] = int(v[ti_]) self._day_cache = {} self._geom_cache = {} diff --git a/trainpulse/prepare.py b/trainpulse/prepare.py index c3b0c76..488b9e4 100644 --- a/trainpulse/prepare.py +++ b/trainpulse/prepare.py @@ -1,9 +1,10 @@ -"""Fahrplan-Update: holt den Schweizer GTFS-Fahrplan, filtert auf Bahn und rechnet mit pfaedle -die echten Streckenverläufe (shapes) aus OpenStreetMap. Läuft stündlich per systemd-Timer und -macht nichts, solange opentransportdata.swiss keinen neuen Fahrplan veröffentlicht hat. +"""Fahrplan-Update: holt für jede Quelle (Schweiz, Deutschland) den GTFS-Fahrplan, filtert auf Bahn +und rechnet mit pfaedle die echten Streckenverläufe (shapes) aus OpenStreetMap. Läuft stündlich +und macht für eine Quelle nichts, solange dort kein neuer Fahrplan veröffentlicht wurde. - python -m trainpulse.prepare # normaler Lauf - python -m trainpulse.prepare --zip X # lokale Datei statt Download (zum Testen) + python -m trainpulse.prepare # alle Quellen + python -m trainpulse.prepare --source de --force # eine Quelle neu aufbereiten + python -m trainpulse.prepare --source ch --zip X # lokale Datei statt Download (zum Testen) """ import argparse import csv @@ -39,10 +40,12 @@ def timetable_year(today: dt.date) -> int: return today.year -def gtfs_url() -> str: - # shortcut: in der Nacht des Fahrplanwechsels fehlen bis zum nächsten Stundenlauf die Züge - return C.GTFS_URL or ( - f"https://data.opentransportdata.swiss/dataset/timetable-{timetable_year(dt.date.today())}-gtfs2020/permalink") +def gtfs_url(src: str) -> str: + url = C.SOURCES[src]["gtfs_url"] + if src == "ch" and not url: + # shortcut: in der Nacht des Fahrplanwechsels fehlen bis zum nächsten Stundenlauf die Züge + url = f"https://data.opentransportdata.swiss/dataset/timetable-{timetable_year(dt.date.today())}-gtfs2020/permalink" + return url def download(url: str, dest: Path) -> None: @@ -56,6 +59,19 @@ def download(url: str, dest: Path) -> None: tmp.rename(dest) +def remote_id(src: str, url: str) -> tuple[str, str | None]: + """Kennung der aktuell veröffentlichten Datei, ohne sie herunterzuladen: (Name, Datum JJJJMMTT oder None).""" + if src == "ch": + return latest_gtfs_name(url), None + r = httpx.head(url, follow_redirects=True, timeout=60, headers={"User-Agent": C.USER_AGENT}) + r.raise_for_status() + lm = r.headers.get("last-modified") + if not lm: + raise RuntimeError(f"{url}: kein Last-Modified-Header") + date = dt.datetime.strptime(lm, "%a, %d %b %Y %H:%M:%S GMT").strftime("%Y%m%d") + return f"{src}-{lm}", date + + def latest_gtfs_name(url: str) -> str: # Der Permalink leitet auf .../gtfs_fp2026_YYYYMMDD.zip weiter. HEAD geht beim Speicher nicht, # darum nur die erste Weiterleitung lesen – so sehen wir ohne 300-MB-Download, ob es Neues gibt. @@ -83,26 +99,33 @@ def write_csv(path: Path, header, data) -> int: return n -def filter_by_first_col(z: zipfile.ZipFile, name: str, keys: set, out: Path, collect: str | None = None): - """Schnellfilter für die Riesendateien (stop_times.txt hat ~3.7 GB): vergleicht nur das erste Feld - als Rohtext, schreibt Treffer unverändert weg. Gibt (Anzahl, Werte der Spalte `collect`) zurück.""" +def filter_by_col(z: zipfile.ZipFile, name: str, col: str, keys: set, out: Path, collect: str | None = None): + """Filter für die Riesendateien (stop_times.txt hat mehrere GB): behält Zeilen, deren Spalte `col` + in `keys` ist, und schreibt sie unverändert weg. Steht `col` vorne, wird nur das erste Feld als + Rohtext verglichen (schnell). Gibt (Anzahl, Werte der Spalte `collect`) zurück.""" n, seen = 0, set() with z.open(name) as raw, open(out, "w", newline="", encoding="utf-8") as f: lines = io.TextIOWrapper(raw, encoding="utf-8-sig", newline="") header = next(csv.reader([lines.readline()])) - assert header[0] in ("trip_id", "service_id"), header f.write(",".join(header) + "\n") - ci = header.index(collect) if collect else None + ki, ci = header.index(col), header.index(collect) if collect else None for line in lines: - if line.split(",", 1)[0].strip('"') in keys: - f.write(line) - n += 1 - if ci is not None: - seen.add(next(csv.reader([line]))[ci]) + if ki == 0: + if line.split(",", 1)[0].strip('"') not in keys: + continue + row = next(csv.reader([line])) if ci is not None else None + else: + row = next(csv.reader([line])) + if row[ki] not in keys: + continue + f.write(line) + n += 1 + if ci is not None: + seen.add(row[ci]) return n, seen -def filter_rail(zip_path: Path, out: Path) -> dict: +def filter_rail(zip_path: Path, out: Path, version: str | None = None) -> dict: out.mkdir(parents=True) z = zipfile.ZipFile(zip_path) @@ -119,20 +142,21 @@ def filter_rail(zip_path: Path, out: Path) -> dict: if "frequencies.txt" in z.namelist(): fh, frd = rows(z, "frequencies.txt") - n = sum(1 for r in frd if r[0] in trip_ids) + n = sum(1 for r in frd if r[fh.index("trip_id")] in trip_ids) if n: log.warning("%d Bahnfahrten nur über frequencies.txt – werden ignoriert", n) - n_st, used_stops = filter_by_first_col(z, "stop_times.txt", trip_ids, out / "stop_times.txt", "stop_id") + n_st, used_stops = filter_by_col(z, "stop_times.txt", "trip_id", trip_ids, out / "stop_times.txt", "stop_id") h, rd = rows(z, "stops.txt") + sid, par = h.index("stop_id"), h.index("parent_station") stops = list(rd) - parents = {r[h.index("parent_station")] for r in stops if r[0] in used_stops} - write_csv(out / "stops.txt", h, (r for r in stops if r[0] in used_stops or r[0] in parents)) + parents = {r[par] for r in stops if r[sid] in used_stops} + write_csv(out / "stops.txt", h, (r for r in stops if r[sid] in used_stops or r[sid] in parents)) for name in ("calendar.txt", "calendar_dates.txt"): if name in z.namelist(): - filter_by_first_col(z, name, service_ids, out / name) + filter_by_col(z, name, "service_id", service_ids, out / name) for name in ("agency.txt", "feed_info.txt"): if name in z.namelist(): h, rd = rows(z, name) @@ -140,17 +164,40 @@ def filter_rail(zip_path: Path, out: Path) -> dict: h, rd = rows(z, "feed_info.txt") if "feed_info.txt" in z.namelist() else ([], iter([])) info = dict(zip(h, next(rd, []))) - return {"feed_version": info.get("feed_version") or zip_path.stem, + fv = info.get("feed_version", "") + return {"feed_version": fv if fv.isdigit() else (version or zip_path.stem), # gtfs.de: "latest-free" "routes": len(routes), "trips": len(trips), "stop_times": n_st} def ensure_osm() -> Path: - osm = C.DATA_DIR / "osm" / "region.osm.pbf" - if not osm.exists() or time.time() - osm.stat().st_mtime > 30 * 86400: - osm.parent.mkdir(parents=True, exist_ok=True) - log.info("Lade OSM-Daten %s", C.OSM_URL) - download(C.OSM_URL, osm) - return osm + """Schienennetz aus OpenStreetMap für alle Länder: herunterladen, mit osmium auf Bahn-Objekte + filtern (Deutschland: ~4 GB -> wenige 100 MB) und zusammenführen. Monatlich erneuert.""" + d = C.DATA_DIR / "osm" + rail, urls_file = d / "rail.osm.pbf", d / "urls.txt" + wanted = "\n".join(C.OSM_URLS) + if rail.exists() and time.time() - rail.stat().st_mtime < 30 * 86400 \ + and urls_file.exists() and urls_file.read_text() == wanted: + return rail + d.mkdir(parents=True, exist_ok=True) + parts = [] + for i, url in enumerate(C.OSM_URLS): + raw, part = d / f"download-{i}.osm.pbf", d / f"rail-{i}.osm.pbf" + log.info("Lade OSM-Daten %s", url) + download(url, raw) + log.info("Filtere Schienennetz …") + subprocess.run([C.OSMIUM, "tags-filter", str(raw), "nwr/railway", "nwr/public_transport", "r/route", + "-o", str(part), "--overwrite"], check=True) + raw.unlink() # spart Platz + parts.append(str(part)) + tmp = d / "rail.tmp.osm.pbf" + subprocess.run([C.OSMIUM, "merge", *parts, "-o", str(tmp), "--overwrite"], check=True) + tmp.rename(rail) + urls_file.write_text(wanted) + for p in parts: + Path(p).unlink() + for old in d.glob("region.osm.pbf"): # Datei der alten Version (nur Schweiz) + old.unlink() + return rail def run_pfaedle(work: Path, rail: Path) -> Path | None: @@ -202,9 +249,12 @@ def pack_shapes(d: Path) -> None: def ensure_tiles() -> None: - """Kartenkacheln (OpenStreetMap, von Protomaps) einmalig für die Region herunterladen.""" + """Kartenkacheln (OpenStreetMap, von Protomaps) einmalig für die Region herunterladen + (erneut, wenn sich der Ausschnitt geändert hat).""" dest = C.DATA_DIR / "tiles" / "region.pmtiles" - if dest.exists() and dest.stat().st_size: + info = C.DATA_DIR / "tiles" / "region.txt" + wanted = f"{C.TILES_BBOX} z{C.TILES_MAXZOOM}" + if dest.exists() and dest.stat().st_size and info.exists() and info.read_text() == wanted: return dest.parent.mkdir(parents=True, exist_ok=True) tmp = dest.with_suffix(".tmp") @@ -212,55 +262,48 @@ def ensure_tiles() -> None: d = (dt.date.today() - dt.timedelta(days=days)).strftime("%Y%m%d") log.info("Lade Kartenkacheln vom %s …", d) r = subprocess.run([C.PMTILES, "extract", f"https://build.protomaps.com/{d}.pmtiles", str(tmp), - f"--bbox={C.TILES_BBOX}", "--maxzoom=15"]) + f"--bbox={C.TILES_BBOX}", f"--maxzoom={C.TILES_MAXZOOM}"]) if r.returncode == 0 and tmp.exists() and tmp.stat().st_size: tmp.rename(dest) + info.write_text(wanted) return log.error("Kartenkacheln konnten nicht geladen werden – Karte bleibt ohne Hintergrund") -def publish(src: Path, meta: dict) -> None: +def publish(source: str, src: Path, meta: dict) -> None: (src / "meta.json").write_text(json.dumps(meta)) - dest = C.FEEDS_DIR / meta["feed_version"] + base = C.FEEDS_DIR / source + base.mkdir(parents=True, exist_ok=True) + dest = base / meta["feed_version"] if dest.exists(): shutil.rmtree(dest) src.rename(dest) # gleiche Partition -> atomar; der Server sieht nur fertige Ordner - for old in sorted(p for p in C.FEEDS_DIR.iterdir() if p.is_dir())[:-3]: + for old in sorted(p for p in base.iterdir() if p.is_dir())[:-2]: shutil.rmtree(old) -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--zip", type=Path, help="lokale GTFS-Zip statt Download") - ap.add_argument("--force", action="store_true") - args = ap.parse_args() - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - - C.FEEDS_DIR.mkdir(parents=True, exist_ok=True) - if not args.zip: - ensure_tiles() - state_file = C.DATA_DIR / "state.json" - state = json.loads(state_file.read_text()) if state_file.exists() else {} +def prepare(source: str, state: dict, zip_path: Path | None = None, force: bool = False) -> None: work = C.DATA_DIR / "work" shutil.rmtree(work, ignore_errors=True) work.mkdir() - if args.zip: - zip_path, name = args.zip, args.zip.name + version = None + if zip_path: + name = zip_path.name else: - url = gtfs_url() - name = latest_gtfs_name(url) - if name == state.get("last_file") and not args.force: - log.info("Kein neuer Fahrplan (%s)", name) + url = gtfs_url(source) + name, version = remote_id(source, url) + if name == state.get(source) and not force: + log.info("[%s] Kein neuer Fahrplan (%s)", source, name) return - zip_path = work / name - log.info("Lade %s", name) + zip_path = work / "gtfs.zip" + log.info("[%s] Lade Fahrplan %s", source, name) download(url, zip_path) t = time.time() - meta = filter_rail(zip_path, work / "rail") - log.info("Bahn-Filter: %s in %.0f s", meta, time.time() - t) - if not args.zip: + meta = filter_rail(zip_path, work / "rail", version) + log.info("[%s] Bahn-Filter: %s in %.0f s", source, meta, time.time() - t) + if zip_path.parent == work: zip_path.unlink() shaped = run_pfaedle(work, work / "rail") @@ -269,12 +312,41 @@ def main() -> None: pack_shapes(shaped) meta["source_file"] = name meta["prepared_at"] = int(time.time()) - publish(shaped or work / "rail", meta) - - state["last_file"] = name - state_file.write_text(json.dumps(state)) + publish(source, shaped or work / "rail", meta) shutil.rmtree(work, ignore_errors=True) - log.info("Fahrplan %s bereit", meta["feed_version"]) + state[source] = name + log.info("[%s] Fahrplan %s bereit", source, meta["feed_version"]) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--source", choices=list(C.SOURCES), help="nur diese Quelle (Standard: alle aktiven)") + ap.add_argument("--zip", type=Path, help="lokale GTFS-Zip statt Download (nur mit --source)") + ap.add_argument("--force", action="store_true") + args = ap.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + if args.zip and not args.source: + ap.error("--zip braucht --source") + + C.FEEDS_DIR.mkdir(parents=True, exist_ok=True) + for p in C.FEEDS_DIR.iterdir(): # Ordner der alten Version (feeds/) und abgeschaltete Quellen + if p.is_dir() and p.name not in C.SOURCES: + shutil.rmtree(p) + if not args.zip: + ensure_tiles() + state_file = C.DATA_DIR / "state.json" + state = json.loads(state_file.read_text()) if state_file.exists() else {} + + failed = False + for source in [args.source] if args.source else C.ENABLED: + try: + prepare(source, state, args.zip, args.force) + except Exception: # eine kaputte Quelle soll die anderen nicht aufhalten + log.exception("[%s] Aufbereitung fehlgeschlagen", source) + failed = True + state_file.write_text(json.dumps(state)) + if failed: + raise SystemExit(1) if __name__ == "__main__": diff --git a/trainpulse/realtime.py b/trainpulse/realtime.py index 68e597a..ffdb8b1 100644 --- a/trainpulse/realtime.py +++ b/trainpulse/realtime.py @@ -34,8 +34,10 @@ def parse(data: bytes, known_trips: dict): return version, out -def fetch() -> bytes: - r = httpx.get(C.RT_URL, follow_redirects=True, timeout=30, headers={ - "Authorization": f"Bearer {C.RT_KEY}", "User-Agent": C.USER_AGENT}) +def fetch(url: str, key: str = "") -> bytes: + headers = {"User-Agent": C.USER_AGENT} + if key: + headers["Authorization"] = f"Bearer {key}" + r = httpx.get(url, follow_redirects=True, timeout=30, headers=headers) r.raise_for_status() return r.content diff --git a/trainpulse/server.py b/trainpulse/server.py index cfe46a4..0cf2b68 100644 --- a/trainpulse/server.py +++ b/trainpulse/server.py @@ -15,7 +15,7 @@ from fastapi import FastAPI, HTTPException, Request, Response from fastapi.staticfiles import StaticFiles from . import config as C -from . import realtime +from . import border, realtime from .feed import TZ, Feed log = logging.getLogger("trainpulse") @@ -23,13 +23,25 @@ STEP, STEPS = 2, 5 # Positionen alle 2 s für die nächsten 10 s, damit der Bro WEB = Path(__file__).resolve().parent.parent / "web" +class Source: + """Ein Fahrplan (Schweiz, Deutschland, …) mit seinem Echtzeit-Feed.""" + + def __init__(self, name: str): + self.name, self.cfg = name, C.SOURCES[name] + self.feed: Feed | None = None + self.rt: dict = {} + self.rt_version: str | None = None + self.rt_at: float | None = None + self.rt_error: str | None = None + self.rt_tried_at: float | None = None + + @property + def has_rt(self) -> bool: + return bool(self.cfg["rt_url"]) and (bool(self.cfg["rt_key"]) or not self.cfg["needs_key"]) + + class State: - feed: Feed | None = None - rt: dict = {} - rt_version: str | None = None - rt_at: float | None = None - rt_error: str | None = None - rt_tried_at: float | None = None + sources = {name: Source(name) for name in C.ENABLED} vehicles: bytes = b'{"type":"FeatureCollection","features":[]}' vehicles_gz: bytes = gzip.compress(vehicles) count = 0 @@ -39,62 +51,78 @@ class State: S = State() -def loop(every: float, fn): +def loop(every: float, fn, *args): def run(): while True: t = time.time() try: - fn() + fn(*args) except Exception: - log.exception("Fehler in %s", fn.__name__) + log.exception("Fehler in %s %s", fn.__name__, args) time.sleep(max(1.0, every - (time.time() - t))) threading.Thread(target=run, daemon=True, name=fn.__name__).start() def watch_feeds(): - """Lädt den passenden Fahrplan: den, auf den sich die Echtzeitdaten beziehen (die schalten - erst um 15 Uhr auf einen neuen Fahrplan um), sonst den neusten.""" - ready = sorted(m.parent.name for m in C.FEEDS_DIR.glob("*/meta.json")) - if not ready: - return - want = S.rt_version if S.rt_version in ready else ready[-1] + """Lädt je Quelle den passenden Fahrplan: den, auf den sich die Echtzeitdaten beziehen (Schweiz: + die schalten erst um 15 Uhr auf einen neuen Fahrplan um), sonst den neusten.""" today = dt.datetime.now(TZ).date() - if S.feed and S.feed.version == want and S.feed.covers(today + dt.timedelta(days=1)): - return - log.info("Lade Fahrplan %s …", want) - t = time.time() - S.feed = Feed(C.FEEDS_DIR / want) - log.info("Fahrplan %s geladen in %.0f s (%d Fahrten)", want, time.time() - t, len(S.feed.trip_ids)) - try: # Echtzeit sofort holen statt auf den nächsten Abruf zu warten - poll_realtime() - except Exception: - log.exception("Erster Echtzeit-Abruf fehlgeschlagen") - compute_positions() + for src in S.sources.values(): + ready = sorted(m.parent.name for m in (C.FEEDS_DIR / src.name).glob("*/meta.json")) + if not ready: + continue + want = src.rt_version if src.rt_version in ready else ready[-1] + if src.feed and src.feed.version == want and src.feed.covers(today + dt.timedelta(days=1)): + continue + log.info("[%s] Lade Fahrplan %s …", src.name, want) + t = time.time() + src.feed = Feed(C.FEEDS_DIR / src.name / want) + log.info("[%s] Fahrplan %s geladen in %.0f s (%d Fahrten)", src.name, want, time.time() - t, + len(src.feed.trip_ids)) + try: # Echtzeit sofort holen statt auf den nächsten Abruf zu warten + poll_realtime(src) + except Exception: + log.exception("[%s] Erster Echtzeit-Abruf fehlgeschlagen", src.name) + compute_positions() -def poll_realtime(): - if not C.RT_KEY or not S.feed: +def poll_realtime(src: Source): + if not src.has_rt or not src.feed: return try: - S.rt_version, S.rt = realtime.parse(realtime.fetch(), S.feed.trip_index) - S.rt_at, S.rt_error = time.time(), None + src.rt_version, src.rt = realtime.parse(realtime.fetch(src.cfg["rt_url"], src.cfg["rt_key"]), + src.feed.trip_index) + src.rt_at, src.rt_error = time.time(), None except Exception as e: # Daten älter als 10 min sind wertlos -> verwerfen statt falsch anzeigen - S.rt_error = str(e) - if S.rt_at and time.time() - S.rt_at > 600: - S.rt = {} + src.rt_error = str(e) + if src.rt_at and time.time() - src.rt_at > 600: + src.rt = {} raise finally: - S.rt_tried_at = time.time() + src.rt_tried_at = time.time() + + +def positions(src: Source, t: float) -> list: + show = border.SHOW.get(src.name) if len(S.sources) > 1 else None + feats = [] + for f in src.feed.vehicles(t, src.rt): + if show and not show(*f["geometry"]["coordinates"]): + continue # gehört zum Fahrplan des Nachbarlandes + f["properties"]["id"] = f"{src.name}/{f['properties']['id']}" + feats.append(f) + return feats def compute_positions(): - if not S.feed: - return now = time.time() - feats = S.feed.vehicles(now, S.rt) - future = [{f["properties"]["id"]: f["geometry"]["coordinates"] + [f["properties"]["heading"]] - for f in S.feed.vehicles(now + STEP * k, S.rt)} - for k in range(1, STEPS + 1)] + feats, future = [], [{} for _ in range(STEPS)] + for src in list(S.sources.values()): + if not src.feed: + continue + feats += positions(src, now) + for k in range(STEPS): + future[k].update({f"{src.name}/{f['properties']['id']}": f["geometry"]["coordinates"] + [f["properties"]["heading"]] + for f in src.feed.vehicles(now + STEP * (k + 1), src.rt)}) for f in feats: path = [] for pos in future: @@ -114,7 +142,8 @@ async def lifespan(app): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") C.FEEDS_DIR.mkdir(parents=True, exist_ok=True) loop(60, watch_feeds) - loop(C.RT_INTERVAL, poll_realtime) + for src in S.sources.values(): + loop(src.cfg["rt_interval"], poll_realtime, src) loop(3, compute_positions) yield @@ -130,28 +159,36 @@ def vehicles(request: Request): return Response(S.vehicles, media_type="application/json", headers=headers) -@app.get("/api/trip/{date}/{ti}") -def trip(date: str, ti: int): - f = S.feed +@app.get("/api/trip/{source}/{date}/{ti}") +def trip(source: str, date: str, ti: int): + src = S.sources.get(source) + f = src.feed if src else None if not f or not (0 <= ti < len(f.trip_ids)) or not (len(date) == 8 and date.isdigit()): raise HTTPException(404) - return f.trip_detail(date, ti, time.time(), S.rt) + return f.trip_detail(date, ti, time.time(), src.rt) @app.get("/api/status") def status(): - f = S.feed + loaded = [s for s in S.sources.values() if s.feed] + rt = [s for s in loaded if s.has_rt] + ok = [s for s in rt if not s.rt_error and s.rt_at and time.time() - s.rt_at < 180] return { - "feed_version": f.version if f else None, - "shapes": f.meta.get("shapes") if f else None, - "realtime": bool(C.RT_KEY), - "rt_version": S.rt_version, - "rt_age": round(time.time() - S.rt_at) if S.rt_at else None, - "rt_error": S.rt_error, - "rt_trips": len(S.rt), - "vehicles": S.count, + "loaded": bool(loaded), # erst "bereit", wenn Positionen mit (dem ersten Versuch) Echtzeit berechnet sind - "ready": bool(f and S.computed_at and (not C.RT_KEY or (S.rt_tried_at and S.computed_at >= S.rt_tried_at))), + "ready": bool(loaded and S.computed_at and all(s.rt_tried_at and S.computed_at >= s.rt_tried_at for s in rt)), + "vehicles": S.count, + "rt_ok": bool(ok) and len(ok) == len(loaded), + "rt_some": bool(ok), + "sources": {s.name: { + "feed_version": s.feed.version if s.feed else None, + "shapes": s.feed.meta.get("shapes") if s.feed else None, + "realtime": s.has_rt, + "rt_version": s.rt_version, + "rt_age": round(time.time() - s.rt_at) if s.rt_at else None, + "rt_error": s.rt_error, + "rt_trips": len(s.rt), + } for s in S.sources.values()}, } diff --git a/web/app.js b/web/app.js index 69a012a..354de84 100644 --- a/web/app.js +++ b/web/app.js @@ -25,8 +25,8 @@ const delayColor = ["case", const map = new maplibregl.Map({ container: "map", - center: [8.23, 46.82], - zoom: 7.4, + center: [9.6, 49.2], // Deutschland + Schweiz + zoom: 5.6, hash: true, style: { version: 8, @@ -50,7 +50,8 @@ const map = new maplibregl.Map({ route: { type: "geojson", data: EMPTY }, trains: { type: "geojson", data: EMPTY, - attribution: 'Fahrplan & Echtzeit: opentransportdata.swiss', + attribution: 'Fahrplan & Echtzeit: opentransportdata.swiss, ' + + 'gtfs.de (DELFI e.V., CC BY-SA 4.0)', }, }, layers: [ @@ -234,7 +235,7 @@ async function refreshStatus() { try { const s = await (await fetch("api/status")).json(); if (!s.ready) { - loading.textContent = s.feed_version ? "Züge werden geladen …" + loading.textContent = s.loaded ? "Züge werden geladen …" : "Fahrplan wird vorbereitet … beim ersten Start dauert das 15–20 Minuten."; loading.hidden = false; return; @@ -244,8 +245,9 @@ async function refreshStatus() { await refreshTrains(); loading.hidden = true; } - const rtOk = s.realtime && !s.rt_error && s.rt_age !== null && s.rt_age < 180; - el.textContent = `${s.vehicles} Züge unterwegs` + (rtOk ? "" : " · Verspätungen zurzeit nicht verfügbar"); + const rtOk = s.rt_ok; + el.textContent = `${s.vehicles} Züge unterwegs` + (rtOk ? "" : s.rt_some + ? " · Verspätungen teilweise nicht verfügbar" : " · Verspätungen zurzeit nicht verfügbar"); el.className = rtOk ? "" : "warn"; } catch (e) { loading.textContent = "Keine Verbindung zum Server …";