Streckenverläufe kompakt speichern und nur bei Bedarf lesen

pfaedle erzeugt für die Schweiz eine 2.6-GB-shapes.txt; die wird jetzt
in shapes.bin (670 MB, mmap) umgewandelt. Halt-Positionen kommen aus
shape_dist_traveled. RAM-Empfehlung im README ergänzt.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X48pE4bzjHej8w9HyZXCjb

README.md

@@ -26,7 +26,7 @@ sich die Echtzeitdaten gerade beziehen (`feed_version`), und schaltet automatisc
 ## Installation auf Proxmox
 
 1. **Container anlegen** (Proxmox-Oberfläche → *Create CT*):
-   Debian 12 oder 13 Template, unprivileged, 2 Kerne, **RAM_EMPFEHLUNG**, 20 GB Disk, Netz per DHCP.
+   Debian 12 oder 13 Template, unprivileged, 2 Kerne, **4 GB RAM** (pfaedle braucht beim Aufbereiten bis ~2 GB, der Server ~0,5 GB), 20 GB Disk, Netz per DHCP.
 2. **Projekt in den Container kopieren**, z. B. `scp trainpulse.zip root@<ip>:` und dort
    `apt install -y unzip && unzip trainpulse.zip`.
 3. **Installieren:** `bash trainpulse/deploy/install.sh`

deploy/install.sh

@@ -73,7 +73,7 @@ ip=$(hostname -I | awk '{print $1}')
 cat <<EOF
 
 Fertig. Die Karte läuft unter  http://$ip:8080
-Der erste Fahrplan wird jetzt im Hintergrund geladen und aufbereitet (ca. 10–20 min).
+Der erste Fahrplan wird jetzt im Hintergrund geladen und aufbereitet (ca. 10–15 min).
 Fortschritt:   journalctl -fu trainpulse-update
 Echtzeit:      API-Key in /etc/trainpulse.env eintragen, dann  systemctl restart trainpulse
 EOF

tests/test_core.py

@@ -19,9 +19,9 @@ FILES = {
                   "R1,11,IC5,,IC,102\nB1,11,31,,B,700\n",
     "trips.txt": "route_id,service_id,trip_id,trip_headsign,trip_short_name,shape_id\n"
                  "R1,S,T1,Ziel,812,SH1\nB1,S,BUS,Irgendwo,1,\n",
-    "stop_times.txt": "trip_id,arrival_time,departure_time,stop_id,stop_sequence\n"
+    "stop_times.txt": "trip_id,arrival_time,departure_time,stop_id,stop_sequence,shape_dist_traveled\n"
-                      "T1,08:00:00,08:00:00,A,1\nT1,08:10:00,08:12:00,B,2\nT1,08:20:00,08:20:00,C,3\n"
+                      "T1,08:00:00,08:00:00,A,1,0\nT1,08:10:00,08:12:00,B,2,8000\nT1,08:20:00,08:20:00,C,3,19000\n"
-                      "BUS,08:00:00,08:00:00,A,1\nBUS,08:05:00,08:05:00,X,2\n",
+                      "BUS,08:00:00,08:00:00,A,1,\nBUS,08:05:00,08:05:00,X,2,\n",
     "stops.txt": "stop_id,stop_name,stop_lat,stop_lon,parent_station,platform_code\n"
                  "A,Alpha,47.0,8.0,,1\nB,Beta,47.0,8.1,,2\nC,Gamma,47.1,8.1,,3\nX,Bushalt,47.5,8.5,,\n",
     "calendar.txt": "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\n"
@@ -30,8 +30,9 @@ FILES = {
     "feed_info.txt": "feed_publisher_name,feed_version\nX,20261017\n",
 }
 # Streckenverlauf mit Umweg nach Süden zwischen A und B
-SHAPE = "shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence\n" \
+# so wie pfaedle es schreibt: mit shape_dist_traveled in Metern (absichtlich nicht sortiert)
-        "SH1,47.0,8.0,1\nSH1,46.98,8.05,2\nSH1,47.0,8.1,3\nSH1,47.1,8.1,4\n"
+SHAPE = "shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled\n" \
+        "SH1,46.98,8.05,2,4000\nSH1,47.0,8.0,1,0\nSH1,47.0,8.1,3,8000\nSH1,47.1,8.1,4,19000\n"
 DAY = dt.date(2026, 10, 21)
 
 
@@ -45,6 +46,8 @@ def make_feed(tmp: Path, shapes=True) -> Feed:
     assert meta["feed_version"] == "20261017" and meta["trips"] == 1 and meta["stop_times"] == 3, meta
     if shapes:
         (tmp / "rail" / "shapes.txt").write_text(SHAPE)
+        prepare.pack_shapes(tmp / "rail")
+        assert not (tmp / "rail" / "shapes.txt").exists()
         with open(tmp / "rail" / "trips.txt") as f:
             assert "SH1" in f.read()
     (tmp / "rail" / "meta.json").write_text(json.dumps(meta))
@@ -82,9 +85,8 @@ def test_all():
 
     with tempfile.TemporaryDirectory() as d:
         f = make_feed(Path(d))
-        # Halt B liegt auf dem Streckenverlauf bei Punkt 3, nicht auf der Luftlinie
         pts, cum, sd = f.geometry(0)
-        assert abs(sd[1] - cum[2]) < 1 and abs(sd[2] - cum[3]) < 1
+        assert len(pts) == 4 and abs(pts[1][1] - 46.98) < 1e-4 and sd == [0, 8000, 19000] and cum[1] == 4000
 
         assert f.vehicles(at(7, 59), {}) == []
         v = f.vehicles(at(8, 5), {})

trainpulse/feed.py

@@ -4,6 +4,7 @@ import csv
 import datetime as dt
 import json
 import math
+import mmap
 from array import array
 from bisect import bisect_right
 from pathlib import Path
@@ -75,28 +76,28 @@ class Feed:
         n = len(self.trip_route)
         self.off = array("i", [0]) * (n + 1)
         self.st_stop, self.st_seq, self.st_arr, self.st_dep = array("i"), array("i"), array("i"), array("i")
+        self.st_dist = array("f")  # Meter entlang des Streckenverlaufs (von pfaedle), -1 = unbekannt
         rows = {}
         for h, r in read(path / "stop_times.txt"):
+            dist = r[h["shape_dist_traveled"]] if "shape_dist_traveled" in h else ""
             rows.setdefault(self.trip_index[r[h["trip_id"]]], []).append(
-                (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]), hms(r[h["departure_time"]])))
+                (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]),
+                 hms(r[h["departure_time"]]), float(dist) if dist else -1.0))
         for ti in range(n):
             self.off[ti] = len(self.st_stop)
-            for seq, s, a, d in sorted(rows.pop(ti, ())):
+            for seq, s, a, d, m in sorted(rows.pop(ti, ())):
                 self.st_seq.append(seq)
                 self.st_stop.append(s)
                 self.st_arr.append(a)
                 self.st_dep.append(d)
+                self.st_dist.append(m)
         self.off[n] = len(self.st_stop)
 
-        self.shapes = {}
+        self.shape_index, self.shape_data = {}, None
-        if (path / "shapes.txt").exists():
+        if (path / "shapes.bin").exists() and (path / "shapes.bin").stat().st_size:
-            pts = {}
+            self.shape_index = json.loads((path / "shapes.json").read_text())
-            for h, r in read(path / "shapes.txt"):
+            with open(path / "shapes.bin", "rb") as f:
-                pts.setdefault(r[h["shape_id"]], []).append(
+                self.shape_data = memoryview(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)).cast("f")
-                    (int(r[h["shape_pt_sequence"]]), float(r[h["shape_pt_lon"]]), float(r[h["shape_pt_lat"]])))
-            for sid, p in pts.items():
-                p.sort()
-                self.shapes[sid] = [(lon, lat) for _, lon, lat in p]
 
         self.calendar = {}
         for h, r in read(path / "calendar.txt"):
@@ -145,39 +146,28 @@ class Feed:
     # ---------- Geometrie ----------
 
     def geometry(self, ti: int):
-        """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts."""
+        """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts.
+        Ohne (brauchbaren) Streckenverlauf: Luftlinie zwischen den Halten."""
         o0, o1 = self.off[ti], self.off[ti + 1]
-        stops = tuple(self.st_stop[o0:o1])
+        key = (self.trip_shape[ti], tuple(self.st_stop[o0:o1]))
-        key = (self.trip_shape[ti], stops)
         g = self._geom_cache.get(key)
         if g:
             return g
-        shape = self.shapes.get(self.trip_shape[ti])
+        shp = self.shape_index.get(self.trip_shape[ti])
-        pts = shape or [(self.stop_lon[s], self.stop_lat[s]) for s in stops]
+        stop_d = list(self.st_dist[o0:o1])
-        lat0 = pts[0][1]
+        if shp and shp[1] >= 2 and min(stop_d) >= 0:
-        P = [xy(lon, lat, lat0) for lon, lat in pts]
+            v = self.shape_data[shp[0] * 3:(shp[0] + shp[1]) * 3].tolist()
-        cum = [0.0]
+            pts, cum = list(zip(v[0::3], v[1::3])), v[2::3]
-        for (x1, y1), (x2, y2) in zip(P, P[1:]):
-            cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1))
-        if shape:
-            stop_d, j = [], 0
-            for s in stops:
-                px, py = xy(self.stop_lon[s], self.stop_lat[s], lat0)
-                best = (float("inf"), cum[j], j)
-                for k in range(j, len(P) - 1):
-                    (x1, y1), (x2, y2) = P[k], P[k + 1]
-                    dx, dy = x2 - x1, y2 - y1
-                    seg = dx * dx + dy * dy
-                    t = 0.0 if seg == 0 else max(0.0, min(1.0, ((px - x1) * dx + (py - y1) * dy) / seg))
-                    dist = math.hypot(x1 + t * dx - px, y1 + t * dy - py)
-                    if dist < best[0]:
-                        best = (dist, cum[k] + t * (cum[k + 1] - cum[k]), k)
-                    elif best[0] < 100 and dist > best[0] + 1000:
-                        break  # nächster Halt gefunden, nicht bis ans Streckenende weitersuchen
-                stop_d.append(best[1])
-                j = best[2]
         else:
+            pts = [(self.stop_lon[s], self.stop_lat[s]) for s in self.st_stop[o0:o1]]
+            lat0 = pts[0][1]
+            P = [xy(lon, lat, lat0) for lon, lat in pts]
+            cum = [0.0]
+            for (x1, y1), (x2, y2) in zip(P, P[1:]):
+                cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1))
             stop_d = cum
+        if len(self._geom_cache) > 20000:
+            self._geom_cache.clear()
         g = (pts, cum, stop_d)
         self._geom_cache[key] = g
         return g

trainpulse/prepare.py

@@ -15,6 +15,7 @@ import shutil
 import subprocess
 import time
 import zipfile
+from array import array
 from pathlib import Path
 
 import httpx
@@ -168,6 +169,38 @@ def run_pfaedle(work: Path, rail: Path) -> Path | None:
     return out
 
 
+def pack_shapes(d: Path) -> None:
+    """shapes.txt ist für die Schweiz ~2.6 GB (56 Mio. Punkte) – viel zu gross für den Speicher.
+    Umwandeln in shapes.bin (float32: lon, lat, Meter ab Start) + shapes.json (id -> [Start, Anzahl]),
+    der Server liest daraus nur die Strecken, die er gerade braucht."""
+    src = d / "shapes.txt"
+    if not src.exists():
+        return
+    index, cur, pts, n = {}, None, [], 0
+    with open(d / "shapes.bin", "wb") as out:
+        def flush():
+            nonlocal n
+            if cur is not None:
+                pts.sort()
+                array("f", [v for p in pts for v in p[1:]]).tofile(out)
+                index[cur] = [n, len(pts)]
+                n += len(pts)
+        with open(src, encoding="utf-8-sig", newline="") as f:
+            r = csv.reader(f)
+            h = {k: i for i, k in enumerate(next(r))}
+            sid, lat, lon, seq = h["shape_id"], h["shape_pt_lat"], h["shape_pt_lon"], h["shape_pt_sequence"]
+            dist = h["shape_dist_traveled"]
+            for row in r:
+                if row[sid] != cur:
+                    flush()
+                    assert row[sid] not in index, "shapes.txt nicht nach shape_id gruppiert"
+                    cur, pts = row[sid], []
+                pts.append((int(row[seq]), float(row[lon]), float(row[lat]), float(row[dist])))
+            flush()
+    (d / "shapes.json").write_text(json.dumps(index, separators=(",", ":")))
+    src.unlink()
+
+
 def publish(src: Path, meta: dict) -> None:
     (src / "meta.json").write_text(json.dumps(meta))
     dest = C.FEEDS_DIR / meta["feed_version"]
@@ -212,6 +245,8 @@ def main() -> None:
 
     shaped = run_pfaedle(work, work / "rail")
     meta["shapes"] = shaped is not None
+    if shaped:
+        pack_shapes(shaped)
     meta["source_file"] = name
     meta["prepared_at"] = int(time.time())
     publish(shaped or work / "rail", meta)