Streckenverläufe kompakt speichern und nur bei Bedarf lesen

pfaedle erzeugt für die Schweiz eine 2.6-GB-shapes.txt; die wird jetzt
in shapes.bin (670 MB, mmap) umgewandelt. Halt-Positionen kommen aus
shape_dist_traveled. RAM-Empfehlung im README ergänzt.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X48pE4bzjHej8w9HyZXCjb

trainpulse/feed.py

@@ -4,6 +4,7 @@ import csv
 import datetime as dt
 import json
 import math
+import mmap
 from array import array
 from bisect import bisect_right
 from pathlib import Path
@@ -75,28 +76,28 @@ class Feed:
         n = len(self.trip_route)
         self.off = array("i", [0]) * (n + 1)
         self.st_stop, self.st_seq, self.st_arr, self.st_dep = array("i"), array("i"), array("i"), array("i")
+        self.st_dist = array("f")  # Meter entlang des Streckenverlaufs (von pfaedle), -1 = unbekannt
         rows = {}
         for h, r in read(path / "stop_times.txt"):
+            dist = r[h["shape_dist_traveled"]] if "shape_dist_traveled" in h else ""
             rows.setdefault(self.trip_index[r[h["trip_id"]]], []).append(
-                (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]), hms(r[h["departure_time"]])))
+                (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]),
+                 hms(r[h["departure_time"]]), float(dist) if dist else -1.0))
         for ti in range(n):
             self.off[ti] = len(self.st_stop)
-            for seq, s, a, d in sorted(rows.pop(ti, ())):
+            for seq, s, a, d, m in sorted(rows.pop(ti, ())):
                 self.st_seq.append(seq)
                 self.st_stop.append(s)
                 self.st_arr.append(a)
                 self.st_dep.append(d)
+                self.st_dist.append(m)
         self.off[n] = len(self.st_stop)
 
-        self.shapes = {}
-        if (path / "shapes.txt").exists():
-            pts = {}
-            for h, r in read(path / "shapes.txt"):
-                pts.setdefault(r[h["shape_id"]], []).append(
-                    (int(r[h["shape_pt_sequence"]]), float(r[h["shape_pt_lon"]]), float(r[h["shape_pt_lat"]])))
-            for sid, p in pts.items():
-                p.sort()
-                self.shapes[sid] = [(lon, lat) for _, lon, lat in p]
+        self.shape_index, self.shape_data = {}, None
+        if (path / "shapes.bin").exists() and (path / "shapes.bin").stat().st_size:
+            self.shape_index = json.loads((path / "shapes.json").read_text())
+            with open(path / "shapes.bin", "rb") as f:
+                self.shape_data = memoryview(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)).cast("f")
 
         self.calendar = {}
         for h, r in read(path / "calendar.txt"):
@@ -145,39 +146,28 @@ class Feed:
     # ---------- Geometrie ----------
 
     def geometry(self, ti: int):
-        """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts."""
+        """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts.
+        Ohne (brauchbaren) Streckenverlauf: Luftlinie zwischen den Halten."""
         o0, o1 = self.off[ti], self.off[ti + 1]
-        stops = tuple(self.st_stop[o0:o1])
-        key = (self.trip_shape[ti], stops)
+        key = (self.trip_shape[ti], tuple(self.st_stop[o0:o1]))
         g = self._geom_cache.get(key)
         if g:
             return g
-        shape = self.shapes.get(self.trip_shape[ti])
-        pts = shape or [(self.stop_lon[s], self.stop_lat[s]) for s in stops]
-        lat0 = pts[0][1]
-        P = [xy(lon, lat, lat0) for lon, lat in pts]
-        cum = [0.0]
-        for (x1, y1), (x2, y2) in zip(P, P[1:]):
-            cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1))
-        if shape:
-            stop_d, j = [], 0
-            for s in stops:
-                px, py = xy(self.stop_lon[s], self.stop_lat[s], lat0)
-                best = (float("inf"), cum[j], j)
-                for k in range(j, len(P) - 1):
-                    (x1, y1), (x2, y2) = P[k], P[k + 1]
-                    dx, dy = x2 - x1, y2 - y1
-                    seg = dx * dx + dy * dy
-                    t = 0.0 if seg == 0 else max(0.0, min(1.0, ((px - x1) * dx + (py - y1) * dy) / seg))
-                    dist = math.hypot(x1 + t * dx - px, y1 + t * dy - py)
-                    if dist < best[0]:
-                        best = (dist, cum[k] + t * (cum[k + 1] - cum[k]), k)
-                    elif best[0] < 100 and dist > best[0] + 1000:
-                        break  # nächster Halt gefunden, nicht bis ans Streckenende weitersuchen
-                stop_d.append(best[1])
-                j = best[2]
+        shp = self.shape_index.get(self.trip_shape[ti])
+        stop_d = list(self.st_dist[o0:o1])
+        if shp and shp[1] >= 2 and min(stop_d) >= 0:
+            v = self.shape_data[shp[0] * 3:(shp[0] + shp[1]) * 3].tolist()
+            pts, cum = list(zip(v[0::3], v[1::3])), v[2::3]
         else:
+            pts = [(self.stop_lon[s], self.stop_lat[s]) for s in self.st_stop[o0:o1]]
+            lat0 = pts[0][1]
+            P = [xy(lon, lat, lat0) for lon, lat in pts]
+            cum = [0.0]
+            for (x1, y1), (x2, y2) in zip(P, P[1:]):
+                cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1))
             stop_d = cum
+        if len(self._geom_cache) > 20000:
+            self._geom_cache.clear()
         g = (pts, cum, stop_d)
         self._geom_cache[key] = g
         return g

trainpulse/prepare.py

@@ -15,6 +15,7 @@ import shutil
 import subprocess
 import time
 import zipfile
+from array import array
 from pathlib import Path
 
 import httpx
@@ -168,6 +169,38 @@ def run_pfaedle(work: Path, rail: Path) -> Path | None:
     return out
 
 
+def pack_shapes(d: Path) -> None:
+    """shapes.txt ist für die Schweiz ~2.6 GB (56 Mio. Punkte) – viel zu gross für den Speicher.
+    Umwandeln in shapes.bin (float32: lon, lat, Meter ab Start) + shapes.json (id -> [Start, Anzahl]),
+    der Server liest daraus nur die Strecken, die er gerade braucht."""
+    src = d / "shapes.txt"
+    if not src.exists():
+        return
+    index, cur, pts, n = {}, None, [], 0
+    with open(d / "shapes.bin", "wb") as out:
+        def flush():
+            nonlocal n
+            if cur is not None:
+                pts.sort()
+                array("f", [v for p in pts for v in p[1:]]).tofile(out)
+                index[cur] = [n, len(pts)]
+                n += len(pts)
+        with open(src, encoding="utf-8-sig", newline="") as f:
+            r = csv.reader(f)
+            h = {k: i for i, k in enumerate(next(r))}
+            sid, lat, lon, seq = h["shape_id"], h["shape_pt_lat"], h["shape_pt_lon"], h["shape_pt_sequence"]
+            dist = h["shape_dist_traveled"]
+            for row in r:
+                if row[sid] != cur:
+                    flush()
+                    assert row[sid] not in index, "shapes.txt nicht nach shape_id gruppiert"
+                    cur, pts = row[sid], []
+                pts.append((int(row[seq]), float(row[lon]), float(row[lat]), float(row[dist])))
+            flush()
+    (d / "shapes.json").write_text(json.dumps(index, separators=(",", ":")))
+    src.unlink()
+
+
 def publish(src: Path, meta: dict) -> None:
     (src / "meta.json").write_text(json.dumps(meta))
     dest = C.FEEDS_DIR / meta["feed_version"]
@@ -212,6 +245,8 @@ def main() -> None:
 
     shaped = run_pfaedle(work, work / "rail")
     meta["shapes"] = shaped is not None
+    if shaped:
+        pack_shapes(shaped)
     meta["source_file"] = name
     meta["prepared_at"] = int(time.time())
     publish(shaped or work / "rail", meta)