From 0cd2c3951a194ad2a37f8ae8da3f4003323d7be9 Mon Sep 17 00:00:00 2001 From: Guido Date: Wed, 21 Oct 2026 15:07:14 +0200 Subject: [PATCH] =?UTF-8?q?Streckenverl=C3=A4ufe=20kompakt=20speichern=20u?= =?UTF-8?q?nd=20nur=20bei=20Bedarf=20lesen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pfaedle erzeugt für die Schweiz eine 2.6-GB-shapes.txt; die wird jetzt in shapes.bin (670 MB, mmap) umgewandelt. Halt-Positionen kommen aus shape_dist_traveled. RAM-Empfehlung im README ergänzt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X48pE4bzjHej8w9HyZXCjb --- README.md | 2 +- deploy/install.sh | 2 +- tests/test_core.py | 16 ++++++----- trainpulse/feed.py | 66 ++++++++++++++++++------------------------- trainpulse/prepare.py | 35 +++++++++++++++++++++++ 5 files changed, 74 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index f614840..e277151 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ sich die Echtzeitdaten gerade beziehen (`feed_version`), und schaltet automatisc ## Installation auf Proxmox 1. **Container anlegen** (Proxmox-Oberfläche → *Create CT*): - Debian 12 oder 13 Template, unprivileged, 2 Kerne, **RAM_EMPFEHLUNG**, 20 GB Disk, Netz per DHCP. + Debian 12 oder 13 Template, unprivileged, 2 Kerne, **4 GB RAM** (pfaedle braucht beim Aufbereiten bis ~2 GB, der Server ~0,5 GB), 20 GB Disk, Netz per DHCP. 2. **Projekt in den Container kopieren**, z. B. `scp trainpulse.zip root@:` und dort `apt install -y unzip && unzip trainpulse.zip`. 3. **Installieren:** `bash trainpulse/deploy/install.sh` diff --git a/deploy/install.sh b/deploy/install.sh index 373392d..c37c1b0 100644 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -73,7 +73,7 @@ ip=$(hostname -I | awk '{print $1}') cat < Feed: assert meta["feed_version"] == "20261017" and meta["trips"] == 1 and meta["stop_times"] == 3, meta if shapes: (tmp / "rail" / "shapes.txt").write_text(SHAPE) + prepare.pack_shapes(tmp / "rail") + assert not (tmp / "rail" / "shapes.txt").exists() with open(tmp / "rail" / "trips.txt") as f: assert "SH1" in f.read() (tmp / "rail" / "meta.json").write_text(json.dumps(meta)) @@ -82,9 +85,8 @@ def test_all(): with tempfile.TemporaryDirectory() as d: f = make_feed(Path(d)) - # Halt B liegt auf dem Streckenverlauf bei Punkt 3, nicht auf der Luftlinie pts, cum, sd = f.geometry(0) - assert abs(sd[1] - cum[2]) < 1 and abs(sd[2] - cum[3]) < 1 + assert len(pts) == 4 and abs(pts[1][1] - 46.98) < 1e-4 and sd == [0, 8000, 19000] and cum[1] == 4000 assert f.vehicles(at(7, 59), {}) == [] v = f.vehicles(at(8, 5), {}) diff --git a/trainpulse/feed.py b/trainpulse/feed.py index 046c26a..de8dec5 100644 --- a/trainpulse/feed.py +++ b/trainpulse/feed.py @@ -4,6 +4,7 @@ import csv import datetime as dt import json import math +import mmap from array import array from bisect import bisect_right from pathlib import Path @@ -75,28 +76,28 @@ class Feed: n = len(self.trip_route) self.off = array("i", [0]) * (n + 1) self.st_stop, self.st_seq, self.st_arr, self.st_dep = array("i"), array("i"), array("i"), array("i") + self.st_dist = array("f") # Meter entlang des Streckenverlaufs (von pfaedle), -1 = unbekannt rows = {} for h, r in read(path / "stop_times.txt"): + dist = r[h["shape_dist_traveled"]] if "shape_dist_traveled" in h else "" rows.setdefault(self.trip_index[r[h["trip_id"]]], []).append( - (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]), hms(r[h["departure_time"]]))) + (int(r[h["stop_sequence"]]), self.stop_index[r[h["stop_id"]]], hms(r[h["arrival_time"]]), + hms(r[h["departure_time"]]), float(dist) if dist else -1.0)) for ti in range(n): self.off[ti] = len(self.st_stop) - for seq, s, a, d in sorted(rows.pop(ti, ())): + for seq, s, a, d, m in sorted(rows.pop(ti, ())): self.st_seq.append(seq) self.st_stop.append(s) self.st_arr.append(a) self.st_dep.append(d) + self.st_dist.append(m) self.off[n] = len(self.st_stop) - self.shapes = {} - if (path / "shapes.txt").exists(): - pts = {} - for h, r in read(path / "shapes.txt"): - pts.setdefault(r[h["shape_id"]], []).append( - (int(r[h["shape_pt_sequence"]]), float(r[h["shape_pt_lon"]]), float(r[h["shape_pt_lat"]]))) - for sid, p in pts.items(): - p.sort() - self.shapes[sid] = [(lon, lat) for _, lon, lat in p] + self.shape_index, self.shape_data = {}, None + if (path / "shapes.bin").exists() and (path / "shapes.bin").stat().st_size: + self.shape_index = json.loads((path / "shapes.json").read_text()) + with open(path / "shapes.bin", "rb") as f: + self.shape_data = memoryview(mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)).cast("f") self.calendar = {} for h, r in read(path / "calendar.txt"): @@ -145,39 +146,28 @@ class Feed: # ---------- Geometrie ---------- def geometry(self, ti: int): - """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts.""" + """Streckenverlauf einer Fahrt: Punkte, kumulierte Meter, Meter-Position jedes Halts. + Ohne (brauchbaren) Streckenverlauf: Luftlinie zwischen den Halten.""" o0, o1 = self.off[ti], self.off[ti + 1] - stops = tuple(self.st_stop[o0:o1]) - key = (self.trip_shape[ti], stops) + key = (self.trip_shape[ti], tuple(self.st_stop[o0:o1])) g = self._geom_cache.get(key) if g: return g - shape = self.shapes.get(self.trip_shape[ti]) - pts = shape or [(self.stop_lon[s], self.stop_lat[s]) for s in stops] - lat0 = pts[0][1] - P = [xy(lon, lat, lat0) for lon, lat in pts] - cum = [0.0] - for (x1, y1), (x2, y2) in zip(P, P[1:]): - cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1)) - if shape: - stop_d, j = [], 0 - for s in stops: - px, py = xy(self.stop_lon[s], self.stop_lat[s], lat0) - best = (float("inf"), cum[j], j) - for k in range(j, len(P) - 1): - (x1, y1), (x2, y2) = P[k], P[k + 1] - dx, dy = x2 - x1, y2 - y1 - seg = dx * dx + dy * dy - t = 0.0 if seg == 0 else max(0.0, min(1.0, ((px - x1) * dx + (py - y1) * dy) / seg)) - dist = math.hypot(x1 + t * dx - px, y1 + t * dy - py) - if dist < best[0]: - best = (dist, cum[k] + t * (cum[k + 1] - cum[k]), k) - elif best[0] < 100 and dist > best[0] + 1000: - break # nächster Halt gefunden, nicht bis ans Streckenende weitersuchen - stop_d.append(best[1]) - j = best[2] + shp = self.shape_index.get(self.trip_shape[ti]) + stop_d = list(self.st_dist[o0:o1]) + if shp and shp[1] >= 2 and min(stop_d) >= 0: + v = self.shape_data[shp[0] * 3:(shp[0] + shp[1]) * 3].tolist() + pts, cum = list(zip(v[0::3], v[1::3])), v[2::3] else: + pts = [(self.stop_lon[s], self.stop_lat[s]) for s in self.st_stop[o0:o1]] + lat0 = pts[0][1] + P = [xy(lon, lat, lat0) for lon, lat in pts] + cum = [0.0] + for (x1, y1), (x2, y2) in zip(P, P[1:]): + cum.append(cum[-1] + math.hypot(x2 - x1, y2 - y1)) stop_d = cum + if len(self._geom_cache) > 20000: + self._geom_cache.clear() g = (pts, cum, stop_d) self._geom_cache[key] = g return g diff --git a/trainpulse/prepare.py b/trainpulse/prepare.py index 40aed0c..b6ff332 100644 --- a/trainpulse/prepare.py +++ b/trainpulse/prepare.py @@ -15,6 +15,7 @@ import shutil import subprocess import time import zipfile +from array import array from pathlib import Path import httpx @@ -168,6 +169,38 @@ def run_pfaedle(work: Path, rail: Path) -> Path | None: return out +def pack_shapes(d: Path) -> None: + """shapes.txt ist für die Schweiz ~2.6 GB (56 Mio. Punkte) – viel zu gross für den Speicher. + Umwandeln in shapes.bin (float32: lon, lat, Meter ab Start) + shapes.json (id -> [Start, Anzahl]), + der Server liest daraus nur die Strecken, die er gerade braucht.""" + src = d / "shapes.txt" + if not src.exists(): + return + index, cur, pts, n = {}, None, [], 0 + with open(d / "shapes.bin", "wb") as out: + def flush(): + nonlocal n + if cur is not None: + pts.sort() + array("f", [v for p in pts for v in p[1:]]).tofile(out) + index[cur] = [n, len(pts)] + n += len(pts) + with open(src, encoding="utf-8-sig", newline="") as f: + r = csv.reader(f) + h = {k: i for i, k in enumerate(next(r))} + sid, lat, lon, seq = h["shape_id"], h["shape_pt_lat"], h["shape_pt_lon"], h["shape_pt_sequence"] + dist = h["shape_dist_traveled"] + for row in r: + if row[sid] != cur: + flush() + assert row[sid] not in index, "shapes.txt nicht nach shape_id gruppiert" + cur, pts = row[sid], [] + pts.append((int(row[seq]), float(row[lon]), float(row[lat]), float(row[dist]))) + flush() + (d / "shapes.json").write_text(json.dumps(index, separators=(",", ":"))) + src.unlink() + + def publish(src: Path, meta: dict) -> None: (src / "meta.json").write_text(json.dumps(meta)) dest = C.FEEDS_DIR / meta["feed_version"] @@ -212,6 +245,8 @@ def main() -> None: shaped = run_pfaedle(work, work / "rail") meta["shapes"] = shaped is not None + if shaped: + pack_shapes(shaped) meta["source_file"] = name meta["prepared_at"] = int(time.time()) publish(shaped or work / "rail", meta)