From 0c58242ce70b250a9b273aa2357b2a869cc6425f Mon Sep 17 00:00:00 2001 From: EiSiMo Date: Tue, 21 Apr 2026 13:42:21 +0200 Subject: [PATCH] map debug + coord backfill, remove email channel, countdown label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Surface "X/Y passende Wohnungen mit Koordinaten" on the Karte view + admin-only "Koordinaten nachladen" button (POST /actions/backfill-coords) that geocodes missing flats via Google Maps directly from the web container - Add googlemaps dep + GMAPS_API_KEY env to web service - Light console.log in map.js ("rendering N/M markers", "building Leaflet…") so the browser DevTools shows what's happening - Drop e-mail channel from notifications UI, notify dispatcher, and _alert_status; coerce legacy 'email' channel rows back to 'ui' on save - Countdown said "Aktualisierung läuft…" next to "nächste Aktualisierung" → shortened to "aktualisiere…" Co-Authored-By: Claude Opus 4.7 (1M context) --- docker-compose.yml | 1 + web/app.py | 53 ++++++++++++++++++---- web/db.py | 29 ++++++++++++ web/geocode.py | 46 +++++++++++++++++++ web/notifications.py | 41 +---------------- web/requirements.txt | 1 + web/settings.py | 3 ++ web/static/app.js | 2 +- web/static/map.js | 3 ++ web/templates/_settings_notifications.html | 7 --- web/templates/_wohnungen_body.html | 21 ++++++++- 11 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 web/geocode.py diff --git a/docker-compose.yml b/docker-compose.yml index 10fc595..026c454 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: - SMTP_PASSWORD=${SMTP_PASSWORD:-} - SMTP_FROM=${SMTP_FROM:-lazyflat@localhost} - SMTP_STARTTLS=${SMTP_STARTTLS:-true} + - GMAPS_API_KEY=${GMAPS_API_KEY:-} volumes: - lazyflat_data:/data expose: diff --git a/web/app.py b/web/app.py index 42e4538..4d4fc9e 100644 --- a/web/app.py +++ b/web/app.py @@ -33,6 +33,7 @@ except Exception: BERLIN_TZ = timezone.utc import db +import geocode import notifications import retention from apply_client import ApplyClient, _row_to_profile @@ -220,9 +221,9 @@ def _has_filters(f) -> bool: def _alert_status(notifications_row) -> tuple[str, str]: """Return (label, chip_kind) for the user's alarm (notification) setup. - 'aktiv' only if a real push channel (telegram/email) is configured with - credentials. 'ui' is not a real alarm — the dashboard already shows - matches when you happen to be looking. + 'aktiv' only if a real push channel is configured with credentials. + 'ui' is not a real alarm — the dashboard already shows matches when you + happen to be looking. """ if not notifications_row: return "nicht eingerichtet", "warn" @@ -231,10 +232,6 @@ def _alert_status(notifications_row) -> tuple[str, str]: if notifications_row["telegram_bot_token"] and notifications_row["telegram_chat_id"]: return "aktiv (Telegram)", "ok" return "unvollständig", "warn" - if ch == "email": - if notifications_row["email_address"]: - return "aktiv (E-Mail)", "ok" - return "unvollständig", "warn" return "nicht eingerichtet", "warn" @@ -405,6 +402,12 @@ def _wohnungen_context(user) -> dict: flats_view.append({"row": f, "last": last}) rejected_view = db.rejected_flats(uid) + matched_count = len(flats_view) + matched_with_coords = sum( + 1 for item in flats_view + if item["row"]["lat"] is not None and item["row"]["lng"] is not None + ) + matched_without_coords = matched_count - matched_with_coords allowed, reason = _manual_apply_allowed() alert_label, alert_chip = _alert_status(notif_row) @@ -426,6 +429,10 @@ def _wohnungen_context(user) -> dict: "flats": flats_view, "rejected_flats": rejected_view, "map_points": map_points, + "map_matched_total": matched_count, + "map_matched_with_coords": matched_with_coords, + "map_matched_without_coords": matched_without_coords, + "gmaps_available": bool(geocode._get_client() is not None), "has_filters": _has_filters(filters_row), "alert_label": alert_label, "alert_chip": alert_chip, @@ -551,6 +558,31 @@ async def action_reject( return _wohnungen_partial_or_redirect(request, user) +@app.post("/actions/backfill-coords") +async def action_backfill_coords( + request: Request, + csrf: str = Form(...), + admin=Depends(require_admin), +): + require_csrf(admin["id"], csrf) + rows = db.flats_missing_coords(limit=500) + total = len(rows) + resolved = 0 + skipped = 0 + for row in rows: + coords = geocode.geocode(row["address"]) + if coords is None: + skipped += 1 + continue + db.set_flat_coords(row["id"], coords[0], coords[1]) + resolved += 1 + summary = f"{resolved}/{total} geocoded, {skipped} übersprungen" + logger.info("coord backfill: %s", summary) + db.log_audit(admin["username"], "coords.backfill", summary, + user_id=admin["id"], ip=client_ip(request)) + return _wohnungen_partial_or_redirect(request, admin) + + @app.post("/actions/unreject") async def action_unreject( request: Request, @@ -854,11 +886,14 @@ async def action_notifications(request: Request, user=Depends(require_user)): form = await request.form() require_csrf(user["id"], form.get("csrf", "")) def _b(n): return 1 if form.get(n, "").lower() in ("on", "true", "1", "yes") else 0 + channel = form.get("channel", "ui") + if channel not in ("ui", "telegram"): + channel = "ui" db.update_notifications(user["id"], { - "channel": form.get("channel", "ui"), + "channel": channel, "telegram_bot_token": form.get("telegram_bot_token", ""), "telegram_chat_id": form.get("telegram_chat_id", ""), - "email_address": form.get("email_address", ""), + "email_address": "", "notify_on_match": _b("notify_on_match"), "notify_on_apply_success": _b("notify_on_apply_success"), "notify_on_apply_fail": _b("notify_on_apply_fail"), diff --git a/web/db.py b/web/db.py index 449cbe4..1deabba 100644 --- a/web/db.py +++ b/web/db.py @@ -439,6 +439,35 @@ def get_flat(flat_id: str) -> Optional[sqlite3.Row]: return _conn.execute("SELECT * FROM flats WHERE id = ?", (flat_id,)).fetchone() +def flats_missing_coords(limit: int = 500) -> list[sqlite3.Row]: + return list(_conn.execute( + """SELECT id, address FROM flats + WHERE (lat IS NULL OR lng IS NULL) AND address IS NOT NULL AND address != '' + ORDER BY discovered_at DESC LIMIT ?""", + (limit,), + ).fetchall()) + + +def set_flat_coords(flat_id: str, lat: float, lng: float) -> None: + with _lock: + _conn.execute( + "UPDATE flats SET lat = ?, lng = ? WHERE id = ?", + (lat, lng, flat_id), + ) + + +def count_flats_coords() -> tuple[int, int]: + """Return (total_flats, with_coords).""" + row = _conn.execute( + "SELECT COUNT(*) AS total, " + " SUM(CASE WHEN lat IS NOT NULL AND lng IS NOT NULL THEN 1 ELSE 0 END) AS geo " + "FROM flats" + ).fetchone() + total = int(row["total"] or 0) + geo = int(row["geo"] or 0) + return total, geo + + # --------------------------------------------------------------------------- # Applications # --------------------------------------------------------------------------- diff --git a/web/geocode.py b/web/geocode.py new file mode 100644 index 0000000..6da8e2e --- /dev/null +++ b/web/geocode.py @@ -0,0 +1,46 @@ +"""Thin Google-Maps geocoding wrapper used only for the admin coord backfill. + +Runtime geocoding during scraping happens in the alert service. This module +exists so the web service can lazy-fix flats that were scraped before the +lat/lng migration (or where the alert run skipped geocoding). +""" +import logging +from typing import Optional + +from settings import GMAPS_API_KEY + +logger = logging.getLogger("web.geocode") + +_client = None + + +def _get_client(): + global _client + if _client is not None: + return _client + if not GMAPS_API_KEY: + return None + try: + import googlemaps + _client = googlemaps.Client(key=GMAPS_API_KEY) + return _client + except Exception as e: + logger.warning("googlemaps client init failed: %s", e) + return None + + +def geocode(address: str) -> Optional[tuple[float, float]]: + if not address: + return None + gm = _get_client() + if gm is None: + return None + try: + res = gm.geocode(f"{address}, Berlin, Germany") + if not res: + return None + loc = res[0]["geometry"]["location"] + return float(loc["lat"]), float(loc["lng"]) + except Exception as e: + logger.warning("geocode failed for %r: %s", address, e) + return None diff --git a/web/notifications.py b/web/notifications.py index 45facd2..2bc3ffa 100644 --- a/web/notifications.py +++ b/web/notifications.py @@ -4,25 +4,14 @@ User-level notification dispatcher. Channels: - 'ui' → no-op (dashboard shows the events anyway) - 'telegram' → per-user bot token + chat id -- 'email' → system SMTP (one outbox, per-user recipient) """ import logging -import smtplib -from email.mime.text import MIMEText from typing import Optional import requests import db -from settings import ( - PUBLIC_URL, - SMTP_FROM, - SMTP_HOST, - SMTP_PASSWORD, - SMTP_PORT, - SMTP_STARTTLS, - SMTP_USERNAME, -) +from settings import PUBLIC_URL logger = logging.getLogger("web.notifications") @@ -46,28 +35,6 @@ def _telegram_send(token: str, chat_id: str, text: str) -> bool: return False -def _email_send(recipient: str, subject: str, body: str) -> bool: - if not SMTP_HOST or not recipient: - logger.info("email skipped (SMTP_HOST=%r recipient=%r)", SMTP_HOST, recipient) - return False - try: - msg = MIMEText(body, _charset="utf-8") - msg["Subject"] = subject - msg["From"] = SMTP_FROM - msg["To"] = recipient - with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=15) as s: - if SMTP_STARTTLS: - s.starttls() - if SMTP_USERNAME: - s.login(SMTP_USERNAME, SMTP_PASSWORD) - s.send_message(msg) - logger.info("email sent to %s", recipient) - return True - except Exception as e: - logger.warning("email send failed: %s", e) - return False - - def _should_notify(notif, event: EventType) -> bool: if event == "match": return bool(notif["notify_on_match"]) @@ -86,17 +53,11 @@ def notify_user(user_id: int, event: EventType, *, if not notif or not _should_notify(notif, event): return channel = notif["channel"] or "ui" - if channel == "ui": - return if channel == "telegram": token = notif["telegram_bot_token"] chat = notif["telegram_chat_id"] if token and chat: _telegram_send(token, chat, body_markdown or body_plain) - elif channel == "email": - addr = notif["email_address"] - if addr: - _email_send(addr, subject, body_plain) except Exception: logger.exception("notify_user failed for user=%s event=%s", user_id, event) diff --git a/web/requirements.txt b/web/requirements.txt index 56293ef..b76195b 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -6,3 +6,4 @@ itsdangerous==2.2.0 python-multipart==0.0.17 python-dotenv==1.0.1 requests==2.32.5 +googlemaps==4.10.0 diff --git a/web/settings.py b/web/settings.py index a33d45d..d726733 100644 --- a/web/settings.py +++ b/web/settings.py @@ -63,3 +63,6 @@ SMTP_STARTTLS: bool = getenv("SMTP_STARTTLS", "true").lower() in ("true", "1", " # --- App URL (used to build links in notifications) --------------------------- PUBLIC_URL: str = getenv("PUBLIC_URL", "https://flat.lab.moritz.run") + +# --- Google Maps (used for one-shot coord backfill via admin action) ---------- +GMAPS_API_KEY: str = getenv("GMAPS_API_KEY", "") diff --git a/web/static/app.js b/web/static/app.js index 727ec5d..037745c 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -18,7 +18,7 @@ function fmtCountdown(iso) { const ts = Date.parse(iso); if (!iso || Number.isNaN(ts)) return "—"; const secs = Math.floor((ts - Date.now()) / 1000); - if (secs <= 0) return "Aktualisierung läuft…"; + if (secs <= 0) return "aktualisiere…"; if (secs < 60) return `in ${secs} s`; if (secs < 3600) return `in ${Math.floor(secs / 60)} min`; return `in ${Math.floor(secs / 3600)} h`; diff --git a/web/static/map.js b/web/static/map.js index fa5e5e5..9f6ec13 100644 --- a/web/static/map.js +++ b/web/static/map.js @@ -39,6 +39,8 @@ function renderMarkers(data) { const fp = fingerprintOf(data); if (fp === currentFingerprint) return; currentFingerprint = fp; + const geo = data.filter((f) => typeof f.lat === "number" && typeof f.lng === "number").length; + console.log(`[lazyflat.map] rendering ${geo}/${data.length} markers`); if (markerLayer) { markerLayer.clearLayers(); @@ -70,6 +72,7 @@ function renderMarkers(data) { } function buildMap(el) { + console.log(`[lazyflat.map] building Leaflet instance (container ${el.clientWidth}×${el.clientHeight})`); mapInstance = L.map(el, { zoomControl: false, scrollWheelZoom: false, diff --git a/web/templates/_settings_notifications.html b/web/templates/_settings_notifications.html index 202ec67..5650dba 100644 --- a/web/templates/_settings_notifications.html +++ b/web/templates/_settings_notifications.html @@ -11,7 +11,6 @@ @@ -26,12 +25,6 @@ -
- - -
-