diff --git a/web/app.py b/web/app.py index 3b9b198..6688720 100644 --- a/web/app.py +++ b/web/app.py @@ -1,27 +1,37 @@ """ lazyflat web app. -Five tabs: -- / → Wohnungen (all flats, per-user match highlighting, filter block, auto-apply switch) -- /bewerbungen → Bewerbungen (user's application history, forensics drill-in) -- /logs → Logs (user-scoped audit log) -- /fehler → Fehler (user-scoped error records + admin-global) +Four tabs: +- / → Wohnungen (all flats, per-user match highlighting, filter block, auto-apply switch) +- /bewerbungen → Bewerbungen (history + forensics; failed apps expose a ZIP report download) +- /logs → Logs (user-scoped audit log) - /einstellungen/
→ Einstellungen: profile, filter, notifications, account, admin users All state-changing POSTs require CSRF. Internal endpoints require INTERNAL_API_KEY. """ import asyncio +import base64 import hmac +import io import json import logging import sqlite3 -import threading +import zipfile +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Any from fastapi import Depends, FastAPI, Form, Header, HTTPException, Request, Response, status -from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +try: + from zoneinfo import ZoneInfo + BERLIN_TZ = ZoneInfo("Europe/Berlin") +except Exception: + BERLIN_TZ = timezone.utc + import db import notifications import retention @@ -40,7 +50,12 @@ from auth import ( verify_login, ) from matching import flat_matches_filter, row_to_dict -from settings import APPLY_FAILURE_THRESHOLD, INTERNAL_API_KEY, PUBLIC_URL +from settings import ( + ALERT_SCRAPE_INTERVAL_SECONDS, + APPLY_FAILURE_THRESHOLD, + INTERNAL_API_KEY, + PUBLIC_URL, +) logging.basicConfig( @@ -55,11 +70,9 @@ apply_client = ApplyClient() # --------------------------------------------------------------------------- -# App +# App + Jinja # --------------------------------------------------------------------------- -from contextlib import asynccontextmanager - @asynccontextmanager async def lifespan(_app: FastAPI): db.init_db() @@ -74,6 +87,40 @@ app.mount("/static", StaticFiles(directory="static"), name="static") templates = Jinja2Templates(directory="templates") +def _parse_iso(s: str | None) -> datetime | None: + if not s: + return None + try: + # Python handles +00:00 but not trailing Z. Storage always uses +00:00. + return datetime.fromisoformat(s) + except ValueError: + return None + + +def _de_dt(s: str | None) -> str: + """UTC ISO → 'DD.MM.YYYY HH:MM' in Europe/Berlin.""" + dt = _parse_iso(s) + if dt is None: + return "" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(BERLIN_TZ).strftime("%d.%m.%Y %H:%M") + + +def _iso_utc(s: str | None) -> str: + """Pass-through of a UTC ISO string (for data-attr in JS).""" + dt = _parse_iso(s) + if dt is None: + return "" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat(timespec="seconds") + + +templates.env.filters["de_dt"] = _de_dt +templates.env.filters["iso_utc"] = _iso_utc + + @app.middleware("http") async def security_headers(request: Request, call_next): resp: Response = await call_next(request) @@ -117,17 +164,32 @@ def base_context(request: Request, user, active_tab: str) -> dict: } -def _get_apply_gate(user_id: int) -> tuple[bool, str]: - prefs = db.get_preferences(user_id) - if prefs["kill_switch"]: - return False, "kill switch aktiv" - if prefs["apply_circuit_open"]: - return False, "circuit breaker offen (zu viele fehlgeschlagene Bewerbungen)" +def _manual_apply_allowed() -> tuple[bool, str]: + """Manual 'Bewerben' button is only blocked if the apply service is down.""" if not apply_client.health(): return False, "apply-service nicht erreichbar" return True, "" +def _auto_apply_allowed(prefs) -> bool: + """Auto-apply trigger gate: user must have enabled it and the circuit must be closed.""" + if not prefs["auto_apply_enabled"]: + return False + if prefs["apply_circuit_open"]: + return False + return apply_client.health() + + +def _next_scrape_utc() -> str: + hb = db.get_state("last_alert_heartbeat") + dt = _parse_iso(hb) + if dt is None: + return "" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return (dt + timedelta(seconds=ALERT_SCRAPE_INTERVAL_SECONDS)).astimezone(timezone.utc).isoformat(timespec="seconds") + + def _run_apply_background(user_id: int, flat_id: str, url: str, triggered_by: str) -> None: prefs = db.get_preferences(user_id) profile_row = db.get_profile(user_id) @@ -153,7 +215,6 @@ def _run_apply_background(user_id: int, flat_id: str, url: str, triggered_by: st db.finish_application(app_id, success=success, message=message, provider=provider, forensics=forensics) - # Circuit breaker (per user) if success: db.update_preferences(user_id, {"apply_recent_failures": 0}) else: @@ -165,13 +226,11 @@ def _run_apply_background(user_id: int, flat_id: str, url: str, triggered_by: st summary=f"{failures} aufeinanderfolgende Fehler", application_id=app_id) db.update_preferences(user_id, updates) - # record forensic error row db.log_error(source="apply", kind="apply_failure", user_id=user_id, summary=message or "Bewerbung fehlgeschlagen", application_id=app_id, context={"provider": provider, "url": url}) - # Notify user flat = db.get_flat(flat_id) flat_dict = {"address": flat["address"] if flat else "", "link": url, "rooms": flat["rooms"] if flat else None, @@ -246,17 +305,6 @@ def logout(request: Request): # Tab: Wohnungen # --------------------------------------------------------------------------- -@app.get("/", response_class=HTMLResponse) -def tab_wohnungen(request: Request): - u = current_user(request) - if not u: - return RedirectResponse("/login", status_code=303) - - ctx = base_context(request, u, "wohnungen") - ctx.update(_wohnungen_context(u)) - return templates.TemplateResponse("wohnungen.html", ctx) - - def _wohnungen_context(user) -> dict: uid = user["id"] filters_row = db.get_filters(uid) @@ -266,14 +314,9 @@ def _wohnungen_context(user) -> dict: flats_view = [] for f in flats: - try: - payload = json.loads(f["payload_json"]) - except Exception: - payload = {} last = db.last_application_for_flat(uid, f["id"]) flats_view.append({ "row": f, - "payload": payload, "matched": flat_matches_filter({ "rooms": f["rooms"], "total_rent": f["total_rent"], "size": f["size"], "wbs": f["wbs"], "connectivity": {"morning_time": f["connectivity_morning_time"]}, @@ -281,22 +324,40 @@ def _wohnungen_context(user) -> dict: "last": last, }) - allowed, reason = _get_apply_gate(uid) + allowed, reason = _manual_apply_allowed() return { "flats": flats_view, "filters": filters, "auto_apply_enabled": bool(prefs["auto_apply_enabled"]), "submit_forms": bool(prefs["submit_forms"]), - "kill_switch": bool(prefs["kill_switch"]), "circuit_open": bool(prefs["apply_circuit_open"]), "apply_failures": int(prefs["apply_recent_failures"] or 0), "apply_allowed": allowed, "apply_block_reason": reason, "apply_reachable": apply_client.health(), "last_alert_heartbeat": db.get_state("last_alert_heartbeat") or "", + "next_scrape_utc": _next_scrape_utc(), + "scrape_interval_seconds": ALERT_SCRAPE_INTERVAL_SECONDS, } +@app.get("/", response_class=HTMLResponse) +def tab_wohnungen(request: Request): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + ctx = base_context(request, u, "wohnungen") + ctx.update(_wohnungen_context(u)) + return templates.TemplateResponse("wohnungen.html", ctx) + + +@app.get("/partials/wohnungen", response_class=HTMLResponse) +def partial_wohnungen(request: Request, user=Depends(require_user)): + ctx = base_context(request, user, "wohnungen") + ctx.update(_wohnungen_context(user)) + return templates.TemplateResponse("_wohnungen_body.html", ctx) + + @app.post("/actions/filters") async def action_save_filters( request: Request, @@ -342,21 +403,6 @@ async def action_auto_apply( return RedirectResponse("/", status_code=303) -@app.post("/actions/kill-switch") -async def action_kill_switch( - request: Request, - value: str = Form(...), - csrf: str = Form(...), - user=Depends(require_user), -): - require_csrf(user["id"], csrf) - new = 1 if value == "on" else 0 - db.update_preferences(user["id"], {"kill_switch": new}) - db.log_audit(user["username"], "kill_switch", "on" if new else "off", - user_id=user["id"], ip=client_ip(request)) - return RedirectResponse("/", status_code=303) - - @app.post("/actions/reset-circuit") async def action_reset_circuit( request: Request, @@ -377,7 +423,7 @@ async def action_apply( user=Depends(require_user), ): require_csrf(user["id"], csrf) - allowed, reason = _get_apply_gate(user["id"]) + allowed, reason = _manual_apply_allowed() if not allowed: raise HTTPException(409, f"apply disabled: {reason}") flat = db.get_flat(flat_id) @@ -418,6 +464,79 @@ def bewerbung_detail(request: Request, app_id: int): return templates.TemplateResponse("bewerbung_detail.html", ctx) +@app.get("/bewerbungen/{app_id}/report.zip") +def bewerbung_zip(request: Request, app_id: int): + u = current_user(request) + if not u: + raise HTTPException(401) + a = db.get_application(app_id) + if not a or (a["user_id"] != u["id"] and not u["is_admin"]): + raise HTTPException(404) + + flat = db.get_flat(a["flat_id"]) + forensics = json.loads(a["forensics_json"]) if a["forensics_json"] else {} + profile = json.loads(a["profile_snapshot_json"]) if a["profile_snapshot_json"] else {} + app_meta = {k: a[k] for k in a.keys() if k not in ("forensics_json", "profile_snapshot_json")} + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "README.txt", + f"lazyflat application report\n" + f"application_id={a['id']}\n" + f"flat_id={a['flat_id']}\n" + f"provider={a['provider']}\n" + f"success={a['success']}\n" + f"started_at_utc={a['started_at']}\n" + f"finished_at_utc={a['finished_at']}\n" + f"submit_forms_used={bool(a['submit_forms_used'])}\n" + f"\n" + f"Contents:\n" + f" application.json DB row + metadata\n" + f" flat.json Flat details at discovery time\n" + f" profile_snapshot.json Profile used for this attempt\n" + f" forensics.json Full captured forensics\n" + f" step_log.txt Human-readable step log\n" + f" page.html Final page HTML (if captured)\n" + f" console.json Browser console entries\n" + f" errors.json Browser pageerror events\n" + f" network.json Network requests + partial responses\n" + f" screenshots/*.jpg Screenshots at key moments\n" + ) + zf.writestr("application.json", json.dumps(app_meta, indent=2, default=str)) + zf.writestr("flat.json", json.dumps(dict(flat) if flat else {}, indent=2, default=str)) + zf.writestr("profile_snapshot.json", json.dumps(profile, indent=2, default=str)) + zf.writestr("forensics.json", json.dumps(forensics, indent=2, default=str)) + + step_lines = [] + for s in forensics.get("steps", []): + step_lines.append(f"[{s.get('ts', 0):7.2f}s] {s.get('step', '?'):<24} {s.get('status', ''):<5} {s.get('detail', '')}") + zf.writestr("step_log.txt", "\n".join(step_lines)) + + if forensics.get("final_html"): + zf.writestr("page.html", forensics["final_html"]) + zf.writestr("console.json", json.dumps(forensics.get("console", []), indent=2)) + zf.writestr("errors.json", json.dumps(forensics.get("errors", []), indent=2)) + zf.writestr("network.json", json.dumps(forensics.get("network", []), indent=2)) + + for idx, s in enumerate(forensics.get("screenshots", []), start=1): + b64 = s.get("b64_jpeg", "") + if b64: + try: + data = base64.b64decode(b64) + label = (s.get("label") or f"step{idx}").replace("/", "_").replace(" ", "_") + zf.writestr(f"screenshots/{idx:02d}_{label}.jpg", data) + except Exception: + pass + + buf.seek(0) + filename = f"lazyflat-report-{a['id']}.zip" + return StreamingResponse( + buf, media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + # --------------------------------------------------------------------------- # Tab: Logs # --------------------------------------------------------------------------- @@ -432,41 +551,6 @@ def tab_logs(request: Request): return templates.TemplateResponse("logs.html", ctx) -# --------------------------------------------------------------------------- -# Tab: Fehler -# --------------------------------------------------------------------------- - -@app.get("/fehler", response_class=HTMLResponse) -def tab_fehler(request: Request): - u = current_user(request) - if not u: - return RedirectResponse("/login", status_code=303) - ctx = base_context(request, u, "fehler") - ctx["errors"] = db.recent_errors(u["id"], limit=200, include_global=bool(u["is_admin"])) - return templates.TemplateResponse("fehler.html", ctx) - - -@app.get("/fehler/{err_id}", response_class=HTMLResponse) -def fehler_detail(request: Request, err_id: int): - u = current_user(request) - if not u: - return RedirectResponse("/login", status_code=303) - e = db.get_error(err_id) - if not e or (e["user_id"] is not None and e["user_id"] != u["id"] and not u["is_admin"]): - raise HTTPException(404, "not found") - app_row = db.get_application(e["application_id"]) if e["application_id"] else None - forensics = None - if app_row and app_row["forensics_json"]: - try: - forensics = json.loads(app_row["forensics_json"]) - except Exception: - forensics = None - context = json.loads(e["context_json"]) if e["context_json"] else None - ctx = base_context(request, u, "fehler") - ctx.update({"error": e, "application": app_row, "forensics": forensics, "context": context}) - return templates.TemplateResponse("fehler_detail.html", ctx) - - # --------------------------------------------------------------------------- # Tab: Einstellungen (sub-tabs) # --------------------------------------------------------------------------- @@ -595,8 +679,6 @@ async def action_submit_forms( return RedirectResponse("/einstellungen/profil", status_code=303) -# --- Admin: Benutzer --------------------------------------------------------- - @app.post("/actions/users/create") async def action_users_create( request: Request, @@ -654,7 +736,6 @@ async def internal_submit_flat( if not is_new: return {"status": "duplicate"} - # per-user matching + auto-apply + notifications for u in db.list_users(): if u["disabled"]: continue @@ -668,7 +749,7 @@ async def internal_submit_flat( notifications.on_match(u["id"], payload) prefs = db.get_preferences(u["id"]) - if prefs["auto_apply_enabled"] and not prefs["kill_switch"] and not prefs["apply_circuit_open"]: + if _auto_apply_allowed(prefs): _kick_apply(u["id"], str(payload["id"]), payload["link"], "auto") db.log_audit("system", "auto_apply_kick", f"user={u['username']} flat={payload['id']}", @@ -689,7 +770,6 @@ async def internal_report_error( payload: dict, _g: None = Depends(require_internal), ): - """Alert/other services can push errors here.""" db.log_error( source=payload.get("source", "unknown"), kind=payload.get("kind", "error"), diff --git a/web/settings.py b/web/settings.py index 04c2b81..a33d45d 100644 --- a/web/settings.py +++ b/web/settings.py @@ -37,6 +37,9 @@ APPLY_URL: str = getenv("APPLY_URL", "http://apply:8000") APPLY_TIMEOUT: int = int(getenv("APPLY_TIMEOUT", "600")) APPLY_FAILURE_THRESHOLD: int = int(getenv("APPLY_FAILURE_THRESHOLD", "3")) +# --- Alert service knob (mirrored so web can predict the next scrape) --------- +ALERT_SCRAPE_INTERVAL_SECONDS: int = int(getenv("ALERT_SCRAPE_INTERVAL_SECONDS", getenv("SLEEP_INTERVALL", "60"))) + # --- Storage ------------------------------------------------------------------ DATA_DIR: Path = Path(getenv("DATA_DIR", "/data")) DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/web/static/app.js b/web/static/app.js new file mode 100644 index 0000000..727ec5d --- /dev/null +++ b/web/static/app.js @@ -0,0 +1,49 @@ +// lazyflat — live time helpers. +// Any element with [data-rel-utc=""] gets its text replaced every 5s +// with a German relative-time string ("vor 3 min"). Elements with +// [data-countdown-utc=""] show "in Xs" counting down each second. + +function fmtRelative(iso) { + const ts = Date.parse(iso); + if (!iso || Number.isNaN(ts)) return "—"; + const diff = Math.max(0, Math.floor((Date.now() - ts) / 1000)); + if (diff < 5) return "gerade eben"; + if (diff < 60) return `vor ${diff} s`; + if (diff < 3600) return `vor ${Math.floor(diff / 60)} min`; + if (diff < 86400) return `vor ${Math.floor(diff / 3600)} h`; + return `vor ${Math.floor(diff / 86400)} Tagen`; +} + +function fmtCountdown(iso) { + const ts = Date.parse(iso); + if (!iso || Number.isNaN(ts)) return "—"; + const secs = Math.floor((ts - Date.now()) / 1000); + if (secs <= 0) return "Aktualisierung läuft…"; + if (secs < 60) return `in ${secs} s`; + if (secs < 3600) return `in ${Math.floor(secs / 60)} min`; + return `in ${Math.floor(secs / 3600)} h`; +} + +function updateRelativeTimes() { + document.querySelectorAll("[data-rel-utc]").forEach((el) => { + el.textContent = fmtRelative(el.dataset.relUtc); + }); +} + +function updateCountdowns() { + document.querySelectorAll("[data-countdown-utc]").forEach((el) => { + el.textContent = fmtCountdown(el.dataset.countdownUtc); + }); +} + +function tick() { + updateRelativeTimes(); + updateCountdowns(); +} + +// Run immediately + on intervals. Also re-run after HTMX swaps so freshly +// injected DOM gets formatted too. +document.addEventListener("DOMContentLoaded", tick); +document.body && document.body.addEventListener("htmx:afterSwap", tick); +setInterval(updateCountdowns, 1000); +setInterval(updateRelativeTimes, 5000); diff --git a/web/templates/_layout.html b/web/templates/_layout.html index ce63ba7..4bd1e60 100644 --- a/web/templates/_layout.html +++ b/web/templates/_layout.html @@ -21,7 +21,6 @@ Wohnungen Bewerbungen Logs - Fehler Einstellungen diff --git a/web/templates/_settings_profil.html b/web/templates/_settings_profil.html index faf0bdc..c915a7e 100644 --- a/web/templates/_settings_profil.html +++ b/web/templates/_settings_profil.html @@ -1,8 +1,4 @@

Bewerbungsdaten

-

- Diese Angaben werden beim Bewerben an die jeweilige Website gesendet. - sensibel — werden nur in der DB gespeichert und pro Bewerbung als Snapshot protokolliert. -

diff --git a/web/templates/_wohnungen_body.html b/web/templates/_wohnungen_body.html new file mode 100644 index 0000000..61e57aa --- /dev/null +++ b/web/templates/_wohnungen_body.html @@ -0,0 +1,178 @@ +{# Renders the full Wohnungen body; used both by the full page and by HTMX poll. #} +
+ + +
+
+
Auto-Bewerben
+
+ {% if auto_apply_enabled %} + aktiviert + bei Match wird automatisch beworben + {% else %} + deaktiviert + Matches werden nur angezeigt + {% endif %} +
+
+ +
+ + + + + + + {% if circuit_open %} +
+ + +
+ {% endif %} +
+
+ +{% if not apply_allowed %} +
+ apply blockiert + {{ apply_block_reason }} +
+{% endif %} + + +
+
+
alert
+
+ {% if last_alert_heartbeat %}live + {% else %}kein Heartbeat{% endif %} +
+
+
+
apply
+
+ {% if apply_reachable %}ok + {% else %}down{% endif %} +
+
+
+
submit_forms
+
+ {% if submit_forms %}echt senden + {% else %}dry-run{% endif %} +
+
+
+
Fehler in Serie
+
+ {% if circuit_open %}circuit open + {% elif apply_failures > 0 %}{{ apply_failures }} + {% else %}0{% endif %} +
+
+
+ + +
+ Eigene Filter +
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + Leer lassen = kein Limit. Filter bestimmen Match-Hervorhebung + Auto-Bewerben. +
+
+
+ + +
+
+

Neueste Wohnungen auf inberlinwohnen.de

+
+ {{ flats|length }} gesehen + {% if next_scrape_utc %} + · nächste Aktualisierung … + {% else %} + · warte auf alert… + {% endif %} +
+
+
+ {% for item in flats %} + {% set f = item.row %} +
+
+
+ + {{ f.address or f.link }} + + {% if item.matched %}match{% endif %} + {% if item.last and item.last.success == 1 %}beworben + {% elif item.last and item.last.success == 0 %}apply fehlgeschlagen + {% elif item.last %}läuft{% endif %} +
+
+ {% if f.rooms %}{{ "%.1f"|format(f.rooms) }} Z{% endif %} + {% if f.size %} · {{ "%.0f"|format(f.size) }} m²{% endif %} + {% if f.total_rent %} · {{ "%.0f"|format(f.total_rent) }} €{% endif %} + {% if f.sqm_price %} ({{ "%.2f"|format(f.sqm_price) }} €/m²){% endif %} + {% if f.connectivity_morning_time %} · {{ "%.0f"|format(f.connectivity_morning_time) }} min morgens{% endif %} + {% if f.wbs %} · WBS: {{ f.wbs }}{% endif %} + · entdeckt … +
+ {% if item.last and item.last.message %} +
↳ {{ item.last.message }}
+ {% endif %} +
+
+ {% if apply_allowed and not (item.last and item.last.success == 1) %} +
+ + + +
+ {% endif %} +
+
+ {% else %} +
Noch keine Wohnungen entdeckt.
+ {% endfor %} +
+
+ +
diff --git a/web/templates/base.html b/web/templates/base.html index 36b7b01..3a0c1f1 100644 --- a/web/templates/base.html +++ b/web/templates/base.html @@ -7,6 +7,7 @@ {% block title %}lazyflat{% endblock %} +