"""
wohnungsdidi web app.

Tabs:
- /                           → Wohnungen
- /bewerbungen                → Bewerbungen (history + forensics ZIP for failed runs)
- /einstellungen/<section>    → Einstellungen: profil | filter | benachrichtigungen | account
- /admin/<section>            → Admin-only: protokoll | benutzer

All state-changing POSTs require CSRF. Internal endpoints require INTERNAL_API_KEY.
"""
import asyncio
import base64
import hmac
import io
import json
import logging
import mimetypes
import sqlite3
import zipfile
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from typing import Any

from fastapi import Depends, FastAPI, Form, Header, HTTPException, Request, Response, status
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates

try:
    from zoneinfo import ZoneInfo
    BERLIN_TZ = ZoneInfo("Europe/Berlin")
except Exception:
    BERLIN_TZ = timezone.utc

import db
import enrichment
import notifications
import retention
from apply_client import ApplyClient, _row_to_profile
from auth import (
    bootstrap_admin,
    clear_session_cookie,
    current_user,
    hash_password,
    issue_csrf_token,
    issue_session_cookie,
    rate_limit_login,
    require_admin,
    require_csrf,
    require_user,
    verify_login,
)
from matching import flat_matches_filter, row_to_dict
from settings import (
    ALERT_SCRAPE_INTERVAL_SECONDS,
    APPLY_FAILURE_THRESHOLD,
    INTERNAL_API_KEY,
    PUBLIC_URL,
)


logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s %(levelname)-5s %(name)s: %(message)s",
    datefmt="%H:%M:%S",
)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logger = logging.getLogger("web")

apply_client = ApplyClient()


# ---------------------------------------------------------------------------
# App + Jinja
# ---------------------------------------------------------------------------

@asynccontextmanager
async def lifespan(_app: FastAPI):
    db.init_db()
    db.seed_secrets_from_env()
    bootstrap_admin()
    retention.start()
    logger.info("web service ready")
    yield


app = FastAPI(lifespan=lifespan, title="wohnungsdidi", docs_url=None, redoc_url=None, openapi_url=None)
app.mount("/static", StaticFiles(directory="static"), name="static")
templates = Jinja2Templates(directory="templates")


def _parse_iso(s: str | None) -> datetime | None:
    if not s:
        return None
    try:
        # Python handles +00:00 but not trailing Z. Storage always uses +00:00.
        return datetime.fromisoformat(s)
    except ValueError:
        return None


def _de_dt(s: str | None) -> str:
    """UTC ISO → 'DD.MM.YYYY HH:MM' in Europe/Berlin."""
    dt = _parse_iso(s)
    if dt is None:
        return ""
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(BERLIN_TZ).strftime("%d.%m.%Y %H:%M")


def _iso_utc(s: str | None) -> str:
    """Pass-through of a UTC ISO string (for data-attr in JS)."""
    dt = _parse_iso(s)
    if dt is None:
        return ""
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc).isoformat(timespec="seconds")


templates.env.filters["de_dt"] = _de_dt
templates.env.filters["iso_utc"] = _iso_utc
templates.env.filters["flat_slug"] = lambda s: enrichment.flat_slug(str(s or ""))


@app.middleware("http")
async def security_headers(request: Request, call_next):
    resp: Response = await call_next(request)
    resp.headers.setdefault("X-Frame-Options", "DENY")
    resp.headers.setdefault("X-Content-Type-Options", "nosniff")
    resp.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
    resp.headers.setdefault("Permissions-Policy", "geolocation=(), camera=(), microphone=()")
    resp.headers.setdefault(
        "Content-Security-Policy",
        "default-src 'self'; "
        "script-src 'self' https://cdn.tailwindcss.com https://unpkg.com; "
        "style-src 'self' https://cdn.tailwindcss.com https://unpkg.com 'unsafe-inline'; "
        "img-src 'self' data: "
        "https://*.basemaps.cartocdn.com https://basemaps.cartocdn.com "
        "https://unpkg.com; "
        "connect-src 'self'; frame-ancestors 'none';"
    )
    return resp


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def client_ip(request: Request) -> str:
    xff = request.headers.get("x-forwarded-for")
    if xff:
        return xff.split(",")[0].strip()
    return request.client.host if request.client else "unknown"


def require_internal(x_internal_api_key: str | None = Header(default=None)) -> None:
    if not x_internal_api_key or not hmac.compare_digest(x_internal_api_key, INTERNAL_API_KEY):
        raise HTTPException(status_code=401, detail="invalid internal key")


def base_context(request: Request, user, active_tab: str) -> dict:
    return {
        "request": request,
        "user": user,
        "csrf": issue_csrf_token(user["id"]),
        "active_tab": active_tab,
        "is_admin": bool(user["is_admin"]),
    }


def _is_htmx(request: Request) -> bool:
    return request.headers.get("hx-request", "").lower() == "true"


def _wohnungen_partial_or_redirect(request: Request, user):
    """If called via HTMX, render the body partial; otherwise redirect to /."""
    if _is_htmx(request):
        ctx = base_context(request, user, "wohnungen")
        ctx.update(_wohnungen_context(user))
        return templates.TemplateResponse("_wohnungen_body.html", ctx)
    return RedirectResponse("/", status_code=303)


def _manual_apply_allowed() -> tuple[bool, str]:
    """Manual 'Bewerben' button is only blocked if the apply service is down."""
    if not apply_client.health():
        return False, "apply-service nicht erreichbar"
    return True, ""


def _auto_apply_allowed(prefs) -> bool:
    """Auto-apply trigger gate: user must have enabled it and the circuit must be closed."""
    if not prefs["auto_apply_enabled"]:
        return False
    if prefs["apply_circuit_open"]:
        return False
    return apply_client.health()


def _next_scrape_utc() -> str:
    hb = db.get_state("last_alert_heartbeat")
    dt = _parse_iso(hb)
    if dt is None:
        return ""
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return (dt + timedelta(seconds=ALERT_SCRAPE_INTERVAL_SECONDS)).astimezone(timezone.utc).isoformat(timespec="seconds")


def _last_scrape_utc() -> str:
    hb = db.get_state("last_alert_heartbeat")
    dt = _parse_iso(hb)
    if dt is None:
        return ""
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc).isoformat(timespec="seconds")


FILTER_KEYS = ("rooms_min", "rooms_max", "max_rent", "min_size",
               "wbs_required", "max_age_hours")


def _has_filters(f) -> bool:
    if not f:
        return False
    for k in FILTER_KEYS:
        v = f[k] if hasattr(f, "keys") else None
        if v not in (None, "", 0, 0.0):
            return True
    return False


def _alert_status(notifications_row) -> tuple[str, str]:
    """Return (label, chip_kind) for the user's alarm (notification) setup.

    'aktiv' only if a real push channel is configured with credentials.
    'ui' is not a real alarm — the dashboard already shows matches when you
    happen to be looking.
    """
    if not notifications_row:
        return "nicht eingerichtet", "warn"
    ch = (notifications_row["channel"] or "ui").strip()
    if ch == "telegram":
        if notifications_row["telegram_bot_token"] and notifications_row["telegram_chat_id"]:
            return "aktiv (Telegram)", "ok"
        return "unvollständig", "warn"
    return "nicht eingerichtet", "warn"


def _filter_summary(f) -> str:
    if not _has_filters(f):
        return "—"
    parts = []
    rmin, rmax = f["rooms_min"], f["rooms_max"]
    if rmin or rmax:
        def fmt(x):
            return "–" if x is None else ("%g" % x)
        parts.append(f"{fmt(rmin)}–{fmt(rmax)} Zi")
    if f["max_rent"]:
        parts.append(f"≤ {int(f['max_rent'])} €")
    if f["min_size"]:
        parts.append(f"≥ {int(f['min_size'])} m²")
    if f["wbs_required"] == "yes":
        parts.append("WBS")
    elif f["wbs_required"] == "no":
        parts.append("ohne WBS")
    if f["max_age_hours"]:
        parts.append(f"≤ {int(f['max_age_hours'])} h alt")
    return " · ".join(parts)


def _has_running_application(user_id: int) -> bool:
    return db.has_running_application(user_id)


def _finish_apply_background(app_id: int, user_id: int, flat_id: str, url: str,
                             profile: dict, submit_forms: bool) -> None:
    """Called on a worker thread AFTER the application row already exists.
    The HTMX response has already shipped the running state to the user."""
    logger.info("apply.running user=%s flat=%s application=%s submit=%s",
                user_id, flat_id, app_id, submit_forms)
    result = apply_client.apply(url=url, profile=profile,
                                submit_forms=submit_forms, application_id=app_id)
    success = bool(result.get("success"))
    message = result.get("message", "")
    provider = result.get("provider", "")
    forensics = result.get("forensics") or {}

    db.finish_application(app_id, success=success, message=message,
                          provider=provider, forensics=forensics)

    prefs = db.get_preferences(user_id)
    if success:
        db.update_preferences(user_id, {"apply_recent_failures": 0})
    else:
        failures = int(prefs["apply_recent_failures"] or 0) + 1
        updates = {"apply_recent_failures": failures}
        if failures >= APPLY_FAILURE_THRESHOLD:
            updates["apply_circuit_open"] = 1
            db.log_error(source="apply", kind="circuit_open", user_id=user_id,
                         summary=f"{failures} aufeinanderfolgende Fehler",
                         application_id=app_id)
        db.update_preferences(user_id, updates)
        db.log_error(source="apply", kind="apply_failure", user_id=user_id,
                     summary=message or "Bewerbung fehlgeschlagen",
                     application_id=app_id,
                     context={"provider": provider, "url": url})

    flat = db.get_flat(flat_id)
    flat_dict = {"address": flat["address"] if flat else "", "link": url,
                 "rooms": flat["rooms"] if flat else None,
                 "total_rent": flat["total_rent"] if flat else None}
    if success:
        notifications.on_apply_ok(user_id, flat_dict, message)
    else:
        notifications.on_apply_fail(user_id, flat_dict, message)

    db.log_audit("system", "apply_finished", f"app={app_id} success={success}", user_id=user_id)


def _kick_apply(user_id: int, flat_id: str, url: str, triggered_by: str) -> None:
    """Insert the application row synchronously so the immediate HTMX response
    already renders the row's "läuft…" state. The long-running Playwright call
    is then offloaded to a background thread."""
    prefs = db.get_preferences(user_id)
    profile_row = db.get_profile(user_id)
    profile = _row_to_profile(profile_row)
    submit_forms = bool(prefs["submit_forms"])

    app_id = db.start_application(
        user_id=user_id, flat_id=flat_id, url=url,
        triggered_by=triggered_by, submit_forms=submit_forms,
        profile_snapshot=profile,
    )

    asyncio.create_task(asyncio.to_thread(
        _finish_apply_background, app_id, user_id, flat_id, url, profile, submit_forms,
    ))


# ---------------------------------------------------------------------------
# Public routes
# ---------------------------------------------------------------------------

@app.get("/health")
def health():
    return {"status": "ok"}


@app.get("/login", response_class=HTMLResponse)
def login_form(request: Request, error: str | None = None):
    if current_user(request):
        return RedirectResponse("/", status_code=303)
    return templates.TemplateResponse("login.html", {"request": request, "error": error})


@app.post("/login")
def login_submit(request: Request, username: str = Form(...), password: str = Form(...)):
    ip = client_ip(request)
    if not rate_limit_login(ip):
        db.log_audit(username or "?", "login_rate_limited", ip=ip)
        db.log_error(source="web", kind="rate_limit", summary=f"login throttled for {ip}",
                     context={"username": username or ""})
        return templates.TemplateResponse(
            "login.html",
            {"request": request, "error": "Zu viele Versuche. Bitte später erneut."},
            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
        )
    user = verify_login(username, password)
    if not user:
        db.log_audit(username or "?", "login_failed", ip=ip)
        return templates.TemplateResponse(
            "login.html",
            {"request": request, "error": "Login fehlgeschlagen."},
            status_code=status.HTTP_401_UNAUTHORIZED,
        )
    response = RedirectResponse("/", status_code=303)
    issue_session_cookie(response, user["id"])
    db.log_audit(user["username"], "login_success", user_id=user["id"], ip=ip)
    return response


@app.post("/logout")
def logout(request: Request):
    u = current_user(request)
    response = RedirectResponse("/login", status_code=303)
    clear_session_cookie(response)
    if u:
        db.log_audit(u["username"], "logout", user_id=u["id"], ip=client_ip(request))
    return response


# ---------------------------------------------------------------------------
# Tab: Wohnungen
# ---------------------------------------------------------------------------

def _wohnungen_context(user) -> dict:
    uid = user["id"]
    filters_row = db.get_filters(uid)
    notif_row = db.get_notifications(uid)
    prefs = db.get_preferences(uid)
    filters = row_to_dict(filters_row)
    flats = db.recent_flats(100)

    rejected = db.rejected_flat_ids(uid)
    max_age_hours = filters_row["max_age_hours"] if filters_row else None
    age_cutoff = None
    if max_age_hours:
        age_cutoff = datetime.now(timezone.utc) - timedelta(hours=int(max_age_hours))
    flats_view = []
    for f in flats:
        if f["id"] in rejected:
            continue
        if age_cutoff is not None:
            disc = _parse_iso(f["discovered_at"])
            if disc is None:
                continue
            if disc.tzinfo is None:
                disc = disc.replace(tzinfo=timezone.utc)
            if disc < age_cutoff:
                continue
        if not flat_matches_filter({
            "rooms": f["rooms"], "total_rent": f["total_rent"], "size": f["size"],
            "wbs": f["wbs"], "connectivity": {"morning_time": f["connectivity_morning_time"]},
        }, filters):
            continue
        last = db.last_application_for_flat(uid, f["id"])
        flats_view.append({"row": f, "last": last})

    rejected_view = db.rejected_flats(uid)
    enrichment_counts = db.enrichment_counts()

    partner = db.get_partner_user(uid)
    partner_info = None
    if partner:
        partner_profile = db.get_profile(partner["id"])
        initial = ((partner_profile["firstname"] if partner_profile else "")
                   or partner["username"] or "?")[:1].upper()
        display_name = (partner_profile["firstname"]
                        if partner_profile and partner_profile["firstname"]
                        else partner["username"])
        actions = db.partner_flat_actions(partner["id"])
        partner_info = {
            "initial": initial,
            "name": display_name,
            "applied_flat_ids": actions["applied"],
            "rejected_flat_ids": actions["rejected"],
        }

    allowed, reason = _manual_apply_allowed()
    alert_label, alert_chip = _alert_status(notif_row)
    has_running = _has_running_application(uid)
    map_points = []
    for item in flats_view:
        f = item["row"]
        if f["lat"] is None or f["lng"] is None:
            continue
        last = item["last"]
        is_running = bool(last and last["finished_at"] is None)
        already_applied = bool(last and last["success"] == 1)
        if is_running:
            status = {"label": "läuft…", "chip": "warn"}
        elif already_applied:
            status = {"label": "beworben", "chip": "ok"}
        elif last and last["success"] == 0:
            status = {"label": "fehlgeschlagen", "chip": "bad"}
        else:
            status = None
        map_points.append({
            "id": f["id"],
            "lat": f["lat"], "lng": f["lng"],
            "address": f["address"] or f["link"],
            "link": f["link"],
            "rent": f["total_rent"],
            "rooms": f["rooms"],
            "size": f["size"],
            "status": status,
            "can_apply": allowed and not already_applied,
            "is_running": is_running,
        })
    return {
        "flats": flats_view,
        "rejected_flats": rejected_view,
        "enrichment_counts": enrichment_counts,
        "partner": partner_info,
        "map_points": map_points,
        "has_filters": _has_filters(filters_row),
        "alert_label": alert_label,
        "alert_chip": alert_chip,
        "filter_summary": _filter_summary(filters_row),
        "auto_apply_enabled": bool(prefs["auto_apply_enabled"]),
        "submit_forms": bool(prefs["submit_forms"]),
        "circuit_open": bool(prefs["apply_circuit_open"]),
        "apply_failures": int(prefs["apply_recent_failures"] or 0),
        "apply_allowed": allowed,
        "apply_block_reason": reason,
        "apply_reachable": apply_client.health(),
        "next_scrape_utc": _next_scrape_utc(),
        "last_scrape_utc": _last_scrape_utc(),
        "has_running_apply": has_running,
        "poll_interval": 3 if has_running else 30,
    }


@app.get("/", response_class=HTMLResponse)
def tab_wohnungen(request: Request):
    u = current_user(request)
    if not u:
        return RedirectResponse("/login", status_code=303)
    ctx = base_context(request, u, "wohnungen")
    ctx.update(_wohnungen_context(u))
    return templates.TemplateResponse("wohnungen.html", ctx)


@app.get("/partials/wohnungen", response_class=HTMLResponse)
def partial_wohnungen(request: Request, user=Depends(require_user)):
    ctx = base_context(request, user, "wohnungen")
    ctx.update(_wohnungen_context(user))
    return templates.TemplateResponse("_wohnungen_body.html", ctx)


@app.get("/partials/wohnung/{flat_id:path}", response_class=HTMLResponse)
def partial_wohnung_detail(request: Request, flat_id: str, user=Depends(require_user)):
    flat = db.get_flat(flat_id)
    if not flat:
        raise HTTPException(404)
    slug = enrichment.flat_slug(flat_id)
    image_urls = [
        f"/flat-images/{slug}/{i}"
        for i in range(1, int(flat["image_count"] or 0) + 1)
    ]
    ctx = {
        "request": request,
        "flat": flat,
        "enrichment_status": flat["enrichment_status"],
        "image_urls": image_urls,
    }
    return templates.TemplateResponse("_wohnung_detail.html", ctx)


@app.get("/flat-images/{slug}/{index}")
def flat_image(slug: str, index: int):
    """Serve a downloaded flat image by slug + 1-based index.

    `slug` is derived from enrichment.flat_slug(flat_id) and is filesystem-safe
    (hex), so it can be composed into a path without sanitisation concerns."""
    if not slug.isalnum() or not 1 <= index <= 99:
        raise HTTPException(404)
    d = enrichment.IMAGES_DIR / slug
    if not d.exists():
        raise HTTPException(404)
    # Files are named NN.<ext>; try the usual extensions.
    prefix = f"{index:02d}."
    for f in d.iterdir():
        if f.name.startswith(prefix):
            media = mimetypes.guess_type(f.name)[0] or "image/jpeg"
            return Response(content=f.read_bytes(), media_type=media,
                            headers={"Cache-Control": "public, max-age=3600"})
    raise HTTPException(404)


@app.post("/actions/filters")
async def action_save_filters(
    request: Request,
    csrf: str = Form(...),
    rooms_min: str = Form(""),
    rooms_max: str = Form(""),
    max_rent: str = Form(""),
    min_size: str = Form(""),
    wbs_required: str = Form(""),
    max_age_hours: str = Form(""),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)

    def _f(v):
        v = (v or "").strip().replace(",", ".")
        return float(v) if v else None

    def _i(v):
        v = (v or "").strip()
        try:
            return int(v) if v else None
        except ValueError:
            return None

    db.update_filters(user["id"], {
        "rooms_min": _f(rooms_min),
        "rooms_max": _f(rooms_max),
        "max_rent": _f(max_rent),
        "min_size": _f(min_size),
        "wbs_required": (wbs_required or "").strip(),
        "max_age_hours": _i(max_age_hours),
    })
    db.log_audit(user["username"], "filters.updated", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/", status_code=303)


@app.post("/actions/auto-apply")
async def action_auto_apply(
    request: Request,
    value: str = Form(default="off"),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    new = 1 if value == "on" else 0
    db.update_preferences(user["id"], {"auto_apply_enabled": new})
    db.log_audit(user["username"], "auto_apply", "on" if new else "off",
                 user_id=user["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, user)


@app.post("/actions/reset-circuit")
async def action_reset_circuit(
    request: Request,
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    db.update_preferences(user["id"], {"apply_circuit_open": 0, "apply_recent_failures": 0})
    db.log_audit(user["username"], "reset_circuit", user_id=user["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, user)


@app.post("/actions/apply")
async def action_apply(
    request: Request,
    flat_id: str = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    allowed, reason = _manual_apply_allowed()
    if not allowed:
        raise HTTPException(409, f"apply disabled: {reason}")
    flat = db.get_flat(flat_id)
    if not flat:
        raise HTTPException(404, "flat not found")
    last = db.last_application_for_flat(user["id"], flat_id)
    if last and last["finished_at"] is None:
        # Another apply is already running for this user+flat; don't queue a second.
        return _wohnungen_partial_or_redirect(request, user)
    if last and last["success"] == 1:
        # Already successfully applied — no point in re-running.
        return _wohnungen_partial_or_redirect(request, user)
    db.log_audit(user["username"], "trigger_apply", f"flat_id={flat_id}",
                 user_id=user["id"], ip=client_ip(request))
    _kick_apply(user["id"], flat_id, flat["link"], "user")
    return _wohnungen_partial_or_redirect(request, user)


@app.post("/actions/reject")
async def action_reject(
    request: Request,
    flat_id: str = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    db.reject_flat(user["id"], flat_id)
    db.log_audit(user["username"], "flat.rejected", f"flat_id={flat_id}",
                 user_id=user["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, user)


@app.post("/actions/unreject")
async def action_unreject(
    request: Request,
    flat_id: str = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    db.unreject_flat(user["id"], flat_id)
    db.log_audit(user["username"], "flat.unrejected", f"flat_id={flat_id}",
                 user_id=user["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, user)


# ---------------------------------------------------------------------------
# Tab: Bewerbungen
# ---------------------------------------------------------------------------

@app.get("/bewerbungen", response_class=HTMLResponse)
def tab_bewerbungen(request: Request):
    u = current_user(request)
    if not u:
        return RedirectResponse("/login", status_code=303)
    ctx = base_context(request, u, "bewerbungen")
    ctx["applications"] = db.recent_applications(u["id"], limit=100)
    return templates.TemplateResponse("bewerbungen.html", ctx)


@app.get("/bewerbungen/{app_id}/report.zip")
def bewerbung_zip(request: Request, app_id: int):
    u = current_user(request)
    if not u:
        raise HTTPException(401)
    a = db.get_application(app_id)
    if not a or (a["user_id"] != u["id"] and not u["is_admin"]):
        raise HTTPException(404)

    flat = db.get_flat(a["flat_id"])
    forensics = json.loads(a["forensics_json"]) if a["forensics_json"] else {}
    profile = json.loads(a["profile_snapshot_json"]) if a["profile_snapshot_json"] else {}
    app_meta = {k: a[k] for k in a.keys() if k not in ("forensics_json", "profile_snapshot_json")}

    buf = io.BytesIO()
    with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
        zf.writestr(
            "README.txt",
            f"wohnungsdidi application report\n"
            f"application_id={a['id']}\n"
            f"flat_id={a['flat_id']}\n"
            f"provider={a['provider']}\n"
            f"success={a['success']}\n"
            f"started_at_utc={a['started_at']}\n"
            f"finished_at_utc={a['finished_at']}\n"
            f"submit_forms_used={bool(a['submit_forms_used'])}\n"
            f"\n"
            f"Contents:\n"
            f"  application.json     DB row + metadata\n"
            f"  flat.json            Flat details at discovery time\n"
            f"  profile_snapshot.json Profile used for this attempt\n"
            f"  forensics.json       Full captured forensics\n"
            f"  step_log.txt         Human-readable step log\n"
            f"  page.html            Final page HTML (if captured)\n"
            f"  console.json         Browser console entries\n"
            f"  errors.json          Browser pageerror events\n"
            f"  network.json         Network requests + partial responses\n"
            f"  snapshots/NN_*.jpg   Screenshot at each step (NN = order)\n"
            f"  snapshots/NN_*.html  Page HTML at each step\n"
        )
        zf.writestr("application.json", json.dumps(app_meta, indent=2, default=str))
        zf.writestr("flat.json", json.dumps(dict(flat) if flat else {}, indent=2, default=str))
        zf.writestr("profile_snapshot.json", json.dumps(profile, indent=2, default=str))
        zf.writestr("forensics.json", json.dumps(forensics, indent=2, default=str))

        step_lines = []
        for s in forensics.get("steps", []):
            step_lines.append(f"[{s.get('ts', 0):7.2f}s] {s.get('step', '?'):<24} {s.get('status', ''):<5} {s.get('detail', '')}")
        zf.writestr("step_log.txt", "\n".join(step_lines))

        if forensics.get("final_html"):
            zf.writestr("page.html", forensics["final_html"])
        zf.writestr("console.json", json.dumps(forensics.get("console", []), indent=2))
        zf.writestr("errors.json", json.dumps(forensics.get("errors", []), indent=2))
        zf.writestr("network.json", json.dumps(forensics.get("network", []), indent=2))

        for idx, s in enumerate(forensics.get("screenshots", []), start=1):
            label = (s.get("label") or f"step{idx}").replace("/", "_").replace(" ", "_")
            b64 = s.get("b64_jpeg", "")
            if b64:
                try:
                    data = base64.b64decode(b64)
                    zf.writestr(f"snapshots/{idx:02d}_{label}.jpg", data)
                except Exception:
                    pass
            html = s.get("html") or ""
            if html:
                zf.writestr(f"snapshots/{idx:02d}_{label}.html", html)

    buf.seek(0)
    filename = f"wohnungsdidi-report-{a['id']}.zip"
    return StreamingResponse(
        buf, media_type="application/zip",
        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
    )


# ---------------------------------------------------------------------------
# Tab: Logs
# ---------------------------------------------------------------------------

def _parse_date_range(from_str: str | None, to_str: str | None) -> tuple[str | None, str | None]:
    """Parse 'YYYY-MM-DD' local-Berlin date inputs into UTC ISO bounds.
    Bounds are inclusive start-of-day and start-of-next-day."""
    def _to_utc_iso(s: str, end_of_day: bool) -> str | None:
        try:
            d = datetime.strptime(s, "%Y-%m-%d").replace(tzinfo=BERLIN_TZ)
        except ValueError:
            return None
        if end_of_day:
            d = d + timedelta(days=1)
        return d.astimezone(timezone.utc).isoformat(timespec="seconds")

    start = _to_utc_iso(from_str, False) if from_str else None
    end = _to_utc_iso(to_str, True) if to_str else None
    return start, end


def _collect_events(start_iso: str | None, end_iso: str | None) -> list[dict]:
    users = {row["id"]: row["username"] for row in db.list_users()}
    events: list[dict] = []
    for a in db.recent_audit(None, limit=5000):
        if start_iso and a["timestamp"] < start_iso: continue
        if end_iso and a["timestamp"] >= end_iso: continue
        events.append({
            "kind": "audit", "ts": a["timestamp"], "source": "web",
            "actor": a["actor"], "action": a["action"],
            "details": a["details"] or "",
            "user": users.get(a["user_id"], ""),
            "ip": a["ip"] or "",
        })
    for e in db.recent_errors(None, limit=5000):
        if start_iso and e["timestamp"] < start_iso: continue
        if end_iso and e["timestamp"] >= end_iso: continue
        events.append({
            "kind": "error", "ts": e["timestamp"], "source": e["source"],
            "actor": e["source"], "action": e["kind"],
            "details": e["summary"] or "",
            "user": users.get(e["user_id"], "") if e["user_id"] else "",
            "ip": "",
        })
    events.sort(key=lambda x: x["ts"], reverse=True)
    return events


@app.get("/logs")
def tab_logs_legacy():
    # Old top-level Protokoll tab was merged into /admin/protokoll.
    return RedirectResponse("/admin/protokoll", status_code=301)


ADMIN_SECTIONS = ("protokoll", "benutzer", "geheimnisse")


def _mask_secret(value: str) -> str:
    if not value:
        return ""
    if len(value) <= 10:
        return "•" * len(value)
    return value[:6] + "…" + value[-4:]


@app.get("/admin", response_class=HTMLResponse)
def tab_admin_root(request: Request):
    return RedirectResponse("/admin/protokoll", status_code=303)


@app.get("/admin/{section}", response_class=HTMLResponse)
def tab_admin(request: Request, section: str):
    u = current_user(request)
    if not u:
        return RedirectResponse("/login", status_code=303)
    if not u["is_admin"]:
        raise HTTPException(403, "admin only")
    if section not in ADMIN_SECTIONS:
        raise HTTPException(404)

    ctx = base_context(request, u, "admin")
    ctx["section"] = section

    if section == "protokoll":
        q = request.query_params
        from_str = q.get("from") or ""
        to_str = q.get("to") or ""
        start_iso, end_iso = _parse_date_range(from_str or None, to_str or None)
        ctx.update({
            "events": _collect_events(start_iso, end_iso)[:500],
            "from_str": from_str, "to_str": to_str,
        })
    elif section == "benutzer":
        ctx["users"] = db.list_users()
    elif section == "geheimnisse":
        secrets = db.all_secrets()
        ctx["secrets_masked"] = {k: _mask_secret(secrets.get(k, "")) for k in db.SECRET_KEYS}
        ctx["secret_flash"] = request.query_params.get("ok")
    return templates.TemplateResponse("admin.html", ctx)


@app.get("/logs/export.csv")
def tab_logs_export(request: Request):
    u = current_user(request)
    if not u:
        raise HTTPException(401)
    if not u["is_admin"]:
        raise HTTPException(403)

    import csv as _csv
    q = request.query_params
    start_iso, end_iso = _parse_date_range(q.get("from") or None, q.get("to") or None)
    events = _collect_events(start_iso, end_iso)

    buf = io.StringIO()
    w = _csv.writer(buf, delimiter=",", quoting=_csv.QUOTE_MINIMAL)
    w.writerow(["timestamp_utc", "timestamp_berlin", "kind", "source", "actor", "action", "user", "details", "ip"])
    for e in events:
        w.writerow([
            e["ts"],
            _de_dt(e["ts"]),
            e["kind"],
            e["source"],
            e["actor"],
            e["action"],
            e["user"],
            e["details"],
            e["ip"],
        ])
    body = buf.getvalue().encode("utf-8")
    filename = "wohnungsdidi-protokoll"
    if q.get("from"): filename += f"-{q['from']}"
    if q.get("to"):   filename += f"-bis-{q['to']}"
    filename += ".csv"
    return Response(
        content=body, media_type="text/csv; charset=utf-8",
        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
    )


# ---------------------------------------------------------------------------
# Tab: Einstellungen (sub-tabs)
# ---------------------------------------------------------------------------

VALID_SECTIONS = ("profil", "filter", "benachrichtigungen", "partner", "account")


@app.get("/einstellungen", response_class=HTMLResponse)
def tab_settings_root(request: Request):
    return RedirectResponse("/einstellungen/profil", status_code=303)


@app.get("/einstellungen/{section}", response_class=HTMLResponse)
def tab_settings(request: Request, section: str):
    u = current_user(request)
    if not u:
        return RedirectResponse("/login", status_code=303)
    # Benutzer verwaltung lives under /admin/benutzer since the admin tab rework.
    if section == "benutzer":
        return RedirectResponse("/admin/benutzer", status_code=301)
    if section not in VALID_SECTIONS:
        raise HTTPException(404)

    ctx = base_context(request, u, "einstellungen")
    ctx["section"] = section

    if section == "profil":
        ctx["profile"] = db.get_profile(u["id"])
    elif section == "filter":
        ctx["filters"] = row_to_dict(db.get_filters(u["id"]))
    elif section == "benachrichtigungen":
        ctx["notifications"] = db.get_notifications(u["id"])
    elif section == "partner":
        ctx["partner"] = db.get_partner_user(u["id"])
        ctx["partner_profile"] = db.get_profile(ctx["partner"]["id"]) if ctx["partner"] else None
        ctx["incoming_requests"] = db.partnership_incoming(u["id"])
        ctx["outgoing_requests"] = db.partnership_outgoing(u["id"])
        ctx["partner_flash"] = request.query_params.get("flash") or ""
    return templates.TemplateResponse("einstellungen.html", ctx)


@app.post("/actions/profile")
async def action_profile(request: Request, user=Depends(require_user)):
    form = await request.form()
    require_csrf(user["id"], form.get("csrf", ""))

    def _b(name): return form.get(name, "").lower() in ("true", "on", "yes", "1")
    def _i(name):
        try: return int(form.get(name) or 0)
        except ValueError: return 0

    # Field names are intentionally opaque ("contact_addr", "immomio_login",
    # "immomio_secret") to keep password managers — specifically Bitwarden —
    # from recognising the form as a login/identity form and autofilling.
    db.update_profile(user["id"], {
        "salutation": form.get("salutation", ""),
        "firstname": form.get("firstname", ""),
        "lastname": form.get("lastname", ""),
        "email": form.get("contact_addr", ""),
        "telephone": form.get("telephone", ""),
        "street": form.get("street", ""),
        "house_number": form.get("house_number", ""),
        "postcode": form.get("postcode", ""),
        "city": form.get("city", ""),
        "is_possessing_wbs": 1 if _b("is_possessing_wbs") else 0,
        "wbs_type": form.get("wbs_type", "0"),
        "wbs_valid_till": form.get("wbs_valid_till", "1970-01-01"),
        "wbs_rooms": _i("wbs_rooms"),
        "wbs_adults": _i("wbs_adults"),
        "wbs_children": _i("wbs_children"),
        "is_prio_wbs": 1 if _b("is_prio_wbs") else 0,
        "immomio_email": form.get("immomio_login", ""),
        "immomio_password": form.get("immomio_secret", ""),
    })
    db.log_audit(user["username"], "profile.updated", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/profil", status_code=303)


@app.post("/actions/partner/request")
async def action_partner_request(
    request: Request,
    partner_username: str = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    target = db.get_user_by_username((partner_username or "").strip())
    if not target or target["id"] == user["id"]:
        return RedirectResponse("/einstellungen/partner?flash=nouser", status_code=303)
    req_id = db.partnership_request(user["id"], target["id"])
    if req_id is None:
        return RedirectResponse("/einstellungen/partner?flash=exists", status_code=303)
    db.log_audit(user["username"], "partner.requested",
                 f"target={target['username']}", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/partner?flash=sent", status_code=303)


@app.post("/actions/partner/accept")
async def action_partner_accept(
    request: Request,
    request_id: int = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    if not db.partnership_accept(request_id, user["id"]):
        return RedirectResponse("/einstellungen/partner?flash=accept_failed", status_code=303)
    db.log_audit(user["username"], "partner.accepted",
                 f"request={request_id}", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/partner?flash=accepted", status_code=303)


@app.post("/actions/partner/decline")
async def action_partner_decline(
    request: Request,
    request_id: int = Form(...),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    db.partnership_decline(request_id, user["id"])
    db.log_audit(user["username"], "partner.declined",
                 f"request={request_id}", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/partner?flash=declined", status_code=303)


@app.post("/actions/partner/unlink")
async def action_partner_unlink(
    request: Request,
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    db.partnership_unlink(user["id"])
    db.log_audit(user["username"], "partner.unlinked", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/partner?flash=unlinked", status_code=303)


@app.post("/actions/notifications")
async def action_notifications(request: Request, user=Depends(require_user)):
    form = await request.form()
    require_csrf(user["id"], form.get("csrf", ""))
    def _b(n): return 1 if form.get(n, "").lower() in ("on", "true", "1", "yes") else 0
    channel = form.get("channel", "ui")
    if channel not in ("ui", "telegram"):
        channel = "ui"
    db.update_notifications(user["id"], {
        "channel": channel,
        "telegram_bot_token": form.get("telegram_bot_token", ""),
        "telegram_chat_id": form.get("telegram_chat_id", ""),
        "email_address": "",
        "notify_on_match": _b("notify_on_match"),
        "notify_on_apply_success": _b("notify_on_apply_success"),
        "notify_on_apply_fail": _b("notify_on_apply_fail"),
    })
    db.log_audit(user["username"], "notifications.updated", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/benachrichtigungen", status_code=303)


@app.post("/actions/account/password")
async def action_password(
    request: Request,
    old_password: str = Form(""),
    new_password: str = Form(""),
    new_password_repeat: str = Form(""),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    if not new_password or new_password != new_password_repeat:
        return RedirectResponse("/einstellungen/account?err=mismatch", status_code=303)
    if len(new_password) < 10:
        return RedirectResponse("/einstellungen/account?err=tooshort", status_code=303)
    row = db.get_user_by_username(user["username"])
    from auth import verify_hash
    if not row or not verify_hash(row["password_hash"], old_password):
        return RedirectResponse("/einstellungen/account?err=wrongold", status_code=303)
    db.set_user_password(user["id"], hash_password(new_password))
    db.log_audit(user["username"], "password.changed", user_id=user["id"], ip=client_ip(request))
    return RedirectResponse("/einstellungen/account?ok=1", status_code=303)


@app.post("/actions/submit-forms")
async def action_submit_forms(
    request: Request,
    value: str = Form(default="off"),
    csrf: str = Form(...),
    user=Depends(require_user),
):
    require_csrf(user["id"], csrf)
    new = 1 if value == "on" else 0
    db.update_preferences(user["id"], {"submit_forms": new})
    db.log_audit(user["username"], "submit_forms", "on" if new else "off",
                 user_id=user["id"], ip=client_ip(request))
    if _is_htmx(request):
        return _wohnungen_partial_or_redirect(request, user)
    return RedirectResponse(request.headers.get("referer", "/einstellungen/profil"), status_code=303)


@app.post("/actions/users/create")
async def action_users_create(
    request: Request,
    username: str = Form(...),
    password: str = Form(...),
    is_admin: str = Form(""),
    csrf: str = Form(...),
    admin=Depends(require_admin),
):
    require_csrf(admin["id"], csrf)
    username = (username or "").strip()
    if not username or len(password) < 10:
        raise HTTPException(400, "username required, password >= 10 chars")
    try:
        uid = db.create_user(username, hash_password(password),
                             is_admin=(is_admin.lower() in ("on", "true", "yes", "1")))
    except sqlite3.IntegrityError:
        return RedirectResponse("/admin/benutzer?err=exists", status_code=303)
    db.log_audit(admin["username"], "user.created", f"new_user={username} id={uid}",
                 user_id=admin["id"], ip=client_ip(request))
    return RedirectResponse("/admin/benutzer?ok=1", status_code=303)


@app.post("/actions/users/disable")
async def action_users_disable(
    request: Request,
    target_id: int = Form(...),
    value: str = Form(...),
    csrf: str = Form(...),
    admin=Depends(require_admin),
):
    require_csrf(admin["id"], csrf)
    if target_id == admin["id"]:
        raise HTTPException(400, "refusing to disable self")
    db.set_user_disabled(target_id, value == "on")
    db.log_audit(admin["username"], "user.toggle_disable",
                 f"target={target_id} disabled={value=='on'}",
                 user_id=admin["id"], ip=client_ip(request))
    return RedirectResponse("/admin/benutzer", status_code=303)


@app.post("/actions/secrets")
async def action_secrets(request: Request, admin=Depends(require_admin)):
    form = await request.form()
    require_csrf(admin["id"], form.get("csrf", ""))
    changed = []
    for key in db.SECRET_KEYS:
        raw = (form.get(key) or "").strip()
        if not raw:
            continue
        db.set_secret(key, raw)
        changed.append(key)
    db.log_audit(admin["username"], "secrets.updated",
                 ",".join(changed) or "no-op",
                 user_id=admin["id"], ip=client_ip(request))
    return RedirectResponse("/admin/geheimnisse?ok=1", status_code=303)


@app.post("/actions/enrich-all")
async def action_enrich_all(
    request: Request,
    csrf: str = Form(...),
    admin=Depends(require_admin),
):
    require_csrf(admin["id"], csrf)
    queued = enrichment.kick_backfill()
    db.log_audit(admin["username"], "enrichment.backfill",
                 f"queued={queued}", user_id=admin["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, admin)


@app.post("/actions/enrich-flat")
async def action_enrich_flat(
    request: Request,
    flat_id: str = Form(...),
    csrf: str = Form(...),
    admin=Depends(require_admin),
):
    require_csrf(admin["id"], csrf)
    db.set_flat_enrichment(flat_id, "pending")
    enrichment.kick(flat_id)
    db.log_audit(admin["username"], "enrichment.retry",
                 f"flat={flat_id}", user_id=admin["id"], ip=client_ip(request))
    return _wohnungen_partial_or_redirect(request, admin)


@app.post("/actions/users/delete")
async def action_users_delete(
    request: Request,
    target_id: int = Form(...),
    csrf: str = Form(...),
    admin=Depends(require_admin),
):
    require_csrf(admin["id"], csrf)
    if target_id == admin["id"]:
        raise HTTPException(400, "refusing to delete self")
    target = db.get_user(target_id)
    if not target:
        return RedirectResponse("/admin/benutzer", status_code=303)
    db.delete_user(target_id)
    db.log_audit(admin["username"], "user.deleted",
                 f"target={target_id} username={target['username']}",
                 user_id=admin["id"], ip=client_ip(request))
    return RedirectResponse("/admin/benutzer?deleted=1", status_code=303)


# ---------------------------------------------------------------------------
# Internal endpoints
# ---------------------------------------------------------------------------

@app.post("/internal/flats")
async def internal_submit_flat(
    payload: dict,
    _guard: None = Depends(require_internal),
):
    if not payload.get("id") or not payload.get("link"):
        raise HTTPException(400, "id and link required")

    is_new = db.upsert_flat(payload)
    if not is_new:
        return {"status": "duplicate"}

    # Kick LLM enrichment + image download for this fresh flat.
    enrichment.kick(str(payload["id"]))

    for u in db.list_users():
        if u["disabled"]:
            continue
        filters = row_to_dict(db.get_filters(u["id"]))
        if not flat_matches_filter(payload, filters):
            continue

        db.log_audit("alert", "flat_matched",
                     f"user={u['username']} flat={payload['id']}",
                     user_id=u["id"])
        notifications.on_match(u["id"], payload)

        prefs = db.get_preferences(u["id"])
        if _auto_apply_allowed(prefs):
            _kick_apply(u["id"], str(payload["id"]), payload["link"], "auto")
            db.log_audit("system", "auto_apply_kick",
                         f"user={u['username']} flat={payload['id']}",
                         user_id=u["id"])

    return {"status": "ok"}


@app.post("/internal/heartbeat")
async def internal_heartbeat(payload: dict, _g: None = Depends(require_internal)):
    service = payload.get("service", "unknown")
    db.set_state(f"last_{service}_heartbeat", db.now_iso())
    return {"status": "ok"}


@app.post("/internal/error")
async def internal_report_error(
    payload: dict,
    _g: None = Depends(require_internal),
):
    db.log_error(
        source=payload.get("source", "unknown"),
        kind=payload.get("kind", "error"),
        summary=payload.get("summary", ""),
        context=payload.get("context"),
    )
    return {"status": "ok"}


@app.get("/internal/secrets")
async def internal_secrets(_g: None = Depends(require_internal)):
    """Give sibling services (alert) the current runtime creds that the admin
    may have edited via the UI, so no redeploy is needed when rotating."""
    return db.all_secrets()