diff --git a/apply/actions.py b/apply/actions.py index 9f7a2f4..436e8f3 100644 --- a/apply/actions.py +++ b/apply/actions.py @@ -1,38 +1,198 @@ +""" +Playwright actions + forensic recorder. + +The recorder captures everything a downstream AI agent would need to diagnose +a broken application flow: + +* a structured `step_log` (one entry per `recorder.step(...)`) +* browser console logs +* browser errors +* every network request + selective response bodies +* page HTML at finalize time +* screenshots at key moments + +Payloads are capped so SQLite stays healthy. Screenshots are base64 JPEGs. +""" +import asyncio +import base64 +import logging +import time from contextlib import asynccontextmanager -from playwright.async_api import async_playwright, ViewportSize +from typing import Optional + +from playwright.async_api import ViewportSize, async_playwright from reportlab.pdfgen import canvas -from settings import * -import logging +from settings import BROWSER_HEIGHT, BROWSER_LOCALE, BROWSER_WIDTH, HEADLESS, POST_SUBMISSION_SLEEP_MS logger = logging.getLogger("flat-apply") +MAX_CONSOLE_ENTRIES = 200 +MAX_NETWORK_ENTRIES = 150 +MAX_BODY_SNIPPET = 2000 +MAX_HTML_DUMP = 200_000 # 200 KB +SCREENSHOT_JPEG_QUALITY = 60 + + +class Recorder: + """Captures browser + step telemetry for one apply run.""" + + def __init__(self, url: str): + self.started_at = time.time() + self.url = url + self.steps: list[dict] = [] + self.console: list[dict] = [] + self.errors: list[dict] = [] + self.network: list[dict] = [] + self.screenshots: list[dict] = [] + self.final_html: Optional[str] = None + self.final_url: Optional[str] = None + + # --- step log ----------------------------------------------------------- + def step(self, step_name: str, status: str = "ok", detail: str = "") -> None: + entry = { + "ts": round(time.time() - self.started_at, 3), + "step": step_name, + "status": status, + "detail": str(detail)[:500], + } + self.steps.append(entry) + log = logger.info if status == "ok" else logger.warning + log("step %-20s %-4s %s", step_name, status, detail) + + # --- browser hooks ------------------------------------------------------ + def _attach(self, page) -> None: + def on_console(msg): + try: + text = msg.text + except Exception: + text = "" + if len(self.console) < MAX_CONSOLE_ENTRIES: + self.console.append({ + "ts": round(time.time() - self.started_at, 3), + "type": getattr(msg, "type", "?"), + "text": text[:500], + }) + + def on_pageerror(err): + if len(self.errors) < MAX_CONSOLE_ENTRIES: + self.errors.append({ + "ts": round(time.time() - self.started_at, 3), + "message": str(err)[:1000], + }) + + def on_request(req): + if len(self.network) < MAX_NETWORK_ENTRIES: + self.network.append({ + "ts": round(time.time() - self.started_at, 3), + "kind": "request", + "method": req.method, + "url": req.url, + "resource_type": req.resource_type, + }) + + async def on_response(resp): + if len(self.network) >= MAX_NETWORK_ENTRIES: + return + try: + snippet = "" + if resp.status >= 400: + try: + body = await resp.text() + snippet = body[:MAX_BODY_SNIPPET] + except Exception: + snippet = "" + self.network.append({ + "ts": round(time.time() - self.started_at, 3), + "kind": "response", + "status": resp.status, + "url": resp.url, + "body_snippet": snippet, + }) + except Exception: + pass + + page.on("console", on_console) + page.on("pageerror", on_pageerror) + page.on("request", on_request) + page.on("response", lambda r: asyncio.create_task(on_response(r))) + + # --- screenshots -------------------------------------------------------- + async def snap(self, page, label: str) -> None: + try: + data = await page.screenshot(type="jpeg", quality=SCREENSHOT_JPEG_QUALITY, + full_page=False, timeout=5000) + b64 = base64.b64encode(data).decode("ascii") + self.screenshots.append({ + "ts": round(time.time() - self.started_at, 3), + "label": label, + "url": page.url, + "b64_jpeg": b64, + "size": len(data), + }) + except Exception as e: + logger.warning("snap failed (%s): %s", label, e) + + async def finalize(self, page) -> None: + try: + self.final_url = page.url + html = await page.content() + self.final_html = html[:MAX_HTML_DUMP] + except Exception as e: + logger.warning("finalize html dump failed: %s", e) + await self.snap(page, "final") + + def to_json(self) -> dict: + return { + "url": self.url, + "final_url": self.final_url, + "duration_s": round(time.time() - self.started_at, 3), + "steps": self.steps, + "console": self.console, + "errors": self.errors, + "network": self.network, + "screenshots": self.screenshots, + "final_html": self.final_html, + } + + @asynccontextmanager -async def open_page(url): +async def open_page(url: str, recorder: Optional[Recorder] = None): async with async_playwright() as p: browser = await p.chromium.launch( headless=HEADLESS, - args=["--disable-blink-features=AutomationControlled"] + args=["--disable-blink-features=AutomationControlled"], ) - context = await browser.new_context( - viewport=ViewportSize({ - "width": BROWSER_WIDTH, - "height": BROWSER_HEIGHT}), - locale=BROWSER_LOCALE + viewport=ViewportSize({"width": BROWSER_WIDTH, "height": BROWSER_HEIGHT}), + locale=BROWSER_LOCALE, ) - page = await context.new_page() + if recorder: + recorder._attach(page) + recorder.step("launch", detail=f"headless={HEADLESS}") + + if recorder: + recorder.step("goto", detail=url) await page.goto(url) await page.wait_for_load_state("networkidle") + if recorder: + recorder.step("loaded", detail=page.url) + await recorder.snap(page, "loaded") try: yield page finally: + if recorder: + try: + await recorder.finalize(page) + except Exception: + logger.exception("recorder.finalize failed") await page.wait_for_timeout(POST_SUBMISSION_SLEEP_MS) await browser.close() + def create_dummy_pdf(): logger.info("creating dummy pdf") c = canvas.Canvas("DummyPDF.pdf") diff --git a/apply/classes/profile.py b/apply/classes/profile.py new file mode 100644 index 0000000..c4f8659 --- /dev/null +++ b/apply/classes/profile.py @@ -0,0 +1,66 @@ +"""Applicant profile passed from web → apply on each request.""" +from dataclasses import dataclass, field +from datetime import datetime + + +@dataclass +class Profile: + salutation: str = "Herr" + firstname: str = "" + lastname: str = "" + email: str = "" + telephone: str = "" + street: str = "" + house_number: str = "" + postcode: str = "" + city: str = "" + + # WBS + is_possessing_wbs: bool = False + wbs_type: str = "0" + wbs_valid_till: str = "1970-01-01" # ISO date string + wbs_rooms: int = 0 + wbs_adults: int = 0 + wbs_children: int = 0 + is_prio_wbs: bool = False + + # optional: immomio login for providers that need it + immomio_email: str = "" + immomio_password: str = "" + + @property + def person_count(self) -> int: + return self.wbs_adults + self.wbs_children + + @property + def adult_count(self) -> int: + return self.wbs_adults + + @property + def children_count(self) -> int: + return self.wbs_children + + def wbs_valid_till_dt(self) -> datetime: + try: + return datetime.strptime(self.wbs_valid_till, "%Y-%m-%d") + except ValueError: + return datetime(1970, 1, 1) + + @classmethod + def from_dict(cls, d: dict) -> "Profile": + safe = {k: d.get(k) for k in cls.__dataclass_fields__.keys() if k in d and d[k] is not None} + # normalise booleans + ints + for k in ("is_possessing_wbs", "is_prio_wbs"): + if k in safe: + v = safe[k] + if isinstance(v, str): + safe[k] = v.lower() in ("true", "1", "yes", "on") + else: + safe[k] = bool(v) + for k in ("wbs_rooms", "wbs_adults", "wbs_children"): + if k in safe: + try: + safe[k] = int(safe[k]) + except (TypeError, ValueError): + safe[k] = 0 + return cls(**safe) diff --git a/apply/main.py b/apply/main.py index 79f19d8..59dbadb 100644 --- a/apply/main.py +++ b/apply/main.py @@ -3,81 +3,125 @@ from contextlib import asynccontextmanager from urllib.parse import urlparse from fastapi import Depends, FastAPI, Header, HTTPException, status -from pydantic import BaseModel +from pydantic import BaseModel, Field from rich.console import Console from rich.logging import RichHandler import providers +from actions import Recorder from classes.application_result import ApplicationResult +from classes.profile import Profile from language import _ -from settings import INTERNAL_API_KEY, log_settings +from providers._provider import ApplyContext +from settings import INTERNAL_API_KEY def setup_logging(): logging.basicConfig( - level=logging.WARNING, - format="%(message)s", - datefmt="[%X]", - handlers=[RichHandler(markup=True, console=Console(width=110))], + level=logging.INFO, + format="%(asctime)s %(levelname)-5s %(name)s: %(message)s", + datefmt="%H:%M:%S", + handlers=[RichHandler(markup=False, console=Console(width=140), show_time=False, show_path=False)], ) logging.getLogger("flat-apply").setLevel(logging.DEBUG) + logging.getLogger("playwright").setLevel(logging.INFO) logger = logging.getLogger("flat-apply") setup_logging() +class ProfileModel(BaseModel): + salutation: str = "Herr" + firstname: str = "" + lastname: str = "" + email: str = "" + telephone: str = "" + street: str = "" + house_number: str = "" + postcode: str = "" + city: str = "" + is_possessing_wbs: bool = False + wbs_type: str = "0" + wbs_valid_till: str = "1970-01-01" + wbs_rooms: int = 0 + wbs_adults: int = 0 + wbs_children: int = 0 + is_prio_wbs: bool = False + immomio_email: str = "" + immomio_password: str = "" + + class ApplyRequest(BaseModel): url: str + profile: ProfileModel + submit_forms: bool = False + application_id: int | None = None # echoed back in logs class ApplyResponse(BaseModel): success: bool message: str + provider: str + application_id: int | None = None + forensics: dict def require_api_key(x_internal_api_key: str | None = Header(default=None)) -> None: if not INTERNAL_API_KEY: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="apply service has no INTERNAL_API_KEY configured", - ) + raise HTTPException(status_code=503, detail="INTERNAL_API_KEY not configured") if x_internal_api_key != INTERNAL_API_KEY: - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="invalid api key") + raise HTTPException(status_code=401, detail="invalid api key") @asynccontextmanager async def lifespan(_app: FastAPI): - log_settings() - logger.info(f"apply ready, providers: {sorted(providers.PROVIDERS)}") + logger.info("apply ready, providers: %s", sorted(providers.PROVIDERS)) yield -app = FastAPI(lifespan=lifespan, title="lazyflat-apply") +app = FastAPI(lifespan=lifespan, title="lazyflat-apply", docs_url=None, redoc_url=None) @app.get("/health") def health(): - return {"status": "ok"} + return {"status": "ok", "providers": sorted(providers.PROVIDERS)} @app.post("/apply", response_model=ApplyResponse, dependencies=[Depends(require_api_key)]) async def apply(req: ApplyRequest): url = req.url.strip() domain = urlparse(url).netloc.lower().removeprefix("www.") - logger.info(f"apply request for domain={domain} url={url}") + logger.info("apply request application_id=%s domain=%s submit=%s", + req.application_id, domain, req.submit_forms) + + recorder = Recorder(url) + recorder.step("request.received", detail=f"application_id={req.application_id} domain={domain} submit={req.submit_forms}") if domain not in providers.PROVIDERS: - logger.warning(f"unsupported provider: {domain}") + recorder.step("unsupported_provider", "warn", domain) result = ApplicationResult(False, message=_("unsupported_association")) - return ApplyResponse(success=result.success, message=str(result)) + return ApplyResponse( + success=False, message=str(result), provider="", + application_id=req.application_id, forensics=recorder.to_json(), + ) + + provider = providers.PROVIDERS[domain] + profile = Profile.from_dict(req.profile.model_dump()) + ctx = ApplyContext(profile=profile, submit_forms=req.submit_forms, recorder=recorder) try: - provider = providers.PROVIDERS[domain] - result = await provider.apply_for_flat(url) - logger.info(f"application result: {repr(result)}") + result = await provider.apply_for_flat(url, ctx) + logger.info("apply outcome application_id=%s: %r", req.application_id, result) except Exception as e: - logger.exception("error while applying") + logger.exception("apply crashed application_id=%s", req.application_id) + recorder.step("exception", "error", f"{type(e).__name__}: {e}") result = ApplicationResult(False, f"Script Error:\n{e}") - return ApplyResponse(success=result.success, message=str(result)) + return ApplyResponse( + success=result.success, + message=str(result), + provider=provider.domain, + application_id=req.application_id, + forensics=recorder.to_json(), + ) diff --git a/apply/providers/_provider.py b/apply/providers/_provider.py index cddbcfe..03fb544 100644 --- a/apply/providers/_provider.py +++ b/apply/providers/_provider.py @@ -1,22 +1,37 @@ +"""Abstract provider interface. Concrete providers live next to this file.""" import asyncio -from abc import ABC, abstractmethod import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass +from actions import Recorder from classes.application_result import ApplicationResult +from classes.profile import Profile logger = logging.getLogger("flat-apply") + +@dataclass +class ApplyContext: + """Per-request state passed into every provider call.""" + profile: Profile + submit_forms: bool + recorder: Recorder + + class Provider(ABC): @property @abstractmethod def domain(self) -> str: - """every flat provider needs a domain""" - pass + ... @abstractmethod - async def apply_for_flat(self, url: str) -> ApplicationResult: - """every flat provider needs to be able to apply for flats""" - pass + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + ... - def test_apply(self, url): - print(asyncio.run(self.apply_for_flat(url))) + def test_apply(self, url, profile: Profile | None = None, submit_forms: bool = False): + rec = Recorder(url) + ctx = ApplyContext(profile or Profile(), submit_forms=submit_forms, recorder=rec) + result = asyncio.run(self.apply_for_flat(url, ctx)) + print(repr(result)) + return result, rec.to_json() diff --git a/apply/providers/degewo.py b/apply/providers/degewo.py index 7c2ef05..c27f325 100644 --- a/apply/providers/degewo.py +++ b/apply/providers/degewo.py @@ -1,129 +1,101 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") + class Degewo(Provider): @property def domain(self) -> str: return "degewo.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: accepting cookies") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + async with open_page(url, recorder=r) as page: + r.step("cookies.check") cookie_accept_btn = page.locator("#cookie-consent-submit-all") if await cookie_accept_btn.is_visible(): await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") - else: - logger.debug("\t\tno cookie accept button found") + r.step("cookies.accepted") - logger.info("\tSTEP 2: check if the page was not found") + r.step("page.404_check") if page.url == "https://www.degewo.de/immosuche/404": - logger.debug("\t\t'page not found' message found - returning") - return ApplicationResult( - success=False, - message=_("ad_offline")) - logger.debug("\t\t'page not found' message not found") + return ApplicationResult(False, message=_("ad_offline")) - logger.info("\tSTEP 3: check if the ad is deactivated") + r.step("ad.deactivated_check") if await page.locator("span", has_text="Inserat deaktiviert").is_visible(): - logger.debug("\t\t'ad deactivated' message found - returning") - return ApplicationResult( - success=False, - message=_("ad_deactivated")) - logger.debug("\t\t'ad deactivated' message not found") + return ApplicationResult(False, message=_("ad_deactivated")) - logger.info("\tSTEP 4: check if the page moved") + r.step("page.moved_check") if await page.locator("h1", has_text="Diese Seite ist umgezogen!").is_visible(): - logger.debug("\t\t'page moved' message found - returning") - return ApplicationResult( - success=False, - message=_("ad_offline")) - logger.debug("\t\t'page moved' message not found") + return ApplicationResult(False, message=_("ad_offline")) - - logger.info("\tSTEP 5: go to the application form") - await page.get_by_role("link", name="Kontakt").click() - - logger.info("\tSTEP 6: find the form iframe") + r.step("form.open"); await page.get_by_role("link", name="Kontakt").click() + r.step("iframe.locate") form_frame = page.frame_locator("iframe[src*='wohnungshelden']") - logger.info("\tSTEP 7: fill the form") - await form_frame.locator("#salutation").fill(SALUTATION) - await form_frame.get_by_role("option", name=SALUTATION, exact=True).click() - await form_frame.locator("#firstName").fill(FIRSTNAME) - await form_frame.locator("#lastName").fill(LASTNAME) - await form_frame.locator("#email").fill(EMAIL) - await form_frame.locator("input[title='Telefonnummer']").fill(TELEPHONE) - await form_frame.locator("input[title='Anzahl einziehende Personen']").fill(str(PERSON_COUNT)) + r.step("form.fill_personal") + await form_frame.locator("#salutation").fill(p.salutation) + await form_frame.get_by_role("option", name=p.salutation, exact=True).click() + await form_frame.locator("#firstName").fill(p.firstname) + await form_frame.locator("#lastName").fill(p.lastname) + await form_frame.locator("#email").fill(p.email) + await form_frame.locator("input[title='Telefonnummer']").fill(p.telephone) + await form_frame.locator("input[title='Anzahl einziehende Personen']").fill(str(p.person_count)) await page.wait_for_timeout(1000) + + r.step("form.wbs_section") wbs_question = form_frame.locator("input[id*='wbs_available'][id$='Ja']") if await wbs_question.is_visible(): - if not IS_POSSESSING_WBS: + if not p.is_possessing_wbs: + r.step("wbs_required_abort", "warn") return ApplicationResult(False, _("wbs_required")) await wbs_question.click() - await form_frame.locator("input[title='WBS gültig bis']").fill(WBS_VALID_TILL.strftime("%d.%m.%Y")) + await form_frame.locator("input[title='WBS gültig bis']").fill(p.wbs_valid_till_dt().strftime("%d.%m.%Y")) await page.wait_for_timeout(1000) wbs_rooms_select = form_frame.locator("ng-select[id*='wbs_max_number_rooms']") await wbs_rooms_select.click() await page.wait_for_timeout(1000) - correct_wbs_room_option = form_frame.get_by_role("option", name=str(WBS_ROOMS), exact=True) - await correct_wbs_room_option.click() + await form_frame.get_by_role("option", name=str(p.wbs_rooms), exact=True).click() await page.wait_for_timeout(1000) await form_frame.locator("ng-select[id*='fuer_wen_ist_wohnungsanfrage']").click() await page.wait_for_timeout(1000) await form_frame.get_by_role("option", name="Für mich selbst").click() - logger.info("\tSTEP 8: submit the form") - if not SUBMIT_FORMS: - logger.debug(f"\t\tdry run - not submitting") + if not ctx.submit_forms: + r.step("submit.dry_run") return ApplicationResult(success=True, message=_("application_success_dry")) - await form_frame.locator("button[data-cy*='btn-submit']").click() + + r.step("submit.click"); await form_frame.locator("button[data-cy*='btn-submit']").click() await page.wait_for_timeout(3000) - logger.info("\tSTEP 9: check the success") + r.step("success.check") if await page.locator("h4", has_text="Vielen Dank für das Übermitteln Ihrer Informationen. Sie können dieses Fenster jetzt schließen.").is_visible(): - logger.info(f"\t\tsuccess detected by heading") return ApplicationResult(success=True) - elif await self.is_missing_fields_warning(page): - logger.warning(f"\t\tmissing fields warning detected") + elif await self._missing_fields_warning(page): + r.step("missing_fields_detected", "warn") return ApplicationResult(success=False, message=_("missing_fields")) - elif await self.is_already_applied_warning(page): - logger.warning(f"\t\talready applied warning detected") + elif await self._already_applied_warning(page): + r.step("already_applied_detected", "warn") return ApplicationResult(success=False, message=_("already_applied")) - logger.warning(f"\t\tsubmit conformation not found") + r.step("success.not_found", "warn") return ApplicationResult(success=False, message=_("submit_conformation_msg_not_found")) - async def is_already_applied_warning(self, page): + async def _already_applied_warning(self, page) -> bool: await page.wait_for_timeout(1000) - form_iframe = page.frame_locator("iframe[src*='wohnungshelden']") - already_applied_warning = form_iframe.locator("span.ant-alert-message", - has_text="Es existiert bereits eine Anfrage mit dieser E-Mail Adresse") - if await already_applied_warning.first.is_visible(): - return True - return False + f = page.frame_locator("iframe[src*='wohnungshelden']") + msg = f.locator("span.ant-alert-message", + has_text="Es existiert bereits eine Anfrage mit dieser E-Mail Adresse") + return await msg.first.is_visible() - async def is_missing_fields_warning(self, page): + async def _missing_fields_warning(self, page) -> bool: await page.wait_for_timeout(1000) - form_iframe = page.frame_locator("iframe[src*='wohnungshelden']") - already_applied_warning = form_iframe.locator("span.ant-alert-message", - has_text="Es wurden nicht alle Felder korrekt befüllt. Bitte prüfen Sie ihre Eingaben") - if await already_applied_warning.first.is_visible(): - return True - return False - -if __name__ == "__main__": - # url = "https://www.degewo.de/immosuche/details/neubau-mit-wbs-140-160-180-220-mit-besonderem-wohnbedarf-1" # already applied - # url = "https://www.degewo.de/immosuche/details/wohnung-sucht-neuen-mieter-1" # angebot geschlossen - # url = "https://www.degewo.de/immosuche/details/wohnung-sucht-neuen-mieter-145" # seite nicht gefunden - # url = "https://www.degewo.de/immosuche/details/1-zimmer-mit-balkon-3" - # url = "https://www.degewo.de/immosuche/details/2-zimmer-in-gropiusstadt-4" - url = "https://www.degewo.de/immosuche/details/2-zimmer-in-gropiusstadt-4" - provider = Degewo() - provider.test_apply(url) - + f = page.frame_locator("iframe[src*='wohnungshelden']") + msg = f.locator("span.ant-alert-message", + has_text="Es wurden nicht alle Felder korrekt befüllt. Bitte prüfen Sie ihre Eingaben") + return await msg.first.is_visible() diff --git a/apply/providers/gesobau.py b/apply/providers/gesobau.py index 6c481f0..b198903 100644 --- a/apply/providers/gesobau.py +++ b/apply/providers/gesobau.py @@ -1,81 +1,69 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") + class Gesobau(Provider): @property def domain(self) -> str: return "gesobau.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: extracting immomio link") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + if not (p.immomio_email and p.immomio_password): + r.step("immomio_creds_missing", "warn") + return ApplicationResult(False, _("missing_fields")) + + async with open_page(url, recorder=r) as page: + r.step("extract.immomio_link") immomio_link = await page.get_by_role("link", name="Jetzt bewerben").get_attribute("href") - logger.info("\tSTEP 2: going to auth page") - await page.goto("https://tenant.immomio.com/de/auth/login") + r.step("auth.goto"); await page.goto("https://tenant.immomio.com/de/auth/login") await page.wait_for_timeout(1000) - logger.info("\tSTEP 3: logging in") - await page.locator('input[name="email"]').fill(IMMOMIO_EMAIL) + r.step("auth.login") + await page.locator('input[name="email"]').fill(p.immomio_email) await page.get_by_role("button", name="Anmelden").click() await page.wait_for_timeout(1000) - await page.locator("#password").fill(IMMOMIO_PASSWORD) + await page.locator("#password").fill(p.immomio_password) await page.locator("#kc-login").click() await page.wait_for_timeout(1000) - logger.info("\tSTEP 4: going back to immomio") + r.step("back.to_immomio", detail=immomio_link or "") await page.goto(immomio_link) await page.wait_for_timeout(1000) - logger.info("\tSTEP 5: accepting cookies") + r.step("cookies.check") cookie_accept_btn = page.get_by_role("button", name="Alle erlauben") if await cookie_accept_btn.is_visible(): await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") - else: - logger.debug("\t\tno cookie accept button found") - logger.info("\tSTEP 6: click apply now") - await page.get_by_role("button", name="Jetzt bewerben").click() + r.step("apply.click"); await page.get_by_role("button", name="Jetzt bewerben").click() await page.wait_for_timeout(3000) - logger.info("\tSTEP 7: check if already applied") + r.step("already_applied.check") if page.url == "https://tenant.immomio.com/de/properties/applications": return ApplicationResult(False, message=_("already_applied")) - logger.info("\tSTEP 8: clicking answer questions") + r.step("answer_questions.click") answer_questions_btn = page.get_by_role("button", name="Fragen beantworten") if await answer_questions_btn.is_visible(): await answer_questions_btn.click() - logger.debug("\t\tanswer questions button clicked") await page.wait_for_timeout(2000) - else: - logger.debug("\t\tno answer questions button found") - if await answer_questions_btn.is_visible(): # sometimes this button must be clicked twice + if await answer_questions_btn.is_visible(): await answer_questions_btn.click() - logger.debug("\t\tanswer questions button clicked") await page.wait_for_timeout(2000) - - logger.info("\tSTEP 9: verifying success by answer button vanishing") - if not await answer_questions_btn.is_visible(): # TODO better verify success - logger.info("\t\tsuccess detected by answer button vanishing") + r.step("success.check") + if not await answer_questions_btn.is_visible(): return ApplicationResult(True) - logger.info("\t\tsubmit conformation not found") + r.step("success.not_found", "warn") return ApplicationResult(False, _("submit_conformation_msg_not_found")) - - -if __name__ == "__main__": - # url = "https://www.gesobau.de/?immo_ref=10-03239-00007-1185" # already applied - # url = "https://www.gesobau.de/mieten/wohnungssuche/detailseite/florastrasse-10-12179-00002-1002-1d4d1a94-b555-48f8-b06d-d6fc02aecb0d/" - url = "https://www.gesobau.de/mieten/wohnungssuche/detailseite/rolandstrasse-10-03020-00007-1052-7f47d893-e659-4e4f-a7cd-5dcd53f4e6d7/" - provider = Gesobau() - provider.test_apply(url) diff --git a/apply/providers/gewobag.py b/apply/providers/gewobag.py index 73ef640..05b530a 100644 --- a/apply/providers/gewobag.py +++ b/apply/providers/gewobag.py @@ -1,168 +1,128 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import create_dummy_pdf, open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") + class Gewobag(Provider): @property def domain(self) -> str: return "gewobag.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: accepting cookies") - cookie_accept_btn = page.get_by_text("Alle Cookies akzeptieren") - if await cookie_accept_btn.is_visible(): - await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") - else: - logger.debug("\t\tno cookie accept button found") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + async with open_page(url, recorder=r) as page: + r.step("cookies.check") + btn = page.get_by_text("Alle Cookies akzeptieren") + if await btn.is_visible(): + await btn.click() + r.step("cookies.accepted") - logger.info("\tSTEP 2: check if the page was not found") + r.step("ad.exists_check") if await page.get_by_text("Mietangebot nicht gefunden").first.is_visible(): - logger.debug("\t\t'page not found' message found - returning") - return ApplicationResult( - success=False, - message=_("not_found")) - logger.debug("\t\t'page not found' message not found") + return ApplicationResult(False, message=_("not_found")) - - logger.info("\tSTEP 3: check if ad is still open") + r.step("ad.still_open_check") if await page.locator('#immo-mediation-notice').is_visible(): - logger.debug("\t\tad closed notice found - returning") - return ApplicationResult( - success=False, - message=_("ad_deactivated")) - logger.debug("\t\tno ad closed notice found") + return ApplicationResult(False, message=_("ad_deactivated")) - logger.info("\tSTEP 4: go to the application form") - await page.get_by_role("button", name="Anfrage senden").first.click() + r.step("form.open"); await page.get_by_role("button", name="Anfrage senden").first.click() - logger.info("\tSTEP 5: check if the flat is for seniors only") - if await self.is_senior_flat(page): - logger.debug("\t\tflat is for seniors only - returning") + r.step("form.senior_check") + if await self._is_senior_flat(page): return ApplicationResult(False, _("senior_flat")) - logger.debug("\t\tflat is not seniors only") - logger.info("\tSTEP 6: check if the flat is for special needs wbs only") - if await self.is_special_needs_wbs(page): - logger.debug("\t\tflat is for special needs wbs only - returning") + r.step("form.special_wbs_check") + if await self._is_special_needs_wbs(page): return ApplicationResult(False, _("special_need_wbs_flat")) - logger.debug("\t\tflat is not for special needs wbs only") - logger.info("\tSTEP 7: find the form iframe") + r.step("iframe.locate") form_iframe = page.frame_locator("#contact-iframe") - logger.info("\tSTEP 8: define helper functions") async def fill_field(locator, filling): - logger.debug(f"\t\tfill_field('{locator}', '{filling}')") field = form_iframe.locator(locator) if await field.is_visible(): await field.fill(filling) await page.wait_for_timeout(100) - else: - logger.debug(f"\t\t\tfield was not found") async def select_field(locator, selection): - logger.debug(f"\t\tselect_field('{locator}', '{selection}')") field = form_iframe.locator(locator) if await field.is_visible(): await field.click() await page.wait_for_timeout(100) await form_iframe.get_by_role("option", name=selection, exact=True).click() await page.wait_for_timeout(100) - else: - logger.debug(f"\t\t\tfield was not found") async def check_checkbox(locator): - logger.debug(f"\t\tcheck_checkbox('{locator}')") field = form_iframe.locator(locator) if await field.first.is_visible(): await field.evaluate_all("elements => elements.forEach(el => el.click())") await page.wait_for_timeout(100) - else: - logger.debug(f"\t\t\tfield was not found") async def upload_files(locator, files=None): if not files: create_dummy_pdf() files = ["DummyPDF.pdf"] - - logger.debug(f"\t\tupload_files('{locator}', {str(files)})") - wbs_upload_section = form_iframe.locator(locator) - if await wbs_upload_section.count() > 0: - await wbs_upload_section.locator("input[type='file']").set_input_files(files) + sec = form_iframe.locator(locator) + if await sec.count() > 0: + await sec.locator("input[type='file']").set_input_files(files) await page.wait_for_timeout(2000) - else: - logger.debug(f"\t\t\tfield was not found") + r.step("form.fill_personal") + await select_field("#salutation-dropdown", p.salutation) + await fill_field("#firstName", p.firstname) + await fill_field("#lastName", p.lastname) + await fill_field("#email", p.email) + await fill_field("#phone-number", p.telephone) + await fill_field("#street", p.street) + await fill_field("#house-number", p.house_number) + await fill_field("#zip-code", p.postcode) + await fill_field("#city", p.city) + await fill_field("input[id*='anzahl_erwachsene']", str(p.adult_count)) + await fill_field("input[id*='anzahl_kinder']", str(p.children_count)) + await fill_field("input[id*='gesamtzahl_der_einziehenden_personen']", str(p.person_count)) - logger.info("\tSTEP 9: fill the form") - await select_field("#salutation-dropdown", SALUTATION) - await fill_field("#firstName", FIRSTNAME) - await fill_field("#lastName", LASTNAME) - await fill_field("#email", EMAIL) - await fill_field("#phone-number", TELEPHONE) - await fill_field("#street", STREET) - await fill_field("#house-number", HOUSE_NUMBER) - await fill_field("#zip-code", POSTCODE) - await fill_field("#city", CITY) - await fill_field("input[id*='anzahl_erwachsene']", str(ADULT_COUNT)) - await fill_field("input[id*='anzahl_kinder']", str(CHILDREN_COUNT)) - await fill_field("input[id*='gesamtzahl_der_einziehenden_personen']", str(PERSON_COUNT)) + r.step("form.wbs_fill") await check_checkbox("[data-cy*='wbs_available'][data-cy*='-Ja']") - await fill_field("input[id*='wbs_valid_until']", WBS_VALID_TILL.strftime("%d.%m.%Y")) - await select_field("input[id*='wbs_max_number_rooms']", f"{WBS_ROOMS} Räume") - await select_field("input[id*='art_bezeichnung_des_wbs']", f"WBS {WBS_TYPE}") + await fill_field("input[id*='wbs_valid_until']", p.wbs_valid_till_dt().strftime("%d.%m.%Y")) + await select_field("input[id*='wbs_max_number_rooms']", f"{p.wbs_rooms} Räume") + await select_field("input[id*='art_bezeichnung_des_wbs']", f"WBS {p.wbs_type}") await select_field("input[id*='fuer_wen']", "Für mich selbst") - await fill_field("input[id*='telephone_number']", TELEPHONE) + await fill_field("input[id*='telephone_number']", p.telephone) await check_checkbox("input[id*='datenschutzhinweis']") await upload_files("el-application-form-document-upload", files=None) - logger.info("\tSTEP 10: submit the form") - if not SUBMIT_FORMS: - logger.debug(f"\t\tdry run - not submitting") + if not ctx.submit_forms: + r.step("submit.dry_run") return ApplicationResult(True, _("application_success_dry")) - await form_iframe.get_by_role("button", name="Anfrage versenden").click() + + r.step("submit.click"); await form_iframe.get_by_role("button", name="Anfrage versenden").click() await page.wait_for_timeout(5000) - logger.info("\tSTEP 11: check the success") + r.step("success.check", detail=page.url) if page.url.startswith("https://www.gewobag.de/daten-uebermittelt/"): - logger.info(f"\t\tsuccess detected by page url") return ApplicationResult(True) - elif self.is_missing_fields_warning(page): - logger.warning(f"\t\tmissing fields warning detected") + elif await self._is_missing_fields_warning(page): + r.step("missing_fields_detected", "warn") return ApplicationResult(False, _("missing_fields")) - else: - logger.warning(f"\t\tneither missing fields nor success detected") - return ApplicationResult(False, _("submit_conformation_msg_not_found")) + r.step("success.not_found", "warn") + return ApplicationResult(False, _("submit_conformation_msg_not_found")) - async def is_senior_flat(self, page): - form_iframe = page.frame_locator("#contact-iframe") - return await form_iframe.locator("label[for*='mindestalter_seniorenwohnhaus_erreicht']").first.is_visible() + async def _is_senior_flat(self, page): + f = page.frame_locator("#contact-iframe") + return await f.locator("label[for*='mindestalter_seniorenwohnhaus_erreicht']").first.is_visible() - async def is_special_needs_wbs(self, page): - form_iframe = page.frame_locator("#contact-iframe") - return await form_iframe.locator("label[for*='wbs_mit_besonderem_wohnbedarf_vorhanden']").first.is_visible() + async def _is_special_needs_wbs(self, page): + f = page.frame_locator("#contact-iframe") + return await f.locator("label[for*='wbs_mit_besonderem_wohnbedarf_vorhanden']").first.is_visible() - async def is_missing_fields_warning(self, page): - form_iframe = page.frame_locator("#contact-iframe") - missing_field_msg = form_iframe.locator("span.ant-alert-message", - has_text="Es wurden nicht alle Felder korrekt befüllt.") - if await missing_field_msg.first.is_visible(): - return True - return False - -if __name__ == "__main__": - #url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/0100-01036-0601-0286-vms1/" # wbs - #url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/7100-72401-0101-0011/" # senior - url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/6011-31046-0105-0045/" # more wbs fields - #url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/0100-01036-0401-0191/" # special need wbs - #url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/0100-02571-0103-0169/" - # url = "https://www.gewobag.de/fuer-mietinteressentinnen/mietangebote/0100-02571-0103-169/" # page not found - provider = Gewobag() - provider.test_apply(url) + async def _is_missing_fields_warning(self, page): + f = page.frame_locator("#contact-iframe") + msg = f.locator("span.ant-alert-message", + has_text="Es wurden nicht alle Felder korrekt befüllt.") + return await msg.first.is_visible() diff --git a/apply/providers/howoge.py b/apply/providers/howoge.py index f3cc555..ae961de 100644 --- a/apply/providers/howoge.py +++ b/apply/providers/howoge.py @@ -1,62 +1,55 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") + class Howoge(Provider): @property def domain(self) -> str: return "howoge.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: accepting cookies") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + async with open_page(url, recorder=r) as page: + r.step("cookies.check") cookie_accept_btn = page.get_by_role("button", name="Alles akzeptieren") if await cookie_accept_btn.is_visible(): await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") + r.step("cookies.accepted") else: - logger.debug("\t\tno cookie accept button found") + r.step("cookies.absent") - logger.info("\tSTEP 2: check if the page was not found") + r.step("page.404_check", detail=page.url) if page.url == "https://www.howoge.de/404": - logger.debug("\t\t'page not found' url found - returning") - return ApplicationResult( - success=False, - message=_("not_found")) - logger.debug("\t\t'page not found' url not found") + return ApplicationResult(False, message=_("not_found")) - logger.info("\tSTEP 3: go to the application form") + r.step("form.open") await page.get_by_role("link", name="Besichtigung anfragen").click() - logger.info("\tSTEP 4: fill the form") - await page.get_by_text("Ja, ich habe die Hinweise zum WBS zur Kenntnis genommen.").click() + r.step("form.wbs_hint"); await page.get_by_text("Ja, ich habe die Hinweise zum WBS zur Kenntnis genommen.").click() await page.get_by_role("button", name="Weiter").click() - await page.get_by_text("Ja, ich habe den Hinweis zum Haushaltsnettoeinkommen zur Kenntnis genommen.").click() + r.step("form.income_hint"); await page.get_by_text("Ja, ich habe den Hinweis zum Haushaltsnettoeinkommen zur Kenntnis genommen.").click() await page.get_by_role("button", name="Weiter").click() - await page.get_by_text("Ja, ich habe den Hinweis zur Bonitätsauskunft zur Kenntnis genommen.").click() + r.step("form.bonitaet_hint"); await page.get_by_text("Ja, ich habe den Hinweis zur Bonitätsauskunft zur Kenntnis genommen.").click() await page.get_by_role("button", name="Weiter").click() - await page.locator("#immo-form-firstname").fill(FIRSTNAME) - await page.locator("#immo-form-lastname").fill(LASTNAME) - await page.locator("#immo-form-email").fill(EMAIL) + r.step("form.name"); await page.locator("#immo-form-firstname").fill(p.firstname) + await page.locator("#immo-form-lastname").fill(p.lastname) + await page.locator("#immo-form-email").fill(p.email) - logger.info("\tSTEP 5: submit the form") - if not SUBMIT_FORMS: - logger.debug(f"\t\tdry run - not submitting") + if not ctx.submit_forms: + r.step("submit.dry_run") return ApplicationResult(True, _("application_success_dry")) - await page.get_by_role("button", name="Anfrage senden").click() - logger.info("\tSTEP 6: check the success") + r.step("submit.click"); await page.get_by_role("button", name="Anfrage senden").click() + + r.step("success.check") if await page.get_by_role("heading", name="Vielen Dank.").is_visible(): return ApplicationResult(True) + r.step("success.not_found", "warn") return ApplicationResult(False, _("submit_conformation_msg_not_found")) - -if __name__ == "__main__": - # url = "https://www.howoge.de/wohnungen-gewerbe/wohnungssuche/detail/1770-26279-6.html" # not found - url = "https://www.howoge.de/immobiliensuche/wohnungssuche/detail/1770-27695-194.html" - provider = Howoge() - provider.test_apply(url) diff --git a/apply/providers/stadtundland.py b/apply/providers/stadtundland.py index d891ea3..4074cd7 100644 --- a/apply/providers/stadtundland.py +++ b/apply/providers/stadtundland.py @@ -1,65 +1,56 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") + class Stadtundland(Provider): @property def domain(self) -> str: return "stadtundland.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: accepting cookies") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + async with open_page(url, recorder=r) as page: + r.step("cookies.check") cookie_accept_btn = page.get_by_text("Alle akzeptieren") if await cookie_accept_btn.is_visible(): await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") + r.step("cookies.accepted") else: - logger.debug("\t\tno cookie accept button found") + r.step("cookies.absent") - logger.info("\tSTEP 2: check if ad is still open") + r.step("page.offline_check") if await page.get_by_role("heading", name="Hier ist etwas schief gelaufen").is_visible(): - logger.debug("\t\tsomething went wrong notice found - returning") - return ApplicationResult( - success=False, - message=_("ad_offline")) - logger.debug("\t\tsomething went wrong notice not found") + return ApplicationResult(False, message=_("ad_offline")) - logger.info("\tSTEP 3: fill the form") - await page.locator("#name").fill(FIRSTNAME) - await page.locator("#surname").fill(LASTNAME) - await page.locator("#street").fill(STREET) - await page.locator("#houseNo").fill(HOUSE_NUMBER) - await page.locator("#postalCode").fill(POSTCODE) - await page.locator("#city").fill(CITY) - await page.locator("#phone").fill(TELEPHONE) - await page.locator("#email").fill(EMAIL) + r.step("form.fill") + await page.locator("#name").fill(p.firstname) + await page.locator("#surname").fill(p.lastname) + await page.locator("#street").fill(p.street) + await page.locator("#houseNo").fill(p.house_number) + await page.locator("#postalCode").fill(p.postcode) + await page.locator("#city").fill(p.city) + await page.locator("#phone").fill(p.telephone) + await page.locator("#email").fill(p.email) await page.locator("#privacy").check() await page.locator("#provision").check() - logger.info("\tSTEP 4: submit the form") - if not SUBMIT_FORMS: - logger.debug(f"\t\tdry run - not submitting") - return ApplicationResult(False, _("application_success_dry")) - await page.get_by_role("button", name="Eingaben prüfen").click() - await page.get_by_role("button", name="Absenden").click() + if not ctx.submit_forms: + r.step("submit.dry_run") + return ApplicationResult(True, _("application_success_dry")) + + r.step("submit.review"); await page.get_by_role("button", name="Eingaben prüfen").click() + r.step("submit.send"); await page.get_by_role("button", name="Absenden").click() await page.wait_for_timeout(2000) - logger.info("\tSTEP 5: check the success") + r.step("success.check") if await page.locator("p").filter(has_text="Vielen Dank!").is_visible(): - logger.info(f"\t\tsuccess detected by paragraph text") return ApplicationResult(True) - logger.warning(f"\t\tsuccess message not found") + r.step("success.not_found", "warn") return ApplicationResult(success=False, message=_("submit_conformation_msg_not_found")) - - -if __name__ == "__main__": - # url = "https://stadtundland.de/wohnungssuche/1001%2F0203%2F00310" # offline - url = "https://stadtundland.de/wohnungssuche/1050%2F8222%2F00091" # wbs - provider = Stadtundland() - provider.test_apply(url) diff --git a/apply/providers/wbm.py b/apply/providers/wbm.py index 51bf0a4..c0bc1f0 100644 --- a/apply/providers/wbm.py +++ b/apply/providers/wbm.py @@ -1,10 +1,10 @@ -from actions import * -from language import _ -from classes.application_result import ApplicationResult -from providers._provider import Provider -from settings import * import logging +from actions import open_page +from classes.application_result import ApplicationResult +from language import _ +from providers._provider import ApplyContext, Provider + logger = logging.getLogger("flat-apply") @@ -13,88 +13,71 @@ class Wbm(Provider): def domain(self) -> str: return "wbm.de" - async def apply_for_flat(self, url) -> ApplicationResult: - async with open_page(url) as page: - logger.info("\tSTEP 1: checking if page not found") + async def apply_for_flat(self, url: str, ctx: ApplyContext) -> ApplicationResult: + r = ctx.recorder + p = ctx.profile + async with open_page(url, recorder=r) as page: + r.step("page.404_check") if await page.get_by_role("heading", name="Page Not Found").is_visible(): - logger.debug("\t\t'page not found' message found - returning") - return ApplicationResult( - success=False, - message=_("not_found")) - logger.debug("\t\t'page not found' message not found") + return ApplicationResult(False, message=_("not_found")) - logger.info("\tSTEP 2: accepting cookies") + r.step("cookies.check") cookie_accept_btn = page.get_by_text("Alle zulassen") if await cookie_accept_btn.is_visible(): await cookie_accept_btn.click() - logger.debug("\t\tcookie accept button clicked") + r.step("cookies.accepted") else: - logger.debug("\t\tno cookie accept button found") + r.step("cookies.absent") - logger.info("\tSTEP 3: removing chatbot help icon") - await page.locator('#removeConvaiseChat').click() + r.step("chatbot.remove") + try: + await page.locator('#removeConvaiseChat').click() + except Exception as e: + r.step("chatbot.remove_failed", "warn", str(e)) - logger.info("\tSTEP 4: checking if ad is offline") + r.step("ad.offline_check") if page.url == "https://www.wbm.de/wohnungen-berlin/angebote/": - logger.debug("\t\t'page not found' url found - returning") - return ApplicationResult( - success=False, - message=_("ad_offline")) - logger.debug("\t\t'page not found' url not found") + return ApplicationResult(False, message=_("ad_offline")) + r.step("form.open"); await page.locator('.openimmo-detail__contact-box-button').click() - logger.info("\tSTEP 5: go to the application form") - await page.locator('.openimmo-detail__contact-box-button').click() - - logger.info("\tSTEP 6: filling the application form") - if IS_POSSESSING_WBS: + r.step("form.wbs_section") + if p.is_possessing_wbs: await page.locator('label[for="powermail_field_wbsvorhanden_1"]').click() - await page.locator("input[name*='[wbsgueltigbis]']").fill(WBS_VALID_TILL.strftime("%Y-%m-%d")) - await page.locator("select[name*='[wbszimmeranzahl]']").select_option(str(WBS_ROOMS)) - await page.locator("#powermail_field_einkommensgrenzenacheinkommensbescheinigung9").select_option(WBS_TYPE) - if IS_PRIO_WBS: + await page.locator("input[name*='[wbsgueltigbis]']").fill(p.wbs_valid_till_dt().strftime("%Y-%m-%d")) + await page.locator("select[name*='[wbszimmeranzahl]']").select_option(str(p.wbs_rooms)) + await page.locator("#powermail_field_einkommensgrenzenacheinkommensbescheinigung9").select_option(p.wbs_type) + if p.is_prio_wbs: await page.locator("#powermail_field_wbsmitbesonderemwohnbedarf_1").check(force=True) else: await page.locator('label[for="powermail_field_wbsvorhanden_2"]').click() - await page.locator("#powermail_field_anrede").select_option(SALUTATION) - await page.locator("#powermail_field_name").fill(LASTNAME) - await page.locator("#powermail_field_vorname").fill(FIRSTNAME) - await page.locator("#powermail_field_strasse").fill(STREET) - await page.locator("#powermail_field_plz").fill(POSTCODE) - await page.locator("#powermail_field_ort").fill(CITY) - await page.locator("#powermail_field_e_mail").fill(EMAIL) - await page.locator("#powermail_field_telefon").fill(TELEPHONE) + r.step("form.personal") + await page.locator("#powermail_field_anrede").select_option(p.salutation) + await page.locator("#powermail_field_name").fill(p.lastname) + await page.locator("#powermail_field_vorname").fill(p.firstname) + await page.locator("#powermail_field_strasse").fill(p.street) + await page.locator("#powermail_field_plz").fill(p.postcode) + await page.locator("#powermail_field_ort").fill(p.city) + await page.locator("#powermail_field_e_mail").fill(p.email) + await page.locator("#powermail_field_telefon").fill(p.telephone) await page.locator("#powermail_field_datenschutzhinweis_1").check(force=True) - logger.info("\tSTEP 7: submit the form") - if not SUBMIT_FORMS: - logger.debug(f"\t\tdry run - not submitting") + if not ctx.submit_forms: + r.step("submit.dry_run") return ApplicationResult(success=True, message=_("application_success_dry")) - await page.get_by_role("button", name="Anfrage absenden").click() - logger.info("\tSTEP 8: check the success") + r.step("submit.click"); await page.get_by_role("button", name="Anfrage absenden").click() + + r.step("success.check") if await page.get_by_text("Wir haben Ihre Anfrage für das Wohnungsangebot erhalten.").is_visible(): - logger.info(f"\t\tsuccess detected by text") return ApplicationResult(True) - elif await self.is_missing_fields_warning(page): - logger.warning(f"\t\tmissing fields warning detected") + elif await self._missing_fields_warning(page): + r.step("missing_fields_detected", "warn") return ApplicationResult(False, _("missing_fields")) - else: - logger.warning(f"\t\tneither missing fields nor success detected") - return ApplicationResult(success=False, message=_("submit_conformation_msg_not_found")) + r.step("success.not_found", "warn") + return ApplicationResult(success=False, message=_("submit_conformation_msg_not_found")) - - async def is_missing_fields_warning(self, page): + async def _missing_fields_warning(self, page) -> bool: missing_field_msg = page.get_by_text("Dieses Feld muss ausgefüllt werden!").first - if await missing_field_msg.first.is_visible(): - return True - return False - - - -if __name__ == "__main__": - # url = "https://www.wbm.de/wohnungen-berlin/angebote/details/4-zimmer-wohnung-in-spandau-1/" # not found - url = "https://www.wbm.de/wohnungen-berlin/angebote/details/wbs-160-180-220-perfekt-fuer-kleine-familien-3-zimmer-wohnung-mit-balkon/" - provider = Wbm() - provider.test_apply(url) + return await missing_field_msg.first.is_visible() diff --git a/apply/settings.py b/apply/settings.py index 8c30363..4bd13ff 100644 --- a/apply/settings.py +++ b/apply/settings.py @@ -1,6 +1,12 @@ +""" +apply service config. + +Personal info and WBS fields used to live here as env vars; they now come in +with every /apply request as a Profile. The only remaining settings are +browser + service-level knobs. +""" import logging import sys -from datetime import datetime as dt from os import getenv from dotenv import load_dotenv @@ -8,103 +14,43 @@ logger = logging.getLogger("flat-apply") load_dotenv() -def get_env_or_fail(key: str, default: str = None, required: bool = True) -> str: + +def get_env_or_fail(key: str, default: str | None = None, required: bool = True) -> str: value = getenv(key, default) if required and value is None: logger.error(f"Missing required environment variable: {key}") sys.exit(1) return value -def get_bool_env(key: str, default: str = "False", required: bool = True) -> bool: - return get_env_or_fail(key, default, required).lower() in ("true", "1", "yes", "on") -def get_int_env(key: str, default: str = None, required: bool = True) -> int: - value_str = get_env_or_fail(key, default, required) +def get_bool_env(key: str, default: str = "False") -> bool: + return (getenv(key, default) or "").lower() in ("true", "1", "yes", "on") + + +def get_int_env(key: str, default: str) -> int: + v = getenv(key, default) or default try: - return int(value_str) + return int(v) except ValueError: - logger.error(f"Environment variable {key} must be an integer. Got: {value_str}") + logger.error(f"Env var {key} must be int, got {v!r}") sys.exit(1) -def get_date_env(key: str, fmt: str = "%Y-%m-%d", default: str = None, required: bool = True) -> dt: - value_str = get_env_or_fail(key, default, required) - try: - return dt.strptime(value_str, fmt) - except ValueError: - logger.error(f"Environment variable {key} must be a date in format {fmt}. Got: {value_str}") - sys.exit(1) -# --- General Settings --- -LANGUAGE: str = get_env_or_fail("LANGUAGE", "de", False) +# --- Browser ----------------------------------------------------------------- +HEADLESS: bool = get_bool_env("HEADLESS", "True") +BROWSER_WIDTH: int = get_int_env("BROWSER_WIDTH", "600") +BROWSER_HEIGHT: int = get_int_env("BROWSER_HEIGHT", "800") +BROWSER_LOCALE: str = getenv("BROWSER_LOCALE", "de-DE") +POST_SUBMISSION_SLEEP_MS: int = get_int_env("POST_SUBMISSION_SLEEP_MS", "0") -# --- Browser Settings --- -HEADLESS: bool = get_bool_env("HEADLESS", "True", True) -BROWSER_WIDTH: int = get_int_env("BROWSER_WIDTH", "600", False) -BROWSER_HEIGHT: int = get_int_env("BROWSER_HEIGHT", "800", False) -BROWSER_LOCALE: str = get_env_or_fail("BROWSER_LOCALE", "de-DE", False) -POST_SUBMISSION_SLEEP_MS: int = get_int_env("POST_SUBMISSION_SLEEP_MS", "0", False) - -# --- Automation Mode --- -SUBMIT_FORMS: bool = get_bool_env("SUBMIT_FORMS", "False") - -# --- HTTP Server --- -HTTP_HOST: str = get_env_or_fail("HTTP_HOST", "0.0.0.0", False) -HTTP_PORT: int = get_int_env("HTTP_PORT", "8000", False) - -# --- Personal Information --- -SALUTATION: str = get_env_or_fail("SALUTATION") -LASTNAME: str = get_env_or_fail("LASTNAME") -FIRSTNAME: str = get_env_or_fail("FIRSTNAME") -EMAIL: str = get_env_or_fail("EMAIL") -TELEPHONE: str = get_env_or_fail("TELEPHONE") -STREET: str = get_env_or_fail("STREET") -HOUSE_NUMBER: str = get_env_or_fail("HOUSE_NUMBER") -POSTCODE: str = get_env_or_fail("POSTCODE") -CITY: str = get_env_or_fail("CITY") - -# --- WBS Information --- -IS_POSSESSING_WBS: bool = get_bool_env("IS_POSSESSING_WBS", "False") -WBS_TYPE: str = get_env_or_fail("WBS_TYPE", "0", False) -WBS_VALID_TILL: dt = get_date_env("WBS_VALID_TILL", default="1970-01-01", required=False) -WBS_ROOMS: int = get_int_env("WBS_ROOMS", "0", False) -ADULT_COUNT: int = get_int_env("WBS_ADULTS", "0", False) -CHILDREN_COUNT: int = get_int_env("WBS_CHILDREN", "0", False) -PERSON_COUNT: int = ADULT_COUNT + CHILDREN_COUNT -IS_PRIO_WBS: bool = get_bool_env("IS_PRIO_WBS", "False") - -# --- Secrets --- -IMMOMIO_EMAIL: str = get_env_or_fail("IMMOMIO_EMAIL", required=False) -IMMOMIO_PASSWORD: str = get_env_or_fail("IMMOMIO_PASSWORD", required=False) +# --- Service ----------------------------------------------------------------- INTERNAL_API_KEY: str = get_env_or_fail("INTERNAL_API_KEY") -def log_settings(): + +def log_settings() -> None: logger.debug("--- Settings ---") - logger.debug(f"LANGUAGE: {LANGUAGE}") + logger.debug(f"HEADLESS: {HEADLESS}") logger.debug(f"BROWSER_WIDTH: {BROWSER_WIDTH}") logger.debug(f"BROWSER_HEIGHT: {BROWSER_HEIGHT}") logger.debug(f"BROWSER_LOCALE: {BROWSER_LOCALE}") logger.debug(f"POST_SUBMISSION_SLEEP_MS: {POST_SUBMISSION_SLEEP_MS}") - logger.debug(f"HEADLESS: {HEADLESS}") - logger.debug(f"SUBMIT_FORMS: {SUBMIT_FORMS}") - logger.debug(f"HTTP_HOST: {HTTP_HOST}") - logger.debug(f"HTTP_PORT: {HTTP_PORT}") - logger.debug(f"SALUTATION: {SALUTATION}") - logger.debug(f"LASTNAME: {LASTNAME}") - logger.debug(f"FIRSTNAME: {FIRSTNAME}") - logger.debug(f"EMAIL: {EMAIL}") - logger.debug(f"TELEPHONE: {TELEPHONE}") - logger.debug(f"STREET: {STREET}") - logger.debug(f"HOUSE_NUMBER: {HOUSE_NUMBER}") - logger.debug(f"POSTCODE: {POSTCODE}") - logger.debug(f"CITY: {CITY}") - logger.debug(f"IS_POSSESSING_WBS: {IS_POSSESSING_WBS}") - logger.debug(f"WBS_TYPE: {WBS_TYPE}") - logger.debug(f"WBS_VALID_TILL: {WBS_VALID_TILL}") - logger.debug(f"WBS_ROOMS: {WBS_ROOMS}") - logger.debug(f"WBS_ADULTS: {ADULT_COUNT}") - logger.debug(f"WBS_CHILDREN: {CHILDREN_COUNT}") - logger.debug(f"PERSON_COUNT: {PERSON_COUNT}") - logger.debug(f"IS_PRIO_WBS: {IS_PRIO_WBS}") - logger.debug(f"IMMOMIO_EMAIL: {IMMOMIO_EMAIL}") - masked_password = "***" if IMMOMIO_PASSWORD else "None" - logger.debug(f"IMMOMIO_PASSWORD: {masked_password}") diff --git a/docker-compose.yml b/docker-compose.yml index 45fff2a..10fc595 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,12 +19,17 @@ services: - SESSION_MAX_AGE_SECONDS=${SESSION_MAX_AGE_SECONDS:-604800} - LOGIN_RATE_LIMIT=${LOGIN_RATE_LIMIT:-5} - LOGIN_RATE_WINDOW_SECONDS=${LOGIN_RATE_WINDOW_SECONDS:-900} - - FILTER_ROOMS=${FILTER_ROOMS:-2.0,2.5} - - FILTER_MAX_RENT=${FILTER_MAX_RENT:-1500} - - FILTER_MAX_MORNING_COMMUTE=${FILTER_MAX_MORNING_COMMUTE:-50} + - RETENTION_DAYS=${RETENTION_DAYS:-14} + - RETENTION_RUN_INTERVAL_SECONDS=${RETENTION_RUN_INTERVAL_SECONDS:-3600} + - PUBLIC_URL=${PUBLIC_URL:-https://flat.lab.moritz.run} + - SMTP_HOST=${SMTP_HOST:-} + - SMTP_PORT=${SMTP_PORT:-587} + - SMTP_USERNAME=${SMTP_USERNAME:-} + - SMTP_PASSWORD=${SMTP_PASSWORD:-} + - SMTP_FROM=${SMTP_FROM:-lazyflat@localhost} + - SMTP_STARTTLS=${SMTP_STARTTLS:-true} volumes: - lazyflat_data:/data - # Coolify assigns the public port/domain via labels — no host port needed. expose: - "8000" @@ -32,36 +37,15 @@ services: build: ./apply container_name: lazyflat-apply restart: unless-stopped - # Intentionally NOT exposed to the internet. Reachable only on the compose network. expose: - "8000" environment: - INTERNAL_API_KEY=${INTERNAL_API_KEY} - HEADLESS=true - - SUBMIT_FORMS=${SUBMIT_FORMS:-False} - - LANGUAGE=${LANGUAGE:-de} - BROWSER_WIDTH=${BROWSER_WIDTH:-600} - BROWSER_HEIGHT=${BROWSER_HEIGHT:-800} - BROWSER_LOCALE=${BROWSER_LOCALE:-de-DE} - POST_SUBMISSION_SLEEP_MS=${POST_SUBMISSION_SLEEP_MS:-0} - - SALUTATION=${SALUTATION} - - LASTNAME=${LASTNAME} - - FIRSTNAME=${FIRSTNAME} - - EMAIL=${EMAIL} - - TELEPHONE=${TELEPHONE} - - STREET=${STREET} - - HOUSE_NUMBER=${HOUSE_NUMBER} - - POSTCODE=${POSTCODE} - - CITY=${CITY} - - IS_POSSESSING_WBS=${IS_POSSESSING_WBS:-False} - - WBS_TYPE=${WBS_TYPE:-0} - - WBS_VALID_TILL=${WBS_VALID_TILL:-1970-01-01} - - WBS_ROOMS=${WBS_ROOMS:-0} - - WBS_ADULTS=${WBS_ADULTS:-0} - - WBS_CHILDREN=${WBS_CHILDREN:-0} - - IS_PRIO_WBS=${IS_PRIO_WBS:-False} - - IMMOMIO_EMAIL=${IMMOMIO_EMAIL:-} - - IMMOMIO_PASSWORD=${IMMOMIO_PASSWORD:-} alert: build: ./alert diff --git a/web/app.py b/web/app.py index e4d9309..3b9b198 100644 --- a/web/app.py +++ b/web/app.py @@ -1,48 +1,71 @@ +""" +lazyflat web app. + +Five tabs: +- / → Wohnungen (all flats, per-user match highlighting, filter block, auto-apply switch) +- /bewerbungen → Bewerbungen (user's application history, forensics drill-in) +- /logs → Logs (user-scoped audit log) +- /fehler → Fehler (user-scoped error records + admin-global) +- /einstellungen/
→ Einstellungen: profile, filter, notifications, account, admin users + +All state-changing POSTs require CSRF. Internal endpoints require INTERNAL_API_KEY. +""" import asyncio import hmac +import json import logging +import sqlite3 import threading -from contextlib import asynccontextmanager -from typing import Optional -from fastapi import Depends, FastAPI, Form, HTTPException, Header, Request, Response, status -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi import Depends, FastAPI, Form, Header, HTTPException, Request, Response, status +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates import db -from apply_client import ApplyClient +import notifications +import retention +from apply_client import ApplyClient, _row_to_profile from auth import ( + bootstrap_admin, clear_session_cookie, current_user, + hash_password, issue_csrf_token, issue_session_cookie, rate_limit_login, + require_admin, require_csrf, require_user, - verify_password, -) -from settings import ( - APPLY_FAILURE_THRESHOLD, - AUTH_USERNAME, - FILTER_MAX_MORNING_COMMUTE, - FILTER_MAX_RENT, - FILTER_ROOMS, - INTERNAL_API_KEY, + verify_login, ) +from matching import flat_matches_filter, row_to_dict +from settings import APPLY_FAILURE_THRESHOLD, INTERNAL_API_KEY, PUBLIC_URL -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-5s %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +logging.getLogger("urllib3").setLevel(logging.WARNING) logger = logging.getLogger("web") apply_client = ApplyClient() -_apply_lock = threading.Lock() +# --------------------------------------------------------------------------- +# App +# --------------------------------------------------------------------------- + +from contextlib import asynccontextmanager + @asynccontextmanager async def lifespan(_app: FastAPI): db.init_db() - logger.info("web service started") + bootstrap_admin() + retention.start() + logger.info("web service ready") yield @@ -51,31 +74,26 @@ app.mount("/static", StaticFiles(directory="static"), name="static") templates = Jinja2Templates(directory="templates") -# ----------------------------------------------------------------------------- -# Security headers middleware -# ----------------------------------------------------------------------------- @app.middleware("http") async def security_headers(request: Request, call_next): - response: Response = await call_next(request) - response.headers.setdefault("X-Frame-Options", "DENY") - response.headers.setdefault("X-Content-Type-Options", "nosniff") - response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin") - response.headers.setdefault("Permissions-Policy", "geolocation=(), camera=(), microphone=()") - response.headers.setdefault( + resp: Response = await call_next(request) + resp.headers.setdefault("X-Frame-Options", "DENY") + resp.headers.setdefault("X-Content-Type-Options", "nosniff") + resp.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin") + resp.headers.setdefault("Permissions-Policy", "geolocation=(), camera=(), microphone=()") + resp.headers.setdefault( "Content-Security-Policy", "default-src 'self'; " "script-src 'self' https://cdn.tailwindcss.com https://unpkg.com; " "style-src 'self' https://cdn.tailwindcss.com 'unsafe-inline'; " - "img-src 'self' data:; " - "connect-src 'self'; " - "frame-ancestors 'none';" + "img-src 'self' data:; connect-src 'self'; frame-ancestors 'none';" ) - return response + return resp -# ----------------------------------------------------------------------------- +# --------------------------------------------------------------------------- # Helpers -# ----------------------------------------------------------------------------- +# --------------------------------------------------------------------------- def client_ip(request: Request) -> str: xff = request.headers.get("x-forwarded-for") @@ -86,55 +104,95 @@ def client_ip(request: Request) -> str: def require_internal(x_internal_api_key: str | None = Header(default=None)) -> None: if not x_internal_api_key or not hmac.compare_digest(x_internal_api_key, INTERNAL_API_KEY): - raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="invalid internal key") + raise HTTPException(status_code=401, detail="invalid internal key") -def matches_criteria(payload: dict) -> bool: - rooms = payload.get("rooms") or 0.0 - rent = payload.get("total_rent") or 0.0 - commute = (payload.get("connectivity") or {}).get("morning_time") or 0.0 - if FILTER_ROOMS and rooms not in FILTER_ROOMS: - return False - if rent > FILTER_MAX_RENT: - return False - if commute > FILTER_MAX_MORNING_COMMUTE: - return False - return True +def base_context(request: Request, user, active_tab: str) -> dict: + return { + "request": request, + "user": user, + "csrf": issue_csrf_token(user["id"]), + "active_tab": active_tab, + "is_admin": bool(user["is_admin"]), + } -def apply_allowed() -> tuple[bool, str]: - if db.get_state("kill_switch") == "1": +def _get_apply_gate(user_id: int) -> tuple[bool, str]: + prefs = db.get_preferences(user_id) + if prefs["kill_switch"]: return False, "kill switch aktiv" - if db.get_state("apply_circuit_open") == "1": - return False, "circuit breaker offen (zu viele Fehler)" + if prefs["apply_circuit_open"]: + return False, "circuit breaker offen (zu viele fehlgeschlagene Bewerbungen)" + if not apply_client.health(): + return False, "apply-service nicht erreichbar" return True, "" -def run_apply(flat_id: str, url: str, triggered_by: str) -> None: - """Run synchronously inside a worker thread via asyncio.to_thread.""" - app_id = db.start_application(flat_id, url, triggered_by) - try: - result = apply_client.apply(url) - except Exception as e: - result = {"success": False, "message": f"client error: {e}"} +def _run_apply_background(user_id: int, flat_id: str, url: str, triggered_by: str) -> None: + prefs = db.get_preferences(user_id) + profile_row = db.get_profile(user_id) + profile = _row_to_profile(profile_row) + submit_forms = bool(prefs["submit_forms"]) + app_id = db.start_application( + user_id=user_id, flat_id=flat_id, url=url, + triggered_by=triggered_by, submit_forms=submit_forms, + profile_snapshot=profile, + ) + + logger.info("apply.start user=%s flat=%s application=%s submit=%s", + user_id, flat_id, app_id, submit_forms) + + result = apply_client.apply(url=url, profile=profile, + submit_forms=submit_forms, application_id=app_id) success = bool(result.get("success")) - db.finish_application(app_id, success, result.get("message", "")) + message = result.get("message", "") + provider = result.get("provider", "") + forensics = result.get("forensics") or {} - # Circuit breaker accounting + db.finish_application(app_id, success=success, message=message, + provider=provider, forensics=forensics) + + # Circuit breaker (per user) if success: - db.set_state("apply_recent_failures", "0") + db.update_preferences(user_id, {"apply_recent_failures": 0}) else: - fails = int(db.get_state("apply_recent_failures") or "0") + 1 - db.set_state("apply_recent_failures", str(fails)) - if fails >= APPLY_FAILURE_THRESHOLD: - db.set_state("apply_circuit_open", "1") - db.log_audit("system", "circuit_open", f"{fails} consecutive failures") + failures = int(prefs["apply_recent_failures"] or 0) + 1 + updates = {"apply_recent_failures": failures} + if failures >= APPLY_FAILURE_THRESHOLD: + updates["apply_circuit_open"] = 1 + db.log_error(source="apply", kind="circuit_open", user_id=user_id, + summary=f"{failures} aufeinanderfolgende Fehler", + application_id=app_id) + db.update_preferences(user_id, updates) + # record forensic error row + db.log_error(source="apply", kind="apply_failure", user_id=user_id, + summary=message or "Bewerbung fehlgeschlagen", + application_id=app_id, + context={"provider": provider, "url": url}) + + # Notify user + flat = db.get_flat(flat_id) + flat_dict = {"address": flat["address"] if flat else "", "link": url, + "rooms": flat["rooms"] if flat else None, + "total_rent": flat["total_rent"] if flat else None} + if success: + notifications.on_apply_ok(user_id, flat_dict, message) + else: + notifications.on_apply_fail(user_id, flat_dict, message) + + db.log_audit("system", "apply_finished", f"app={app_id} success={success}", user_id=user_id) -# ----------------------------------------------------------------------------- +def _kick_apply(user_id: int, flat_id: str, url: str, triggered_by: str) -> None: + asyncio.create_task(asyncio.to_thread( + _run_apply_background, user_id, flat_id, url, triggered_by, + )) + + +# --------------------------------------------------------------------------- # Public routes -# ----------------------------------------------------------------------------- +# --------------------------------------------------------------------------- @app.get("/health") def health(): @@ -153,104 +211,134 @@ def login_submit(request: Request, username: str = Form(...), password: str = Fo ip = client_ip(request) if not rate_limit_login(ip): db.log_audit(username or "?", "login_rate_limited", ip=ip) + db.log_error(source="web", kind="rate_limit", summary=f"login throttled for {ip}", + context={"username": username or ""}) return templates.TemplateResponse( "login.html", - {"request": request, "error": "Zu viele Versuche. Bitte später erneut versuchen."}, + {"request": request, "error": "Zu viele Versuche. Bitte später erneut."}, status_code=status.HTTP_429_TOO_MANY_REQUESTS, ) - - if not verify_password(username, password): + user = verify_login(username, password) + if not user: db.log_audit(username or "?", "login_failed", ip=ip) return templates.TemplateResponse( "login.html", {"request": request, "error": "Login fehlgeschlagen."}, status_code=status.HTTP_401_UNAUTHORIZED, ) - response = RedirectResponse("/", status_code=303) - issue_session_cookie(response, username) - db.log_audit(username, "login_success", ip=ip) + issue_session_cookie(response, user["id"]) + db.log_audit(user["username"], "login_success", user_id=user["id"], ip=ip) return response @app.post("/logout") def logout(request: Request): - user = current_user(request) or "?" + u = current_user(request) response = RedirectResponse("/login", status_code=303) clear_session_cookie(response) - db.log_audit(user, "logout", ip=client_ip(request)) + if u: + db.log_audit(u["username"], "logout", user_id=u["id"], ip=client_ip(request)) return response -# ----------------------------------------------------------------------------- -# Authenticated dashboard -# ----------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Tab: Wohnungen +# --------------------------------------------------------------------------- @app.get("/", response_class=HTMLResponse) -def dashboard(request: Request): - user = current_user(request) - if not user: +def tab_wohnungen(request: Request): + u = current_user(request) + if not u: return RedirectResponse("/login", status_code=303) - allowed, reason = apply_allowed() - ctx = { - "request": request, - "user": user, - "csrf": issue_csrf_token(user), - "mode": db.get_state("mode") or "manual", - "kill_switch": db.get_state("kill_switch") == "1", - "circuit_open": db.get_state("apply_circuit_open") == "1", - "apply_failures": int(db.get_state("apply_recent_failures") or "0"), - "last_alert_heartbeat": db.get_state("last_alert_heartbeat") or "", - "apply_reachable": apply_client.health(), + ctx = base_context(request, u, "wohnungen") + ctx.update(_wohnungen_context(u)) + return templates.TemplateResponse("wohnungen.html", ctx) + + +def _wohnungen_context(user) -> dict: + uid = user["id"] + filters_row = db.get_filters(uid) + prefs = db.get_preferences(uid) + filters = row_to_dict(filters_row) + flats = db.recent_flats(100) + + flats_view = [] + for f in flats: + try: + payload = json.loads(f["payload_json"]) + except Exception: + payload = {} + last = db.last_application_for_flat(uid, f["id"]) + flats_view.append({ + "row": f, + "payload": payload, + "matched": flat_matches_filter({ + "rooms": f["rooms"], "total_rent": f["total_rent"], "size": f["size"], + "wbs": f["wbs"], "connectivity": {"morning_time": f["connectivity_morning_time"]}, + }, filters), + "last": last, + }) + + allowed, reason = _get_apply_gate(uid) + return { + "flats": flats_view, + "filters": filters, + "auto_apply_enabled": bool(prefs["auto_apply_enabled"]), + "submit_forms": bool(prefs["submit_forms"]), + "kill_switch": bool(prefs["kill_switch"]), + "circuit_open": bool(prefs["apply_circuit_open"]), + "apply_failures": int(prefs["apply_recent_failures"] or 0), "apply_allowed": allowed, "apply_block_reason": reason, - "flats": db.recent_flats(50), - "applications": db.recent_applications(20), - "audit": db.recent_audit(15), - } - return templates.TemplateResponse("dashboard.html", ctx) - - -@app.get("/partials/dashboard", response_class=HTMLResponse) -def dashboard_partial(request: Request, user: str = Depends(require_user)): - """HTMX partial refresh — avoids leaking data to unauthenticated clients.""" - allowed, reason = apply_allowed() - ctx = { - "request": request, - "user": user, - "csrf": issue_csrf_token(user), - "mode": db.get_state("mode") or "manual", - "kill_switch": db.get_state("kill_switch") == "1", - "circuit_open": db.get_state("apply_circuit_open") == "1", - "apply_failures": int(db.get_state("apply_recent_failures") or "0"), - "last_alert_heartbeat": db.get_state("last_alert_heartbeat") or "", "apply_reachable": apply_client.health(), - "apply_allowed": allowed, - "apply_block_reason": reason, - "flats": db.recent_flats(50), - "applications": db.recent_applications(20), - "audit": db.recent_audit(15), + "last_alert_heartbeat": db.get_state("last_alert_heartbeat") or "", } - return templates.TemplateResponse("_dashboard_body.html", ctx) -# ----------------------------------------------------------------------------- -# State-changing actions (require auth + CSRF) -# ----------------------------------------------------------------------------- - -@app.post("/actions/mode") -async def action_mode( +@app.post("/actions/filters") +async def action_save_filters( request: Request, - mode: str = Form(...), csrf: str = Form(...), - user: str = Depends(require_user), + rooms_min: str = Form(""), + rooms_max: str = Form(""), + max_rent: str = Form(""), + min_size: str = Form(""), + max_morning_commute: str = Form(""), + wbs_required: str = Form(""), + user=Depends(require_user), ): - require_csrf(request, csrf) - if mode not in ("manual", "auto"): - raise HTTPException(400, "invalid mode") - db.set_state("mode", mode) - db.log_audit(user, "set_mode", mode, client_ip(request)) + require_csrf(user["id"], csrf) + + def _f(v): + v = (v or "").strip().replace(",", ".") + return float(v) if v else None + + db.update_filters(user["id"], { + "rooms_min": _f(rooms_min), + "rooms_max": _f(rooms_max), + "max_rent": _f(max_rent), + "min_size": _f(min_size), + "max_morning_commute": _f(max_morning_commute), + "wbs_required": (wbs_required or "").strip(), + }) + db.log_audit(user["username"], "filters.updated", user_id=user["id"], ip=client_ip(request)) + return RedirectResponse("/", status_code=303) + + +@app.post("/actions/auto-apply") +async def action_auto_apply( + request: Request, + value: str = Form(...), + csrf: str = Form(...), + user=Depends(require_user), +): + require_csrf(user["id"], csrf) + new = 1 if value == "on" else 0 + db.update_preferences(user["id"], {"auto_apply_enabled": new}) + db.log_audit(user["username"], "auto_apply", "on" if new else "off", + user_id=user["id"], ip=client_ip(request)) return RedirectResponse("/", status_code=303) @@ -259,12 +347,13 @@ async def action_kill_switch( request: Request, value: str = Form(...), csrf: str = Form(...), - user: str = Depends(require_user), + user=Depends(require_user), ): - require_csrf(request, csrf) - new = "1" if value == "on" else "0" - db.set_state("kill_switch", new) - db.log_audit(user, "set_kill_switch", new, client_ip(request)) + require_csrf(user["id"], csrf) + new = 1 if value == "on" else 0 + db.update_preferences(user["id"], {"kill_switch": new}) + db.log_audit(user["username"], "kill_switch", "on" if new else "off", + user_id=user["id"], ip=client_ip(request)) return RedirectResponse("/", status_code=303) @@ -272,12 +361,11 @@ async def action_kill_switch( async def action_reset_circuit( request: Request, csrf: str = Form(...), - user: str = Depends(require_user), + user=Depends(require_user), ): - require_csrf(request, csrf) - db.set_state("apply_circuit_open", "0") - db.set_state("apply_recent_failures", "0") - db.log_audit(user, "reset_circuit", "", client_ip(request)) + require_csrf(user["id"], csrf) + db.update_preferences(user["id"], {"apply_circuit_open": 0, "apply_recent_failures": 0}) + db.log_audit(user["username"], "reset_circuit", user_id=user["id"], ip=client_ip(request)) return RedirectResponse("/", status_code=303) @@ -286,26 +374,273 @@ async def action_apply( request: Request, flat_id: str = Form(...), csrf: str = Form(...), - user: str = Depends(require_user), + user=Depends(require_user), ): - require_csrf(request, csrf) - allowed, reason = apply_allowed() + require_csrf(user["id"], csrf) + allowed, reason = _get_apply_gate(user["id"]) if not allowed: raise HTTPException(409, f"apply disabled: {reason}") - flat = db.get_flat(flat_id) if not flat: raise HTTPException(404, "flat not found") - - db.log_audit(user, "trigger_apply", f"flat_id={flat_id}", client_ip(request)) - # Run apply in background so the UI returns fast - asyncio.create_task(asyncio.to_thread(run_apply, flat_id, flat["link"], "user")) + db.log_audit(user["username"], "trigger_apply", f"flat_id={flat_id}", + user_id=user["id"], ip=client_ip(request)) + _kick_apply(user["id"], flat_id, flat["link"], "user") return RedirectResponse("/", status_code=303) -# ----------------------------------------------------------------------------- -# Internal endpoints (called by alert/apply services) -# ----------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Tab: Bewerbungen +# --------------------------------------------------------------------------- + +@app.get("/bewerbungen", response_class=HTMLResponse) +def tab_bewerbungen(request: Request): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + ctx = base_context(request, u, "bewerbungen") + ctx["applications"] = db.recent_applications(u["id"], limit=100) + return templates.TemplateResponse("bewerbungen.html", ctx) + + +@app.get("/bewerbungen/{app_id}", response_class=HTMLResponse) +def bewerbung_detail(request: Request, app_id: int): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + a = db.get_application(app_id) + if not a or (a["user_id"] != u["id"] and not u["is_admin"]): + raise HTTPException(404, "not found") + forensics = json.loads(a["forensics_json"]) if a["forensics_json"] else None + profile = json.loads(a["profile_snapshot_json"]) if a["profile_snapshot_json"] else {} + ctx = base_context(request, u, "bewerbungen") + ctx.update({"application": a, "forensics": forensics, "profile_snapshot": profile}) + return templates.TemplateResponse("bewerbung_detail.html", ctx) + + +# --------------------------------------------------------------------------- +# Tab: Logs +# --------------------------------------------------------------------------- + +@app.get("/logs", response_class=HTMLResponse) +def tab_logs(request: Request): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + ctx = base_context(request, u, "logs") + ctx["events"] = db.recent_audit(u["id"], limit=200) + return templates.TemplateResponse("logs.html", ctx) + + +# --------------------------------------------------------------------------- +# Tab: Fehler +# --------------------------------------------------------------------------- + +@app.get("/fehler", response_class=HTMLResponse) +def tab_fehler(request: Request): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + ctx = base_context(request, u, "fehler") + ctx["errors"] = db.recent_errors(u["id"], limit=200, include_global=bool(u["is_admin"])) + return templates.TemplateResponse("fehler.html", ctx) + + +@app.get("/fehler/{err_id}", response_class=HTMLResponse) +def fehler_detail(request: Request, err_id: int): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + e = db.get_error(err_id) + if not e or (e["user_id"] is not None and e["user_id"] != u["id"] and not u["is_admin"]): + raise HTTPException(404, "not found") + app_row = db.get_application(e["application_id"]) if e["application_id"] else None + forensics = None + if app_row and app_row["forensics_json"]: + try: + forensics = json.loads(app_row["forensics_json"]) + except Exception: + forensics = None + context = json.loads(e["context_json"]) if e["context_json"] else None + ctx = base_context(request, u, "fehler") + ctx.update({"error": e, "application": app_row, "forensics": forensics, "context": context}) + return templates.TemplateResponse("fehler_detail.html", ctx) + + +# --------------------------------------------------------------------------- +# Tab: Einstellungen (sub-tabs) +# --------------------------------------------------------------------------- + +VALID_SECTIONS = ("profil", "filter", "benachrichtigungen", "account", "benutzer") + + +@app.get("/einstellungen", response_class=HTMLResponse) +def tab_settings_root(request: Request): + return RedirectResponse("/einstellungen/profil", status_code=303) + + +@app.get("/einstellungen/{section}", response_class=HTMLResponse) +def tab_settings(request: Request, section: str): + u = current_user(request) + if not u: + return RedirectResponse("/login", status_code=303) + if section not in VALID_SECTIONS: + raise HTTPException(404) + if section == "benutzer" and not u["is_admin"]: + raise HTTPException(403) + + ctx = base_context(request, u, "einstellungen") + ctx["section"] = section + + if section == "profil": + ctx["profile"] = db.get_profile(u["id"]) + elif section == "filter": + ctx["filters"] = row_to_dict(db.get_filters(u["id"])) + elif section == "benachrichtigungen": + ctx["notifications"] = db.get_notifications(u["id"]) + elif section == "account": + pass + elif section == "benutzer": + ctx["users"] = db.list_users() + return templates.TemplateResponse("einstellungen.html", ctx) + + +@app.post("/actions/profile") +async def action_profile(request: Request, user=Depends(require_user)): + form = await request.form() + require_csrf(user["id"], form.get("csrf", "")) + + def _b(name): return form.get(name, "").lower() in ("true", "on", "yes", "1") + def _i(name): + try: return int(form.get(name) or 0) + except ValueError: return 0 + + db.update_profile(user["id"], { + "salutation": form.get("salutation", ""), + "firstname": form.get("firstname", ""), + "lastname": form.get("lastname", ""), + "email": form.get("email", ""), + "telephone": form.get("telephone", ""), + "street": form.get("street", ""), + "house_number": form.get("house_number", ""), + "postcode": form.get("postcode", ""), + "city": form.get("city", ""), + "is_possessing_wbs": 1 if _b("is_possessing_wbs") else 0, + "wbs_type": form.get("wbs_type", "0"), + "wbs_valid_till": form.get("wbs_valid_till", "1970-01-01"), + "wbs_rooms": _i("wbs_rooms"), + "wbs_adults": _i("wbs_adults"), + "wbs_children": _i("wbs_children"), + "is_prio_wbs": 1 if _b("is_prio_wbs") else 0, + "immomio_email": form.get("immomio_email", ""), + "immomio_password": form.get("immomio_password", ""), + }) + db.log_audit(user["username"], "profile.updated", user_id=user["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/profil", status_code=303) + + +@app.post("/actions/notifications") +async def action_notifications(request: Request, user=Depends(require_user)): + form = await request.form() + require_csrf(user["id"], form.get("csrf", "")) + def _b(n): return 1 if form.get(n, "").lower() in ("on", "true", "1", "yes") else 0 + db.update_notifications(user["id"], { + "channel": form.get("channel", "ui"), + "telegram_bot_token": form.get("telegram_bot_token", ""), + "telegram_chat_id": form.get("telegram_chat_id", ""), + "email_address": form.get("email_address", ""), + "notify_on_match": _b("notify_on_match"), + "notify_on_apply_success": _b("notify_on_apply_success"), + "notify_on_apply_fail": _b("notify_on_apply_fail"), + }) + db.log_audit(user["username"], "notifications.updated", user_id=user["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/benachrichtigungen", status_code=303) + + +@app.post("/actions/account/password") +async def action_password( + request: Request, + old_password: str = Form(""), + new_password: str = Form(""), + new_password_repeat: str = Form(""), + csrf: str = Form(...), + user=Depends(require_user), +): + require_csrf(user["id"], csrf) + if not new_password or new_password != new_password_repeat: + return RedirectResponse("/einstellungen/account?err=mismatch", status_code=303) + if len(new_password) < 10: + return RedirectResponse("/einstellungen/account?err=tooshort", status_code=303) + row = db.get_user_by_username(user["username"]) + from auth import verify_hash + if not row or not verify_hash(row["password_hash"], old_password): + return RedirectResponse("/einstellungen/account?err=wrongold", status_code=303) + db.set_user_password(user["id"], hash_password(new_password)) + db.log_audit(user["username"], "password.changed", user_id=user["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/account?ok=1", status_code=303) + + +@app.post("/actions/submit-forms") +async def action_submit_forms( + request: Request, + value: str = Form(...), + csrf: str = Form(...), + user=Depends(require_user), +): + require_csrf(user["id"], csrf) + new = 1 if value == "on" else 0 + db.update_preferences(user["id"], {"submit_forms": new}) + db.log_audit(user["username"], "submit_forms", "on" if new else "off", + user_id=user["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/profil", status_code=303) + + +# --- Admin: Benutzer --------------------------------------------------------- + +@app.post("/actions/users/create") +async def action_users_create( + request: Request, + username: str = Form(...), + password: str = Form(...), + is_admin: str = Form(""), + csrf: str = Form(...), + admin=Depends(require_admin), +): + require_csrf(admin["id"], csrf) + username = (username or "").strip() + if not username or len(password) < 10: + raise HTTPException(400, "username required, password >= 10 chars") + try: + uid = db.create_user(username, hash_password(password), + is_admin=(is_admin.lower() in ("on", "true", "yes", "1"))) + except sqlite3.IntegrityError: + return RedirectResponse("/einstellungen/benutzer?err=exists", status_code=303) + db.log_audit(admin["username"], "user.created", f"new_user={username} id={uid}", + user_id=admin["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/benutzer?ok=1", status_code=303) + + +@app.post("/actions/users/disable") +async def action_users_disable( + request: Request, + target_id: int = Form(...), + value: str = Form(...), + csrf: str = Form(...), + admin=Depends(require_admin), +): + require_csrf(admin["id"], csrf) + if target_id == admin["id"]: + raise HTTPException(400, "refusing to disable self") + db.set_user_disabled(target_id, value == "on") + db.log_audit(admin["username"], "user.toggle_disable", + f"target={target_id} disabled={value=='on'}", + user_id=admin["id"], ip=client_ip(request)) + return RedirectResponse("/einstellungen/benutzer", status_code=303) + + +# --------------------------------------------------------------------------- +# Internal endpoints +# --------------------------------------------------------------------------- @app.post("/internal/flats") async def internal_submit_flat( @@ -314,28 +649,51 @@ async def internal_submit_flat( ): if not payload.get("id") or not payload.get("link"): raise HTTPException(400, "id and link required") - matched = matches_criteria(payload) - is_new = db.upsert_flat(payload, matched) + + is_new = db.upsert_flat(payload) if not is_new: return {"status": "duplicate"} - if matched: - db.log_audit("alert", "flat_matched", f"id={payload['id']} rent={payload.get('total_rent')}") - if db.get_state("mode") == "auto": - allowed, reason = apply_allowed() - if allowed: - db.log_audit("system", "auto_apply", f"flat_id={payload['id']}") - asyncio.create_task(asyncio.to_thread(run_apply, str(payload["id"]), payload["link"], "auto")) - else: - db.log_audit("system", "auto_apply_blocked", reason) - return {"status": "ok", "matched": matched} + # per-user matching + auto-apply + notifications + for u in db.list_users(): + if u["disabled"]: + continue + filters = row_to_dict(db.get_filters(u["id"])) + if not flat_matches_filter(payload, filters): + continue + + db.log_audit("alert", "flat_matched", + f"user={u['username']} flat={payload['id']}", + user_id=u["id"]) + notifications.on_match(u["id"], payload) + + prefs = db.get_preferences(u["id"]) + if prefs["auto_apply_enabled"] and not prefs["kill_switch"] and not prefs["apply_circuit_open"]: + _kick_apply(u["id"], str(payload["id"]), payload["link"], "auto") + db.log_audit("system", "auto_apply_kick", + f"user={u['username']} flat={payload['id']}", + user_id=u["id"]) + + return {"status": "ok"} @app.post("/internal/heartbeat") -async def internal_heartbeat( - payload: dict, - _guard: None = Depends(require_internal), -): +async def internal_heartbeat(payload: dict, _g: None = Depends(require_internal)): service = payload.get("service", "unknown") db.set_state(f"last_{service}_heartbeat", db.now_iso()) return {"status": "ok"} + + +@app.post("/internal/error") +async def internal_report_error( + payload: dict, + _g: None = Depends(require_internal), +): + """Alert/other services can push errors here.""" + db.log_error( + source=payload.get("source", "unknown"), + kind=payload.get("kind", "error"), + summary=payload.get("summary", ""), + context=payload.get("context"), + ) + return {"status": "ok"} diff --git a/web/apply_client.py b/web/apply_client.py index 4a4ea9a..bb90a32 100644 --- a/web/apply_client.py +++ b/web/apply_client.py @@ -1,9 +1,31 @@ import logging import requests -from settings import APPLY_URL, APPLY_TIMEOUT, INTERNAL_API_KEY +from settings import APPLY_TIMEOUT, APPLY_URL, INTERNAL_API_KEY -logger = logging.getLogger("web") +logger = logging.getLogger("web.apply_client") + + +def _row_to_profile(profile_row) -> dict: + """Convert a user_profiles row to the apply service Profile dict.""" + if profile_row is None: + return {} + keys = [ + "salutation", "firstname", "lastname", "email", "telephone", + "street", "house_number", "postcode", "city", + "is_possessing_wbs", "wbs_type", "wbs_valid_till", + "wbs_rooms", "wbs_adults", "wbs_children", "is_prio_wbs", + "immomio_email", "immomio_password", + ] + d = {} + for k in keys: + try: + d[k] = profile_row[k] + except (KeyError, IndexError): + pass + for k in ("is_possessing_wbs", "is_prio_wbs"): + d[k] = bool(d.get(k) or 0) + return d class ApplyClient: @@ -19,16 +41,24 @@ class ApplyClient: except requests.RequestException: return False - def apply(self, url: str) -> dict: + def apply(self, url: str, profile: dict, submit_forms: bool, + application_id: int | None = None) -> dict: + body = { + "url": url, + "profile": profile, + "submit_forms": bool(submit_forms), + "application_id": application_id, + } try: r = requests.post( - f"{self.base}/apply", - json={"url": url}, - headers=self.headers, - timeout=self.timeout, + f"{self.base}/apply", json=body, + headers=self.headers, timeout=self.timeout, ) if r.status_code >= 400: - return {"success": False, "message": f"apply HTTP {r.status_code}: {r.text[:300]}"} + return {"success": False, + "message": f"apply HTTP {r.status_code}: {r.text[:300]}", + "provider": "", "forensics": {}} return r.json() except requests.RequestException as e: - return {"success": False, "message": f"apply unreachable: {e}"} + return {"success": False, "message": f"apply unreachable: {e}", + "provider": "", "forensics": {}} diff --git a/web/auth.py b/web/auth.py index c93aca1..56695b3 100644 --- a/web/auth.py +++ b/web/auth.py @@ -1,14 +1,24 @@ +""" +Authentication and session handling. + +- Users live in the DB (users table). +- On boot we seed an admin from env if no user exists yet. +- Sessions are signed cookies (itsdangerous); CSRF is a separate signed token. +- Login is rate-limited per IP. +""" import hmac -import secrets +import logging +import sqlite3 import threading import time from typing import Optional from argon2 import PasswordHasher -from argon2.exceptions import VerifyMismatchError, InvalidHash +from argon2.exceptions import InvalidHash, VerifyMismatchError from fastapi import HTTPException, Request, Response, status from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +import db from settings import ( AUTH_PASSWORD_HASH, AUTH_USERNAME, @@ -20,30 +30,63 @@ from settings import ( SESSION_SECRET, ) +logger = logging.getLogger("web.auth") + _hasher = PasswordHasher() _serializer = URLSafeTimedSerializer(SESSION_SECRET, salt="session") _csrf_serializer = URLSafeTimedSerializer(SESSION_SECRET, salt="csrf") -# ---------- Password & session ---------- +# ---------- Password hashing -------------------------------------------------- -def verify_password(username: str, password: str) -> bool: - if not hmac.compare_digest(username or "", AUTH_USERNAME): - # run hasher anyway to keep timing similar (and not leak whether user exists) - try: - _hasher.verify(AUTH_PASSWORD_HASH, password) - except Exception: - pass - return False +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_hash(password_hash: str, password: str) -> bool: try: - _hasher.verify(AUTH_PASSWORD_HASH, password) + _hasher.verify(password_hash, password) return True except (VerifyMismatchError, InvalidHash): return False -def issue_session_cookie(response: Response, username: str) -> None: - token = _serializer.dumps({"u": username, "iat": int(time.time())}) +# ---------- Admin bootstrap --------------------------------------------------- + +def bootstrap_admin() -> None: + """If no users exist, create one from env vars as admin.""" + existing = db.list_users() + if existing: + logger.info("users already present (%d), skipping admin bootstrap", len(existing)) + return + try: + uid = db.create_user(AUTH_USERNAME, AUTH_PASSWORD_HASH, is_admin=True) + logger.info("bootstrapped admin user '%s' (id=%s) from env", AUTH_USERNAME, uid) + except sqlite3.IntegrityError as e: + logger.warning("bootstrap admin failed: %s", e) + + +# ---------- Login ------------------------------------------------------------- + +def verify_login(username: str, password: str) -> Optional[sqlite3.Row]: + """Return user row on success, None otherwise. Timing-safe path runs hash check either way.""" + user = db.get_user_by_username(username or "") + if user is None: + # Run the hasher anyway so timing doesn't leak existence + try: + _hasher.verify(AUTH_PASSWORD_HASH, password or "") + except Exception: + pass + return None + if not verify_hash(user["password_hash"], password or ""): + return None + return user + + +# ---------- Session cookies --------------------------------------------------- + +def issue_session_cookie(response: Response, user_id: int) -> None: + token = _serializer.dumps({"uid": user_id, "iat": int(time.time())}) response.set_cookie( key=SESSION_COOKIE_NAME, value=token, @@ -57,15 +100,12 @@ def issue_session_cookie(response: Response, username: str) -> None: def clear_session_cookie(response: Response) -> None: response.delete_cookie( - SESSION_COOKIE_NAME, - path="/", - secure=COOKIE_SECURE, - httponly=True, - samesite="strict", + SESSION_COOKIE_NAME, path="/", + secure=COOKIE_SECURE, httponly=True, samesite="strict", ) -def current_user(request: Request) -> Optional[str]: +def current_user(request: Request) -> Optional[sqlite3.Row]: token = request.cookies.get(SESSION_COOKIE_NAME) if not token: return None @@ -73,46 +113,57 @@ def current_user(request: Request) -> Optional[str]: data = _serializer.loads(token, max_age=SESSION_MAX_AGE_SECONDS) except (BadSignature, SignatureExpired): return None - return data.get("u") + uid = data.get("uid") + if not uid: + return None + u = db.get_user(uid) + if u is None or u["disabled"]: + return None + return u -def require_user(request: Request) -> str: - user = current_user(request) - if not user: +def require_user(request: Request) -> sqlite3.Row: + u = current_user(request) + if u is None: raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="login required") - return user + return u -# ---------- CSRF (synchronizer token bound to session) ---------- - -def issue_csrf_token(username: str) -> str: - return _csrf_serializer.dumps({"u": username}) +def require_admin(request: Request) -> sqlite3.Row: + u = require_user(request) + if not u["is_admin"]: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="admin only") + return u -def verify_csrf(request: Request, submitted: str) -> bool: - user = current_user(request) - if not user or not submitted: +# ---------- CSRF -------------------------------------------------------------- + +def issue_csrf_token(user_id: int) -> str: + return _csrf_serializer.dumps({"uid": user_id}) + + +def verify_csrf(user_id: int, submitted: str) -> bool: + if not submitted: return False try: data = _csrf_serializer.loads(submitted, max_age=SESSION_MAX_AGE_SECONDS) except (BadSignature, SignatureExpired): return False - return hmac.compare_digest(str(data.get("u", "")), user) + return int(data.get("uid", -1)) == int(user_id) -def require_csrf(request: Request, token: str) -> None: - if not verify_csrf(request, token): +def require_csrf(user_id: int, token: str) -> None: + if not verify_csrf(user_id, token): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="bad csrf") -# ---------- Login rate limiting (in-memory, per IP) ---------- +# ---------- Login rate limiting ---------------------------------------------- _rate_lock = threading.Lock() _rate_log: dict[str, list[float]] = {} def rate_limit_login(ip: str) -> bool: - """Returns True if the request is allowed.""" now = time.time() cutoff = now - LOGIN_RATE_WINDOW_SECONDS with _rate_lock: @@ -122,13 +173,8 @@ def rate_limit_login(ip: str) -> bool: return False attempts.append(now) _rate_log[ip] = attempts - # opportunistic cleanup if len(_rate_log) > 1024: for k in list(_rate_log.keys()): if not _rate_log[k] or _rate_log[k][-1] < cutoff: _rate_log.pop(k, None) return True - - -def constant_time_compare(a: str, b: str) -> bool: - return hmac.compare_digest(a or "", b or "") diff --git a/web/db.py b/web/db.py index ee22142..847c77a 100644 --- a/web/db.py +++ b/web/db.py @@ -1,10 +1,20 @@ +""" +SQLite data layer for lazyflat. + +Multi-user: users, per-user profiles/filters/notifications/preferences. +All per-user rows are 1:1 with users. Errors and forensics are retained +for 14 days and cleaned up periodically. +""" import json +import logging import sqlite3 import threading -from datetime import datetime, timezone -from typing import Any, Iterable +from datetime import datetime, timedelta, timezone +from typing import Any, Iterable, Optional -from settings import DB_PATH +from settings import DB_PATH, RETENTION_DAYS + +logger = logging.getLogger("web.db") _lock = threading.Lock() @@ -20,126 +30,374 @@ def _connect() -> sqlite3.Connection: _conn: sqlite3.Connection = _connect() -SCHEMA = """ -CREATE TABLE IF NOT EXISTS flats ( - id TEXT PRIMARY KEY, - link TEXT NOT NULL, - address TEXT, - rooms REAL, - size REAL, - total_rent REAL, - sqm_price REAL, - year_built TEXT, - wbs TEXT, - connectivity_morning_time REAL, - connectivity_night_time REAL, - address_link_gmaps TEXT, - payload_json TEXT NOT NULL, - matched_criteria INTEGER NOT NULL DEFAULT 0, - discovered_at TEXT NOT NULL -); +# --------------------------------------------------------------------------- +# Schema +# --------------------------------------------------------------------------- -CREATE INDEX IF NOT EXISTS idx_flats_discovered ON flats(discovered_at DESC); -CREATE INDEX IF NOT EXISTS idx_flats_matched ON flats(matched_criteria); +MIGRATIONS: list[str] = [ + # 0001: drop legacy single-user tables if present (only matters on upgrade + # from the pre-multi-user commit; no production data yet). Safe for fresh DBs. + """ + DROP TABLE IF EXISTS applications; + DROP TABLE IF EXISTS flats; + DROP TABLE IF EXISTS state; + DROP TABLE IF EXISTS audit_log; + """, + # 0002: base + users + """ + CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); -CREATE TABLE IF NOT EXISTS applications ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - flat_id TEXT NOT NULL, - url TEXT NOT NULL, - triggered_by TEXT NOT NULL, -- 'user' | 'auto' - started_at TEXT NOT NULL, - finished_at TEXT, - success INTEGER, - message TEXT, - FOREIGN KEY (flat_id) REFERENCES flats(id) -); + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT UNIQUE NOT NULL COLLATE NOCASE, + password_hash TEXT NOT NULL, + is_admin INTEGER NOT NULL DEFAULT 0, + disabled INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); -CREATE INDEX IF NOT EXISTS idx_applications_flat ON applications(flat_id); -CREATE INDEX IF NOT EXISTS idx_applications_started ON applications(started_at DESC); + CREATE TABLE IF NOT EXISTS user_profiles ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + salutation TEXT DEFAULT '', + firstname TEXT DEFAULT '', + lastname TEXT DEFAULT '', + email TEXT DEFAULT '', + telephone TEXT DEFAULT '', + street TEXT DEFAULT '', + house_number TEXT DEFAULT '', + postcode TEXT DEFAULT '', + city TEXT DEFAULT '', + is_possessing_wbs INTEGER NOT NULL DEFAULT 0, + wbs_type TEXT DEFAULT '0', + wbs_valid_till TEXT DEFAULT '1970-01-01', + wbs_rooms INTEGER NOT NULL DEFAULT 0, + wbs_adults INTEGER NOT NULL DEFAULT 0, + wbs_children INTEGER NOT NULL DEFAULT 0, + is_prio_wbs INTEGER NOT NULL DEFAULT 0, + immomio_email TEXT DEFAULT '', + immomio_password TEXT DEFAULT '', + updated_at TEXT NOT NULL + ); -CREATE TABLE IF NOT EXISTS state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); + CREATE TABLE IF NOT EXISTS user_filters ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + rooms_min REAL, + rooms_max REAL, + max_rent REAL, + min_size REAL, + max_morning_commute REAL, + wbs_required TEXT DEFAULT '', -- '', 'yes', 'no' + updated_at TEXT NOT NULL + ); -CREATE TABLE IF NOT EXISTS audit_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - actor TEXT NOT NULL, - action TEXT NOT NULL, - details TEXT, - ip TEXT -); + CREATE TABLE IF NOT EXISTS user_notifications ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + channel TEXT NOT NULL DEFAULT 'ui', -- 'ui' | 'telegram' | 'email' + telegram_bot_token TEXT DEFAULT '', + telegram_chat_id TEXT DEFAULT '', + email_address TEXT DEFAULT '', + notify_on_match INTEGER NOT NULL DEFAULT 1, + notify_on_apply_success INTEGER NOT NULL DEFAULT 1, + notify_on_apply_fail INTEGER NOT NULL DEFAULT 1, + updated_at TEXT NOT NULL + ); -CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(timestamp DESC); -""" + CREATE TABLE IF NOT EXISTS user_preferences ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + auto_apply_enabled INTEGER NOT NULL DEFAULT 0, + submit_forms INTEGER NOT NULL DEFAULT 0, + kill_switch INTEGER NOT NULL DEFAULT 0, + apply_circuit_open INTEGER NOT NULL DEFAULT 0, + apply_recent_failures INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL + ); -DEFAULTS = { - "mode": "manual", # 'manual' | 'auto' - "kill_switch": "0", # '1' = apply disabled - "apply_circuit_open": "0", # '1' = opened by circuit breaker - "apply_recent_failures": "0", - "last_alert_heartbeat": "", - "last_apply_heartbeat": "", -} + CREATE TABLE IF NOT EXISTS flats ( + id TEXT PRIMARY KEY, + link TEXT NOT NULL, + address TEXT, + rooms REAL, + size REAL, + total_rent REAL, + sqm_price REAL, + year_built TEXT, + wbs TEXT, + connectivity_morning_time REAL, + connectivity_night_time REAL, + address_link_gmaps TEXT, + payload_json TEXT NOT NULL, + discovered_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_flats_discovered ON flats(discovered_at DESC); + + CREATE TABLE IF NOT EXISTS applications ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + flat_id TEXT NOT NULL REFERENCES flats(id), + url TEXT NOT NULL, + triggered_by TEXT NOT NULL, -- 'user' | 'auto' + submit_forms_used INTEGER NOT NULL DEFAULT 0, + provider TEXT DEFAULT '', + started_at TEXT NOT NULL, + finished_at TEXT, + success INTEGER, + message TEXT, + profile_snapshot_json TEXT, + forensics_json TEXT -- structured payload from apply service + ); + CREATE INDEX IF NOT EXISTS idx_applications_user ON applications(user_id, started_at DESC); + CREATE INDEX IF NOT EXISTS idx_applications_started ON applications(started_at DESC); + CREATE INDEX IF NOT EXISTS idx_applications_flat ON applications(flat_id); + + CREATE TABLE IF NOT EXISTS errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + user_id INTEGER, + source TEXT NOT NULL, -- 'apply'|'alert'|'web'|'system' + kind TEXT NOT NULL, -- 'apply_failure'|'scraper_error'|... + summary TEXT NOT NULL, + application_id INTEGER, + context_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_errors_timestamp ON errors(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_errors_user ON errors(user_id, timestamp DESC); + + CREATE TABLE IF NOT EXISTS audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + user_id INTEGER, + actor TEXT NOT NULL, + action TEXT NOT NULL, + details TEXT, + ip TEXT + ); + CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(timestamp DESC); + CREATE INDEX IF NOT EXISTS idx_audit_user ON audit_log(user_id, timestamp DESC); + + CREATE TABLE IF NOT EXISTS system_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """, +] + + +def _current_version() -> int: + try: + row = _conn.execute("SELECT COALESCE(MAX(version), 0) AS v FROM schema_version").fetchone() + return int(row["v"]) if row else 0 + except sqlite3.Error: + return 0 def init_db() -> None: with _lock: - _conn.executescript(SCHEMA) - for k, v in DEFAULTS.items(): - _conn.execute("INSERT OR IGNORE INTO state(key, value) VALUES (?, ?)", (k, v)) + _conn.execute("CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY)") + current = _current_version() + for i, script in enumerate(MIGRATIONS, start=1): + if i <= current: + continue + logger.info("applying migration v%d", i) + _conn.executescript(script) + _conn.execute("INSERT OR IGNORE INTO schema_version(version) VALUES (?)", (i,)) + _conn.execute( + "INSERT OR IGNORE INTO system_state(key, value) VALUES ('last_alert_heartbeat', '')" + ) + logger.info("DB initialized (schema v%d)", _current_version()) def now_iso() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") -def get_state(key: str) -> str | None: - row = _conn.execute("SELECT value FROM state WHERE key = ?", (key,)).fetchone() +# --------------------------------------------------------------------------- +# System state +# --------------------------------------------------------------------------- + +def get_state(key: str) -> Optional[str]: + row = _conn.execute("SELECT value FROM system_state WHERE key = ?", (key,)).fetchone() return row["value"] if row else None def set_state(key: str, value: str) -> None: with _lock: _conn.execute( - "INSERT INTO state(key, value) VALUES (?, ?) " + "INSERT INTO system_state(key, value) VALUES (?, ?) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value), ) -def upsert_flat(payload: dict, matched: bool) -> bool: - """Returns True if this flat is new.""" +# --------------------------------------------------------------------------- +# Users +# --------------------------------------------------------------------------- + +def _ensure_user_rows(user_id: int) -> None: + ts = now_iso() + with _lock: + for q in ( + "INSERT OR IGNORE INTO user_profiles(user_id, updated_at) VALUES (?, ?)", + "INSERT OR IGNORE INTO user_filters(user_id, updated_at) VALUES (?, ?)", + "INSERT OR IGNORE INTO user_notifications(user_id, updated_at) VALUES (?, ?)", + "INSERT OR IGNORE INTO user_preferences(user_id, updated_at) VALUES (?, ?)", + ): + _conn.execute(q, (user_id, ts)) + + +def create_user(username: str, password_hash: str, is_admin: bool = False) -> int: + ts = now_iso() + with _lock: + cur = _conn.execute( + "INSERT INTO users(username, password_hash, is_admin, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?)", + (username, password_hash, 1 if is_admin else 0, ts, ts), + ) + uid = cur.lastrowid + _ensure_user_rows(uid) + logger.info("created user id=%s username=%s admin=%s", uid, username, is_admin) + return uid + + +def get_user_by_username(username: str) -> Optional[sqlite3.Row]: + return _conn.execute( + "SELECT * FROM users WHERE username = ? COLLATE NOCASE AND disabled = 0", (username,) + ).fetchone() + + +def get_user(user_id: int) -> Optional[sqlite3.Row]: + return _conn.execute("SELECT * FROM users WHERE id = ?", (user_id,)).fetchone() + + +def list_users() -> list[sqlite3.Row]: + return list(_conn.execute("SELECT * FROM users ORDER BY username").fetchall()) + + +def set_user_password(user_id: int, password_hash: str) -> None: + with _lock: + _conn.execute( + "UPDATE users SET password_hash = ?, updated_at = ? WHERE id = ?", + (password_hash, now_iso(), user_id), + ) + + +def set_user_disabled(user_id: int, disabled: bool) -> None: + with _lock: + _conn.execute( + "UPDATE users SET disabled = ?, updated_at = ? WHERE id = ?", + (1 if disabled else 0, now_iso(), user_id), + ) + + +# --------------------------------------------------------------------------- +# User profile / filters / notifications / preferences +# --------------------------------------------------------------------------- + +def get_profile(user_id: int) -> sqlite3.Row: + _ensure_user_rows(user_id) + return _conn.execute("SELECT * FROM user_profiles WHERE user_id = ?", (user_id,)).fetchone() + + +def update_profile(user_id: int, data: dict) -> None: + _ensure_user_rows(user_id) + allowed = { + "salutation", "firstname", "lastname", "email", "telephone", + "street", "house_number", "postcode", "city", + "is_possessing_wbs", "wbs_type", "wbs_valid_till", + "wbs_rooms", "wbs_adults", "wbs_children", "is_prio_wbs", + "immomio_email", "immomio_password", + } + clean = {k: v for k, v in data.items() if k in allowed} + if not clean: + return + cols = ", ".join(f"{k} = ?" for k in clean) + vals = list(clean.values()) + [now_iso(), user_id] + with _lock: + _conn.execute(f"UPDATE user_profiles SET {cols}, updated_at = ? WHERE user_id = ?", vals) + + +def get_filters(user_id: int) -> sqlite3.Row: + _ensure_user_rows(user_id) + return _conn.execute("SELECT * FROM user_filters WHERE user_id = ?", (user_id,)).fetchone() + + +def update_filters(user_id: int, data: dict) -> None: + _ensure_user_rows(user_id) + allowed = {"rooms_min", "rooms_max", "max_rent", "min_size", "max_morning_commute", "wbs_required"} + clean = {k: data.get(k) for k in allowed if k in data} + if not clean: + return + cols = ", ".join(f"{k} = ?" for k in clean) + vals = list(clean.values()) + [now_iso(), user_id] + with _lock: + _conn.execute(f"UPDATE user_filters SET {cols}, updated_at = ? WHERE user_id = ?", vals) + + +def get_notifications(user_id: int) -> sqlite3.Row: + _ensure_user_rows(user_id) + return _conn.execute("SELECT * FROM user_notifications WHERE user_id = ?", (user_id,)).fetchone() + + +def update_notifications(user_id: int, data: dict) -> None: + _ensure_user_rows(user_id) + allowed = { + "channel", "telegram_bot_token", "telegram_chat_id", "email_address", + "notify_on_match", "notify_on_apply_success", "notify_on_apply_fail", + } + clean = {k: v for k, v in data.items() if k in allowed} + if not clean: + return + cols = ", ".join(f"{k} = ?" for k in clean) + vals = list(clean.values()) + [now_iso(), user_id] + with _lock: + _conn.execute(f"UPDATE user_notifications SET {cols}, updated_at = ? WHERE user_id = ?", vals) + + +def get_preferences(user_id: int) -> sqlite3.Row: + _ensure_user_rows(user_id) + return _conn.execute("SELECT * FROM user_preferences WHERE user_id = ?", (user_id,)).fetchone() + + +def update_preferences(user_id: int, data: dict) -> None: + _ensure_user_rows(user_id) + allowed = { + "auto_apply_enabled", "submit_forms", "kill_switch", + "apply_circuit_open", "apply_recent_failures", + } + clean = {k: v for k, v in data.items() if k in allowed} + if not clean: + return + cols = ", ".join(f"{k} = ?" for k in clean) + vals = list(clean.values()) + [now_iso(), user_id] + with _lock: + _conn.execute(f"UPDATE user_preferences SET {cols}, updated_at = ? WHERE user_id = ?", vals) + + +# --------------------------------------------------------------------------- +# Flats +# --------------------------------------------------------------------------- + +def upsert_flat(payload: dict) -> bool: flat_id = str(payload["id"]) - conn_info = payload.get("connectivity") or {} with _lock: existing = _conn.execute("SELECT id FROM flats WHERE id = ?", (flat_id,)).fetchone() if existing: return False + c = payload.get("connectivity") or {} _conn.execute( - """ - INSERT INTO flats( + """INSERT INTO flats( id, link, address, rooms, size, total_rent, sqm_price, year_built, wbs, connectivity_morning_time, connectivity_night_time, address_link_gmaps, - payload_json, matched_criteria, discovered_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, + payload_json, discovered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", ( - flat_id, - payload.get("link", ""), - payload.get("address", ""), - payload.get("rooms"), - payload.get("size"), - payload.get("total_rent"), - payload.get("sqm_price"), - str(payload.get("year_built", "")), + flat_id, payload.get("link", ""), payload.get("address", ""), + payload.get("rooms"), payload.get("size"), payload.get("total_rent"), + payload.get("sqm_price"), str(payload.get("year_built", "")), str(payload.get("wbs", "")), - conn_info.get("morning_time"), - conn_info.get("night_time"), + c.get("morning_time"), c.get("night_time"), payload.get("address_link_gmaps"), json.dumps(payload, default=str), - 1 if matched else 0, now_iso(), ), ) @@ -147,73 +405,157 @@ def upsert_flat(payload: dict, matched: bool) -> bool: def recent_flats(limit: int = 50) -> list[sqlite3.Row]: - return list( - _conn.execute( - """ - SELECT f.*, - (SELECT success FROM applications a WHERE a.flat_id = f.id - ORDER BY a.started_at DESC LIMIT 1) AS last_application_success, - (SELECT message FROM applications a WHERE a.flat_id = f.id - ORDER BY a.started_at DESC LIMIT 1) AS last_application_message, - (SELECT started_at FROM applications a WHERE a.flat_id = f.id - ORDER BY a.started_at DESC LIMIT 1) AS last_application_at - FROM flats f - ORDER BY f.discovered_at DESC - LIMIT ? - """, - (limit,), - ).fetchall() - ) + return list(_conn.execute( + "SELECT * FROM flats ORDER BY discovered_at DESC LIMIT ?", (limit,) + ).fetchall()) -def get_flat(flat_id: str) -> sqlite3.Row | None: +def get_flat(flat_id: str) -> Optional[sqlite3.Row]: return _conn.execute("SELECT * FROM flats WHERE id = ?", (flat_id,)).fetchone() -def start_application(flat_id: str, url: str, triggered_by: str) -> int: +# --------------------------------------------------------------------------- +# Applications +# --------------------------------------------------------------------------- + +def start_application(user_id: int, flat_id: str, url: str, triggered_by: str, + submit_forms: bool, profile_snapshot: dict) -> int: with _lock: cur = _conn.execute( - "INSERT INTO applications(flat_id, url, triggered_by, started_at) VALUES (?, ?, ?, ?)", - (flat_id, url, triggered_by, now_iso()), + """INSERT INTO applications( + user_id, flat_id, url, triggered_by, submit_forms_used, + started_at, profile_snapshot_json + ) VALUES (?, ?, ?, ?, ?, ?, ?)""", + (user_id, flat_id, url, triggered_by, 1 if submit_forms else 0, + now_iso(), json.dumps(profile_snapshot)), ) return cur.lastrowid -def finish_application(app_id: int, success: bool, message: str) -> None: +def finish_application(app_id: int, success: bool, message: str, + provider: str = "", forensics: Optional[dict] = None) -> None: with _lock: _conn.execute( - "UPDATE applications SET finished_at = ?, success = ?, message = ? WHERE id = ?", - (now_iso(), 1 if success else 0, message, app_id), + """UPDATE applications SET finished_at = ?, success = ?, message = ?, + provider = ?, forensics_json = ? + WHERE id = ?""", + (now_iso(), 1 if success else 0, message, provider, + json.dumps(forensics) if forensics is not None else None, + app_id), ) -def recent_applications(limit: int = 20) -> list[sqlite3.Row]: - return list( - _conn.execute( - """ - SELECT a.*, f.address, f.link - FROM applications a - JOIN flats f ON f.id = a.flat_id - ORDER BY a.started_at DESC - LIMIT ? - """, - (limit,), - ).fetchall() - ) +def get_application(app_id: int) -> Optional[sqlite3.Row]: + return _conn.execute("SELECT * FROM applications WHERE id = ?", (app_id,)).fetchone() -def log_audit(actor: str, action: str, details: str = "", ip: str = "") -> None: +def recent_applications(user_id: Optional[int], limit: int = 50) -> list[sqlite3.Row]: + if user_id is None: + return list(_conn.execute( + """SELECT a.*, f.address, f.link + FROM applications a LEFT JOIN flats f ON f.id = a.flat_id + ORDER BY a.started_at DESC LIMIT ?""", (limit,) + ).fetchall()) + return list(_conn.execute( + """SELECT a.*, f.address, f.link + FROM applications a LEFT JOIN flats f ON f.id = a.flat_id + WHERE a.user_id = ? + ORDER BY a.started_at DESC LIMIT ?""", + (user_id, limit), + ).fetchall()) + + +def last_application_for_flat(user_id: int, flat_id: str) -> Optional[sqlite3.Row]: + return _conn.execute( + """SELECT * FROM applications + WHERE user_id = ? AND flat_id = ? + ORDER BY started_at DESC LIMIT 1""", + (user_id, flat_id), + ).fetchone() + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + +def log_error(source: str, kind: str, summary: str, + user_id: Optional[int] = None, application_id: Optional[int] = None, + context: Optional[dict] = None) -> int: + with _lock: + cur = _conn.execute( + """INSERT INTO errors(timestamp, user_id, source, kind, summary, application_id, context_json) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (now_iso(), user_id, source, kind, summary, application_id, + json.dumps(context) if context else None), + ) + return cur.lastrowid + + +def recent_errors(user_id: Optional[int], limit: int = 100, + include_global: bool = False) -> list[sqlite3.Row]: + if user_id is None: + return list(_conn.execute( + "SELECT * FROM errors ORDER BY timestamp DESC LIMIT ?", (limit,) + ).fetchall()) + if include_global: + return list(_conn.execute( + """SELECT * FROM errors + WHERE user_id = ? OR user_id IS NULL + ORDER BY timestamp DESC LIMIT ?""", + (user_id, limit), + ).fetchall()) + return list(_conn.execute( + "SELECT * FROM errors WHERE user_id = ? ORDER BY timestamp DESC LIMIT ?", + (user_id, limit), + ).fetchall()) + + +def get_error(error_id: int) -> Optional[sqlite3.Row]: + return _conn.execute("SELECT * FROM errors WHERE id = ?", (error_id,)).fetchone() + + +# --------------------------------------------------------------------------- +# Audit log +# --------------------------------------------------------------------------- + +def log_audit(actor: str, action: str, details: str = "", + user_id: Optional[int] = None, ip: str = "") -> None: with _lock: _conn.execute( - "INSERT INTO audit_log(timestamp, actor, action, details, ip) VALUES (?, ?, ?, ?, ?)", - (now_iso(), actor, action, details, ip), + "INSERT INTO audit_log(timestamp, user_id, actor, action, details, ip) " + "VALUES (?, ?, ?, ?, ?, ?)", + (now_iso(), user_id, actor, action, details, ip), ) -def recent_audit(limit: int = 30) -> list[sqlite3.Row]: - return list( - _conn.execute( - "SELECT * FROM audit_log ORDER BY timestamp DESC LIMIT ?", - (limit,), - ).fetchall() - ) +def recent_audit(user_id: Optional[int], limit: int = 100) -> list[sqlite3.Row]: + if user_id is None: + return list(_conn.execute( + "SELECT * FROM audit_log ORDER BY timestamp DESC LIMIT ?", (limit,) + ).fetchall()) + return list(_conn.execute( + "SELECT * FROM audit_log WHERE user_id = ? OR user_id IS NULL ORDER BY timestamp DESC LIMIT ?", + (user_id, limit), + ).fetchall()) + + +# --------------------------------------------------------------------------- +# Retention cleanup +# --------------------------------------------------------------------------- + +def cleanup_retention() -> dict: + cutoff = (datetime.now(timezone.utc) - timedelta(days=RETENTION_DAYS)).isoformat(timespec="seconds") + stats = {} + with _lock: + for table in ("errors", "audit_log"): + cur = _conn.execute(f"DELETE FROM {table} WHERE timestamp < ?", (cutoff,)) + stats[table] = cur.rowcount + # Drop forensics from old applications (but keep the row itself for history) + cur = _conn.execute( + "UPDATE applications SET forensics_json = NULL WHERE started_at < ? AND forensics_json IS NOT NULL", + (cutoff,), + ) + stats["applications_forensics_wiped"] = cur.rowcount + if any(v for v in stats.values()): + logger.info("retention cleanup: %s", stats) + return stats diff --git a/web/matching.py b/web/matching.py new file mode 100644 index 0000000..09bf0ea --- /dev/null +++ b/web/matching.py @@ -0,0 +1,52 @@ +""" +Per-user filter matching. + +Each user has one row in user_filters. A flat matches the user if all +of the user's non-null constraints are satisfied. Empty filters = matches all. +""" +import logging +from typing import Iterable + +logger = logging.getLogger("web.matching") + + +def flat_matches_filter(flat: dict, f: dict | None) -> bool: + """f is a user_filters row converted to dict (or None = no filter set).""" + if not f: + return True + + rooms = flat.get("rooms") or 0.0 + rent = flat.get("total_rent") or 0.0 + size = flat.get("size") or 0.0 + commute = (flat.get("connectivity") or {}).get("morning_time") or 0.0 + wbs_str = str(flat.get("wbs", "")).strip().lower() + + if f.get("rooms_min") is not None and rooms < float(f["rooms_min"]): + return False + if f.get("rooms_max") is not None and rooms > float(f["rooms_max"]): + return False + if f.get("max_rent") is not None and rent > float(f["max_rent"]): + return False + if f.get("min_size") is not None and size < float(f["min_size"]): + return False + if f.get("max_morning_commute") is not None and commute > float(f["max_morning_commute"]): + return False + + wbs_req = (f.get("wbs_required") or "").strip().lower() + if wbs_req == "yes": + if not wbs_str or wbs_str in ("kein", "nein", "no", "ohne", "-"): + return False + elif wbs_req == "no": + if wbs_str and wbs_str not in ("kein", "nein", "no", "ohne", "-"): + return False + + return True + + +def row_to_dict(row) -> dict: + if row is None: + return {} + try: + return {k: row[k] for k in row.keys()} + except Exception: + return dict(row) diff --git a/web/notifications.py b/web/notifications.py new file mode 100644 index 0000000..45facd2 --- /dev/null +++ b/web/notifications.py @@ -0,0 +1,130 @@ +""" +User-level notification dispatcher. + +Channels: +- 'ui' → no-op (dashboard shows the events anyway) +- 'telegram' → per-user bot token + chat id +- 'email' → system SMTP (one outbox, per-user recipient) +""" +import logging +import smtplib +from email.mime.text import MIMEText +from typing import Optional + +import requests + +import db +from settings import ( + PUBLIC_URL, + SMTP_FROM, + SMTP_HOST, + SMTP_PASSWORD, + SMTP_PORT, + SMTP_STARTTLS, + SMTP_USERNAME, +) + +logger = logging.getLogger("web.notifications") + +EventType = str # 'match' | 'apply_ok' | 'apply_fail' + + +def _telegram_send(token: str, chat_id: str, text: str) -> bool: + try: + r = requests.post( + f"https://api.telegram.org/bot{token}/sendMessage", + json={"chat_id": chat_id, "text": text, "parse_mode": "Markdown", + "disable_web_page_preview": True}, + timeout=10, + ) + if not r.ok: + logger.warning("telegram send failed: %s %s", r.status_code, r.text[:200]) + return False + return True + except requests.RequestException as e: + logger.warning("telegram unreachable: %s", e) + return False + + +def _email_send(recipient: str, subject: str, body: str) -> bool: + if not SMTP_HOST or not recipient: + logger.info("email skipped (SMTP_HOST=%r recipient=%r)", SMTP_HOST, recipient) + return False + try: + msg = MIMEText(body, _charset="utf-8") + msg["Subject"] = subject + msg["From"] = SMTP_FROM + msg["To"] = recipient + with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=15) as s: + if SMTP_STARTTLS: + s.starttls() + if SMTP_USERNAME: + s.login(SMTP_USERNAME, SMTP_PASSWORD) + s.send_message(msg) + logger.info("email sent to %s", recipient) + return True + except Exception as e: + logger.warning("email send failed: %s", e) + return False + + +def _should_notify(notif, event: EventType) -> bool: + if event == "match": + return bool(notif["notify_on_match"]) + if event == "apply_ok": + return bool(notif["notify_on_apply_success"]) + if event == "apply_fail": + return bool(notif["notify_on_apply_fail"]) + return False + + +def notify_user(user_id: int, event: EventType, *, + subject: str, body_plain: str, body_markdown: Optional[str] = None) -> None: + """Fire a notification for one user on one event. Best-effort, non-raising.""" + try: + notif = db.get_notifications(user_id) + if not notif or not _should_notify(notif, event): + return + channel = notif["channel"] or "ui" + if channel == "ui": + return + if channel == "telegram": + token = notif["telegram_bot_token"] + chat = notif["telegram_chat_id"] + if token and chat: + _telegram_send(token, chat, body_markdown or body_plain) + elif channel == "email": + addr = notif["email_address"] + if addr: + _email_send(addr, subject, body_plain) + except Exception: + logger.exception("notify_user failed for user=%s event=%s", user_id, event) + + +# -- Convenience builders ----------------------------------------------------- + +def on_match(user_id: int, flat: dict) -> None: + addr = flat.get("address") or flat.get("link") + rent = flat.get("total_rent") + rooms = flat.get("rooms") + link = flat.get("link", "") + body = f"Neue passende Wohnung: {addr}\nMiete: {rent} €\nZimmer: {rooms}\n{link}" + md = (f"*Neue passende Wohnung*\n[{addr}]({link})\n" + f"Miete: {rent} € · Zimmer: {rooms}\n{PUBLIC_URL}") + notify_user(user_id, "match", subject="[lazyflat] passende Wohnung", body_plain=body, body_markdown=md) + + +def on_apply_ok(user_id: int, flat: dict, message: str) -> None: + addr = flat.get("address") or flat.get("link") + body = f"Bewerbung erfolgreich: {addr}\n{message}" + md = f"*Bewerbung erfolgreich*\n{addr}\n{message}" + notify_user(user_id, "apply_ok", subject="[lazyflat] Bewerbung OK", body_plain=body, body_markdown=md) + + +def on_apply_fail(user_id: int, flat: dict, message: str) -> None: + addr = flat.get("address") or flat.get("link") + body = f"Bewerbung fehlgeschlagen: {addr}\n{message}\n{PUBLIC_URL}/fehler" + md = (f"*Bewerbung fehlgeschlagen*\n{addr}\n{message}\n" + f"[Fehler ansehen]({PUBLIC_URL}/fehler)") + notify_user(user_id, "apply_fail", subject="[lazyflat] Bewerbung fehlgeschlagen", + body_plain=body, body_markdown=md) diff --git a/web/retention.py b/web/retention.py new file mode 100644 index 0000000..67cb691 --- /dev/null +++ b/web/retention.py @@ -0,0 +1,24 @@ +"""Background thread that periodically prunes old logs / errors / forensics.""" +import logging +import threading +import time + +import db +from settings import RETENTION_RUN_INTERVAL_SECONDS + +logger = logging.getLogger("web.retention") + + +def _loop(): + while True: + try: + db.cleanup_retention() + except Exception: + logger.exception("retention cleanup failed") + time.sleep(RETENTION_RUN_INTERVAL_SECONDS) + + +def start() -> None: + t = threading.Thread(target=_loop, name="retention", daemon=True) + t.start() + logger.info("retention thread started (interval=%ss)", RETENTION_RUN_INTERVAL_SECONDS) diff --git a/web/settings.py b/web/settings.py index 2f726b1..04c2b81 100644 --- a/web/settings.py +++ b/web/settings.py @@ -16,39 +16,47 @@ def _required(key: str) -> str: return val -# --- Auth --- +# --- Admin bootstrap ---------------------------------------------------------- +# On first boot the web service seeds this user as an admin in the database. +# Afterwards the user record in SQLite is authoritative: changing the hash in +# env does NOT rotate the DB password — use the /einstellungen UI. AUTH_USERNAME: str = _required("AUTH_USERNAME") -# argon2 hash of the password. Generate via: -# python -c "from argon2 import PasswordHasher; print(PasswordHasher().hash(''))" AUTH_PASSWORD_HASH: str = _required("AUTH_PASSWORD_HASH") -# Signs session cookies. If missing -> ephemeral random secret (invalidates sessions on restart). +# --- Session cookie ----------------------------------------------------------- SESSION_SECRET: str = getenv("SESSION_SECRET") or secrets.token_urlsafe(48) SESSION_COOKIE_NAME: str = "lazyflat_session" SESSION_MAX_AGE_SECONDS: int = int(getenv("SESSION_MAX_AGE_SECONDS", str(60 * 60 * 24 * 7))) - -# When behind an HTTPS proxy (Coolify/Traefik) this MUST be true so cookies are Secure. COOKIE_SECURE: bool = getenv("COOKIE_SECURE", "true").lower() in ("true", "1", "yes", "on") -# --- Internal service auth --- +# --- Internal service auth ---------------------------------------------------- INTERNAL_API_KEY: str = _required("INTERNAL_API_KEY") -# --- Apply service --- +# --- Apply service ------------------------------------------------------------ APPLY_URL: str = getenv("APPLY_URL", "http://apply:8000") APPLY_TIMEOUT: int = int(getenv("APPLY_TIMEOUT", "600")) -# Circuit breaker: disable auto-apply after N consecutive apply failures. APPLY_FAILURE_THRESHOLD: int = int(getenv("APPLY_FAILURE_THRESHOLD", "3")) -# --- Storage --- +# --- Storage ------------------------------------------------------------------ DATA_DIR: Path = Path(getenv("DATA_DIR", "/data")) DATA_DIR.mkdir(parents=True, exist_ok=True) DB_PATH: Path = DATA_DIR / "lazyflat.sqlite" -# --- Rate limiting --- +# Retention (errors / audit / application forensics). Default 14 days. +RETENTION_DAYS: int = int(getenv("RETENTION_DAYS", "14")) +RETENTION_RUN_INTERVAL_SECONDS: int = int(getenv("RETENTION_RUN_INTERVAL_SECONDS", str(60 * 60))) + +# --- Rate limiting ------------------------------------------------------------ LOGIN_RATE_LIMIT: int = int(getenv("LOGIN_RATE_LIMIT", "5")) LOGIN_RATE_WINDOW_SECONDS: int = int(getenv("LOGIN_RATE_WINDOW_SECONDS", "900")) -# --- Filter criteria (mirrored from original flat-alert) --- -FILTER_ROOMS: list[float] = [float(r) for r in getenv("FILTER_ROOMS", "2.0,2.5").split(",") if r.strip()] -FILTER_MAX_RENT: float = float(getenv("FILTER_MAX_RENT", "1500")) -FILTER_MAX_MORNING_COMMUTE: float = float(getenv("FILTER_MAX_MORNING_COMMUTE", "50")) +# --- Email (system-wide SMTP for notifications) ------------------------------- +SMTP_HOST: str = getenv("SMTP_HOST", "") +SMTP_PORT: int = int(getenv("SMTP_PORT", "587")) +SMTP_USERNAME: str = getenv("SMTP_USERNAME", "") +SMTP_PASSWORD: str = getenv("SMTP_PASSWORD", "") +SMTP_FROM: str = getenv("SMTP_FROM", "lazyflat@localhost") +SMTP_STARTTLS: bool = getenv("SMTP_STARTTLS", "true").lower() in ("true", "1", "yes", "on") + +# --- App URL (used to build links in notifications) --------------------------- +PUBLIC_URL: str = getenv("PUBLIC_URL", "https://flat.lab.moritz.run") diff --git a/web/templates/_dashboard_body.html b/web/templates/_dashboard_body.html deleted file mode 100644 index af1bc73..0000000 --- a/web/templates/_dashboard_body.html +++ /dev/null @@ -1,176 +0,0 @@ -
-
-
alert
-
- {% if last_alert_heartbeat %} - live - {% else %} - kein Heartbeat - {% endif %} -
-
letzter Heartbeat: {{ last_alert_heartbeat or "—" }}
-
- -
-
apply
-
- {% if apply_reachable %} - reachable - {% else %} - down - {% endif %} -
-
- {% if circuit_open %} - circuit open - {% elif apply_failures > 0 %} - {{ apply_failures }} recent failure(s) - {% else %} - healthy - {% endif %} -
-
- -
-
Modus
-
- {% if mode == "auto" %} - full-auto - {% else %} - manuell - {% endif %} -
-
- - - -
-
- -
-
Kill‑Switch
-
- {% if kill_switch %} - apply gestoppt - {% else %} - aktiv - {% endif %} -
-
- - - -
- {% if circuit_open %} -
- - -
- {% endif %} -
-
- -{% if not apply_allowed %} -
- apply blockiert - {{ apply_block_reason }} -
-{% endif %} - -
-
-

Wohnungen

- {{ flats|length }} zuletzt gesehen -
-
- {% for flat in flats %} -
-
-
- - {{ flat.address or flat.link }} - - {% if flat.matched_criteria %} - match - {% else %} - info - {% endif %} - {% if flat.last_application_success == 1 %} - beworben - {% elif flat.last_application_success == 0 %} - apply fehlgeschlagen - {% endif %} -
-
- {% if flat.rooms %}{{ "%.1f"|format(flat.rooms) }} Z{% endif %} - {% if flat.size %} · {{ "%.0f"|format(flat.size) }} m²{% endif %} - {% if flat.total_rent %} · {{ "%.0f"|format(flat.total_rent) }} €{% endif %} - {% if flat.sqm_price %} ({{ "%.2f"|format(flat.sqm_price) }} €/m²){% endif %} - {% if flat.connectivity_morning_time %} · {{ "%.0f"|format(flat.connectivity_morning_time) }} min morgens{% endif %} - · entdeckt {{ flat.discovered_at }} -
- {% if flat.last_application_message %} -
↳ {{ flat.last_application_message }}
- {% endif %} -
-
- {% if apply_allowed and not flat.last_application_success %} -
- - - -
- {% endif %} -
-
- {% else %} -
Noch keine Wohnungen gesehen.
- {% endfor %} -
-
- -
-
-

Letzte Bewerbungen

-
- {% for a in applications %} -
-
- {% if a.success == 1 %}ok - {% elif a.success == 0 %}fail - {% else %}läuft{% endif %} - {{ a.triggered_by }} - {{ a.started_at }} -
-
{{ a.address or a.url }}
- {% if a.message %}
{{ a.message }}
{% endif %} -
- {% else %} -
Keine Bewerbungen bisher.
- {% endfor %} -
-
- -
-

Audit-Log

-
- {% for e in audit %} -
- {{ e.timestamp }} - {{ e.actor }} - {{ e.action }} - {% if e.details %}— {{ e.details }}{% endif %} -
- {% else %} -
leer
- {% endfor %} -
-
-
diff --git a/web/templates/_layout.html b/web/templates/_layout.html new file mode 100644 index 0000000..ce63ba7 --- /dev/null +++ b/web/templates/_layout.html @@ -0,0 +1,31 @@ +{# + Shared layout: top bar with brand + user + logout, tab nav, body container. + Used by every authenticated view via `{% extends "_layout.html" %}`. +#} +{% extends "base.html" %} +{% block body %} +
+
+
+
+

lazyflat

+
+
+ {{ user.username }}{% if is_admin %} · admin{% endif %} +
+ +
+
+
+ +
+
+ {% block content %}{% endblock %} +
+{% endblock %} diff --git a/web/templates/_settings_account.html b/web/templates/_settings_account.html new file mode 100644 index 0000000..46767b9 --- /dev/null +++ b/web/templates/_settings_account.html @@ -0,0 +1,27 @@ +

Account

+ +
+ Angemeldet als {{ user.username }}{% if is_admin %} (Administrator){% endif %}. +
+ +{% if request.query_params.get('ok') %}
Passwort geändert.
{% endif %} +{% if request.query_params.get('err') == 'wrongold' %}
Altes Passwort falsch.
{% endif %} +{% if request.query_params.get('err') == 'mismatch' %}
Neue Passwörter stimmen nicht überein.
{% endif %} +{% if request.query_params.get('err') == 'tooshort' %}
Passwort zu kurz (min. 10 Zeichen).
{% endif %} + +
+ +
+ + +
+
+ + +
+
+ + +
+ +
diff --git a/web/templates/_settings_filter.html b/web/templates/_settings_filter.html new file mode 100644 index 0000000..47f3efb --- /dev/null +++ b/web/templates/_settings_filter.html @@ -0,0 +1,40 @@ +

Filter

+

+ Die Filter bestimmen, bei welchen Wohnungen du eine Benachrichtigung bekommst und worauf Auto-Bewerben greift. + Leer lassen = kein Limit. +

+ +
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
diff --git a/web/templates/_settings_notifications.html b/web/templates/_settings_notifications.html new file mode 100644 index 0000000..202ec67 --- /dev/null +++ b/web/templates/_settings_notifications.html @@ -0,0 +1,51 @@ +

Benachrichtigungen

+

+ Wähle einen Kanal und entscheide, welche Events dich erreichen sollen. +

+ +
+ + +
+ + +
+ +
+ + +

Bot bei @BotFather anlegen, Token hier eintragen.

+
+
+ + +
+ +
+ + +
+ +
+ + + +
+ + +
diff --git a/web/templates/_settings_profil.html b/web/templates/_settings_profil.html new file mode 100644 index 0000000..faf0bdc --- /dev/null +++ b/web/templates/_settings_profil.html @@ -0,0 +1,123 @@ +

Bewerbungsdaten

+

+ Diese Angaben werden beim Bewerben an die jeweilige Website gesendet. + sensibel — werden nur in der DB gespeichert und pro Bewerbung als Snapshot protokolliert. +

+ +
+ + +
+ + +
+
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+

WBS

+
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + +
+

Immomio-Login (optional)

+

+ Wird von Anbietern benötigt, die über immomio/tenant vermitteln (z.B. gesobau.de). +

+
+
+ + +
+
+ + +
+ +
+ +
+
+ +
+

Formulare wirklich absenden?

+

+ experimentell + Im Dry-Run-Modus füllt apply das Formular aus und stoppt vor „Senden". Nur einschalten, wenn du jeden Anbieter einmal im Dry-Run verifiziert hast. +

+
+ + + +
+
+ + + +
diff --git a/web/templates/_settings_users.html b/web/templates/_settings_users.html new file mode 100644 index 0000000..6295f05 --- /dev/null +++ b/web/templates/_settings_users.html @@ -0,0 +1,50 @@ +

Benutzer verwalten

+ +{% if request.query_params.get('ok') %}
Benutzer angelegt.
{% endif %} +{% if request.query_params.get('err') == 'exists' %}
Benutzername existiert bereits.
{% endif %} + +
+
+

Neuen Benutzer anlegen

+
+ +
+ + +
+
+ + +
+ + +
+
+ +
+

Alle Benutzer

+
+ {% for u in users %} +
+ {{ u.username }} + {% if u.is_admin %}admin{% endif %} + {% if u.disabled %}deaktiviert + {% else %}aktiv{% endif %} + {% if u.id != user.id %} +
+ + + + +
+ {% endif %} +
+ {% endfor %} +
+
+
diff --git a/web/templates/base.html b/web/templates/base.html index a1135ec..36b7b01 100644 --- a/web/templates/base.html +++ b/web/templates/base.html @@ -9,19 +9,13 @@ - {% block body %}{% endblock %} +{% block body %}{% endblock %} diff --git a/web/templates/bewerbung_detail.html b/web/templates/bewerbung_detail.html new file mode 100644 index 0000000..320fbae --- /dev/null +++ b/web/templates/bewerbung_detail.html @@ -0,0 +1,113 @@ +{% extends "_layout.html" %} +{% block title %}Bewerbung #{{ application.id }} — lazyflat{% endblock %} +{% block content %} +← zurück zu den Bewerbungen + +
+
+

Bewerbung #{{ application.id }}

+ {% if application.success == 1 %}erfolgreich + {% elif application.success == 0 %}fehlgeschlagen + {% else %}läuft{% endif %} + {{ application.triggered_by }} + {% if application.provider %}{{ application.provider }}{% endif %} + {% if application.submit_forms_used %}echt gesendet + {% else %}dry-run{% endif %} +
+
+ +
gestartet: {{ application.started_at }}
+
beendet: {{ application.finished_at or "—" }}
+ {% if application.message %}
Meldung: {{ application.message }}
{% endif %} +
+
+ +
+
+ Profil-Snapshot zum Bewerbungszeitpunkt + 
{{ profile_snapshot | tojson(indent=2) }}
+
+
+ +{% if forensics %} +
+

Forensik (für KI-Debug)

+ +
+ Step-Log ({{ forensics.steps|length }} Einträge, {{ forensics.duration_s }} s) +
+ {% for s in forensics.steps %} +
+ [{{ "%.2f"|format(s.ts) }}s] {{ s.step }} {% if s.status != 'ok' %}({{ s.status }}){% endif %} + {% if s.detail %}— {{ s.detail }}{% endif %} +
+ {% endfor %} +
+
+ + {% if forensics.screenshots %} +
+ Screenshots ({{ forensics.screenshots|length }}) +
+ {% for s in forensics.screenshots %} +
+
{{ s.label }} @ {{ "%.2f"|format(s.ts) }}s — {{ s.url }}
+ {{ s.label }} +
+ {% endfor %} +
+
+ {% endif %} + + {% if forensics.console %} +
+ Browser-Konsole ({{ forensics.console|length }}) +
+ {% for c in forensics.console %} +
[{{ "%.2f"|format(c.ts) }}s] [{{ c.type }}] {{ c.text }}
+ {% endfor %} +
+
+ {% endif %} + + {% if forensics.errors %} +
+ Browser-Errors ({{ forensics.errors|length }}) +
+ {% for e in forensics.errors %} +
[{{ "%.2f"|format(e.ts) }}s] {{ e.message }}
+ {% endfor %} +
+
+ {% endif %} + + {% if forensics.network %} +
+ Netzwerk ({{ forensics.network|length }}) +
+ {% for n in forensics.network %} +
+ [{{ "%.2f"|format(n.ts) }}s] + {% if n.kind == 'response' %}← {{ n.status }} {{ n.url }} + {% if n.body_snippet %}
{{ n.body_snippet[:500] }}
{% endif %} + {% else %}→ {{ n.method }} {{ n.url }} [{{ n.resource_type }}] + {% endif %} +
+ {% endfor %} +
+
+ {% endif %} + + {% if forensics.final_html %} +
+ Page HTML ({{ forensics.final_html|length }} B) + 
{{ forensics.final_html }}
+
+ {% endif %} +
+{% elif application.finished_at %} +
+ Forensik für diese Bewerbung ist nicht mehr verfügbar (älter als Retention-Zeitraum). +
+{% endif %} +{% endblock %} diff --git a/web/templates/bewerbungen.html b/web/templates/bewerbungen.html new file mode 100644 index 0000000..f55b826 --- /dev/null +++ b/web/templates/bewerbungen.html @@ -0,0 +1,32 @@ +{% extends "_layout.html" %} +{% block title %}Bewerbungen — lazyflat{% endblock %} +{% block content %} +
+
+

Meine Bewerbungen

+ {{ applications|length }} +
+
+ {% for a in applications %} +
+
+ {% if a.success == 1 %}ok + {% elif a.success == 0 %}fail + {% else %}läuft{% endif %} + {{ a.triggered_by }} + {% if a.provider %}{{ a.provider }}{% endif %} + {% if a.submit_forms_used %}echt gesendet + {% else %}dry-run{% endif %} + {{ a.started_at }} +
+
+ #{{ a.id }} — {{ a.address or a.url }} +
+ {% if a.message %}
{{ a.message }}
{% endif %} +
+ {% else %} +
Noch keine Bewerbungen.
+ {% endfor %} +
+
+{% endblock %} diff --git a/web/templates/dashboard.html b/web/templates/dashboard.html deleted file mode 100644 index e79bab5..0000000 --- a/web/templates/dashboard.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "base.html" %} -{% block title %}lazyflat dashboard{% endblock %} -{% block body %} -
-
-
-
-

lazyflat

-
-
- {{ user }} -
- -
-
-
-
- -
-
- {% include "_dashboard_body.html" %} -
-
-{% endblock %} diff --git a/web/templates/einstellungen.html b/web/templates/einstellungen.html new file mode 100644 index 0000000..c297e53 --- /dev/null +++ b/web/templates/einstellungen.html @@ -0,0 +1,23 @@ +{% extends "_layout.html" %} +{% block title %}Einstellungen — lazyflat{% endblock %} +{% block content %} +
+ + +
+ {% if section == 'profil' %}{% include "_settings_profil.html" %} + {% elif section == 'filter' %}{% include "_settings_filter.html" %} + {% elif section == 'benachrichtigungen' %}{% include "_settings_notifications.html" %} + {% elif section == 'account' %}{% include "_settings_account.html" %} + {% elif section == 'benutzer' %}{% include "_settings_users.html" %} + {% endif %} +
+
+{% endblock %} diff --git a/web/templates/fehler.html b/web/templates/fehler.html new file mode 100644 index 0000000..33ae51b --- /dev/null +++ b/web/templates/fehler.html @@ -0,0 +1,29 @@ +{% extends "_layout.html" %} +{% block title %}Fehler — lazyflat{% endblock %} +{% block content %} +
+
+

Fehler {% if is_admin %}inkl. globaler{% endif %}

+ {{ errors|length }} +
+
+ {% for e in errors %} + +
+ {{ e.kind }} + {{ e.source }} + {% if e.application_id %}#{{ e.application_id }}{% endif %} + {{ e.timestamp }} +
+
{{ e.summary or "(kein Text)" }}
+ + {% else %} +
Keine Fehler — läuft rund.
+ {% endfor %} +
+
+ Fehler werden 14 Tage aufbewahrt. Bei fehlgeschlagenen Bewerbungen enthält die Detailseite Screenshots, + Step-Log, Browser-Konsole + Netzwerk-Trace für KI-gestützte Fehleranalyse. +
+
+{% endblock %} diff --git a/web/templates/fehler_detail.html b/web/templates/fehler_detail.html new file mode 100644 index 0000000..a62b569 --- /dev/null +++ b/web/templates/fehler_detail.html @@ -0,0 +1,32 @@ +{% extends "_layout.html" %} +{% block title %}Fehler #{{ error.id }} — lazyflat{% endblock %} +{% block content %} +← zurück zu den Fehlern + +
+
+

Fehler #{{ error.id }}

+ {{ error.kind }} + {{ error.source }} +
+
+ {{ error.timestamp }} · {{ error.summary or "(kein Text)" }} +
+ {% if context %} +
+ Kontext + 
{{ context | tojson(indent=2) }}
+
+ {% endif %} +
+ +{% if application %} +
+

Zugehörige Bewerbung

+
+ Bewerbung #{{ application.id }} öffnen + — vollständige Forensik dort. +
+
+{% endif %} +{% endblock %} diff --git a/web/templates/logs.html b/web/templates/logs.html new file mode 100644 index 0000000..a6d034a --- /dev/null +++ b/web/templates/logs.html @@ -0,0 +1,23 @@ +{% extends "_layout.html" %} +{% block title %}Logs — lazyflat{% endblock %} +{% block content %} +
+

Meine Aktionen (Audit-Log)

+
+ {% for e in events %} +
+ {{ e.timestamp }} + {{ e.actor }} + {{ e.action }} + {% if e.details %}— {{ e.details }}{% endif %} + {% if e.ip %} [{{ e.ip }}]{% endif %} +
+ {% else %} +
Keine Log-Einträge.
+ {% endfor %} +
+
+ Einträge werden nach 14 Tagen automatisch gelöscht. +
+
+{% endblock %} diff --git a/web/templates/wohnungen.html b/web/templates/wohnungen.html new file mode 100644 index 0000000..b415a99 --- /dev/null +++ b/web/templates/wohnungen.html @@ -0,0 +1,182 @@ +{% extends "_layout.html" %} +{% block title %}Wohnungen — lazyflat{% endblock %} +{% block content %} + + +
+
+
Auto-Bewerben
+
+ {% if auto_apply_enabled %} + an + bei Match wird automatisch beworben + {% else %} + aus + Matches werden nur angezeigt + {% endif %} +
+
+ +
+ +
+ + + +
+ + +
+ + + +
+ + {% if circuit_open %} +
+ + +
+ {% endif %} +
+
+ +{% if not apply_allowed %} +
+ apply blockiert + {{ apply_block_reason }} +
+{% endif %} + + +
+
+
alert
+
+ {% if last_alert_heartbeat %}live + {% else %}kein Heartbeat{% endif %} +
+
+
+
apply
+
+ {% if apply_reachable %}ok + {% else %}down{% endif %} +
+
+
+
submit_forms
+
+ {% if submit_forms %}echt senden + {% else %}dry-run{% endif %} +
+
+
+
Fehler in Serie
+
+ {% if circuit_open %}circuit open + {% elif apply_failures > 0 %}{{ apply_failures }} + {% else %}0{% endif %} +
+
+
+ + +
+ Eigene Filter +
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + Leer lassen = kein Limit. Filter bestimmen Match-Hervorhebung + Auto-Bewerben. +
+
+
+ + +
+
+

Neueste Wohnungen auf inberlinwohnen.de

+ {{ flats|length }} gesamt +
+
+ {% for item in flats %} + {% set f = item.row %} +
+
+
+ + {{ f.address or f.link }} + + {% if item.matched %}match{% endif %} + {% if item.last and item.last.success == 1 %}beworben + {% elif item.last and item.last.success == 0 %}apply fehlgeschlagen + {% elif item.last %}läuft{% endif %} +
+
+ {% if f.rooms %}{{ "%.1f"|format(f.rooms) }} Z{% endif %} + {% if f.size %} · {{ "%.0f"|format(f.size) }} m²{% endif %} + {% if f.total_rent %} · {{ "%.0f"|format(f.total_rent) }} €{% endif %} + {% if f.sqm_price %} ({{ "%.2f"|format(f.sqm_price) }} €/m²){% endif %} + {% if f.connectivity_morning_time %} · {{ "%.0f"|format(f.connectivity_morning_time) }} min morgens{% endif %} + {% if f.wbs %} · WBS: {{ f.wbs }}{% endif %} + · entdeckt {{ f.discovered_at }} +
+ {% if item.last and item.last.message %} +
↳ {{ item.last.message }}
+ {% endif %} +
+
+ {% if apply_allowed and not (item.last and item.last.success == 1) %} +
+ + + +
+ {% endif %} +
+
+ {% else %} +
Noch keine Wohnungen entdeckt.
+ {% endfor %} +
+
+ +{% endblock %}