From 0aa4c6c2bbf440025bd641e2255e32d9345690d0 Mon Sep 17 00:00:00 2001 From: EiSiMo Date: Tue, 21 Apr 2026 15:29:55 +0200 Subject: [PATCH] enrichment: drop LLM for structured info, dedup images by sha + phash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per user request, the LLM is no longer asked to extract rooms/size/rent/WBS — those come from the inberlinwohnen.de scraper which is reliable. Haiku is now used for one narrow job: pick which URLs from the listing page are actual flat photos (vs. logos, badges, ads, employee portraits). On any LLM failure the unfiltered candidate list passes through. Image dedup runs in two tiers: 1. SHA256 of bytes — drops different URLs that point to byte-identical files 2. Perceptual hash (Pillow + imagehash, Hamming distance ≤ 5) — drops the "same image at a different resolution" duplicates from srcset / CDN variants that were filling galleries with 2–4× copies UI: - Wohnungsliste falls back to scraper-only display (rooms/size/rent/wbs) - Detail panel only shows images + "Zur Original-Anzeige →"; description / features / pros & cons / kv table are gone - Per-row "erneut versuchen" link + the "analysiert…/?" status chips were tied to LLM extraction and are removed; the header "Bilder nachladen (N)" button still surfaces pending/failed batches for admins Co-Authored-By: Claude Opus 4.7 (1M context) --- web/app.py | 15 +--- web/enrichment.py | 117 ++++++++++++++++++----------- web/llm.py | 105 ++++++++++---------------- web/requirements.txt | 2 + web/templates/_wohnung_detail.html | 77 +++---------------- web/templates/_wohnungen_body.html | 54 +++---------- 6 files changed, 137 insertions(+), 233 deletions(-) diff --git a/web/app.py b/web/app.py index 2999224..7ce4dd9 100644 --- a/web/app.py +++ b/web/app.py @@ -403,13 +403,7 @@ def _wohnungen_context(user) -> dict: }, filters): continue last = db.last_application_for_flat(uid, f["id"]) - enrichment_data = None - if f["enrichment_json"]: - try: - enrichment_data = json.loads(f["enrichment_json"]) - except Exception: - enrichment_data = None - flats_view.append({"row": f, "last": last, "enrichment": enrichment_data}) + flats_view.append({"row": f, "last": last}) rejected_view = db.rejected_flats(uid) enrichment_counts = db.enrichment_counts() @@ -489,12 +483,6 @@ def partial_wohnung_detail(request: Request, flat_id: str, user=Depends(require_ flat = db.get_flat(flat_id) if not flat: raise HTTPException(404) - enrichment_data = None - if flat["enrichment_json"]: - try: - enrichment_data = json.loads(flat["enrichment_json"]) - except Exception: - enrichment_data = None slug = enrichment.flat_slug(flat_id) image_urls = [ f"/flat-images/{slug}/{i}" @@ -503,7 +491,6 @@ def partial_wohnung_detail(request: Request, flat_id: str, user=Depends(require_ ctx = { "request": request, "flat": flat, - "enrichment": enrichment_data, "enrichment_status": flat["enrichment_status"], "image_urls": image_urls, } diff --git a/web/enrichment.py b/web/enrichment.py index 56f7f30..fb9bc42 100644 --- a/web/enrichment.py +++ b/web/enrichment.py @@ -1,19 +1,20 @@ -"""Flat-enrichment pipeline. +"""Image enrichment pipeline. For each new flat we: 1. Ask the apply service to fetch the listing via Playwright (bypasses bot guards) -2. Feed the HTML to Haiku via `llm.extract_flat_details` → structured dict -3. Download each image URL directly into /data/flats//NN. -4. Persist result on the flat row (enrichment_json + image_count + status) +2. Optionally let Haiku narrow the URL list down to actual flat photos +3. Download the images into /data/flats//NN., deduplicating by + exact bytes (SHA256) and visual similarity (perceptual hash) -Kicked as a detached asyncio task from /internal/flats so scraping stays fast. -Every failure is caught, stashed in enrichment_json as {"_error": "...", ...} -and mirrored into the errors log so /logs/protokoll explains what went wrong. +Structured fields (rooms / size / rent / WBS) come from the inberlinwohnen.de +scraper and are not re-derived by the LLM. Failures land in +enrichment_status='failed' with the reason in enrichment_json._error. """ from __future__ import annotations import asyncio import hashlib +import io import logging import mimetypes import os @@ -33,12 +34,12 @@ IMAGES_DIR = DATA_DIR / "flats" IMAGES_DIR.mkdir(parents=True, exist_ok=True) MAX_IMAGES = 12 -MAX_IMAGE_BYTES = 3_000_000 # 3 MB per image +MAX_IMAGE_BYTES = 5_000_000 IMAGE_TIMEOUT = 15 +PHASH_DUPLICATE_THRESHOLD = 5 # Hamming distance ≤ N → considered the same picture class EnrichmentError(Exception): - """Raised by each pipeline step with a human-readable reason.""" def __init__(self, step: str, reason: str): self.step = step self.reason = reason @@ -46,7 +47,6 @@ class EnrichmentError(Exception): def flat_slug(flat_id: str) -> str: - """Filesystem-safe short identifier for a flat (IDs are URLs).""" return hashlib.sha1(flat_id.encode("utf-8")).hexdigest()[:16] @@ -77,25 +77,50 @@ def _fetch_listing(url: str) -> dict: raise EnrichmentError("fetch", "apply returned non-JSON response") -def _ext_from_response(resp: requests.Response, url: str) -> str: - ct = resp.headers.get("content-type", "").split(";")[0].strip().lower() +def _ext_for(content_type: str, url: str) -> str: + ct = content_type.split(";")[0].strip().lower() if ct: ext = mimetypes.guess_extension(ct) or "" if ext: return ext.replace(".jpe", ".jpg") - path = urlparse(url).path - _, ext = os.path.splitext(path) + _, ext = os.path.splitext(urlparse(url).path) return ext.lower() or ".jpg" +def _phash(data: bytes): + """Return an imagehash.ImageHash for the bytes, or None if unsupported.""" + try: + from PIL import Image + import imagehash + except ImportError: + return None + try: + with Image.open(io.BytesIO(data)) as img: + img.load() + return imagehash.phash(img) + except Exception: + return None + + def _download_images(flat_id: str, urls: list[str], referer: str) -> int: + """Download up to MAX_IMAGES distinct flat photos. Dedup tiers: + 1. Skip URLs that fail / aren't images + 2. Skip exact byte-equal duplicates (different URL pointing to same file) + 3. Skip visually-equivalent images (same picture re-encoded at a + different size — common with srcset/CDN variants) + """ d = flat_image_dir(flat_id) for old in d.iterdir(): try: old.unlink() except OSError: pass + seen_sha: set[str] = set() + seen_phash: list = [] # list of imagehash objects + saved = 0 - for raw_url in urls[:MAX_IMAGES]: + for raw_url in urls: + if saved >= MAX_IMAGES: + break try: r = requests.get( raw_url, @@ -109,48 +134,53 @@ def _download_images(flat_id: str, urls: list[str], referer: str) -> int: ct = r.headers.get("content-type", "").split(";")[0].strip().lower() if not ct.startswith("image/"): continue - ext = _ext_from_response(r, raw_url) - path = d / f"{saved + 1:02d}{ext}" - total = 0 - with open(path, "wb") as f: - for chunk in r.iter_content(chunk_size=65_536): - if not chunk: - continue - total += len(chunk) - if total > MAX_IMAGE_BYTES: - break - f.write(chunk) - if total == 0: - path.unlink(missing_ok=True) + data = bytearray() + for chunk in r.iter_content(chunk_size=65_536): + if not chunk: + continue + data.extend(chunk) + if len(data) > MAX_IMAGE_BYTES: + break + if not data: continue - saved += 1 except requests.RequestException as e: logger.info("image download failed %s: %s", raw_url, e) continue + + data_bytes = bytes(data) + + sha = hashlib.sha256(data_bytes).hexdigest() + if sha in seen_sha: + continue + seen_sha.add(sha) + + ph = _phash(data_bytes) + if ph is not None: + if any((ph - prev) <= PHASH_DUPLICATE_THRESHOLD for prev in seen_phash): + continue + seen_phash.append(ph) + + ext = _ext_for(ct, raw_url) + path = d / f"{saved + 1:02d}{ext}" + path.write_bytes(data_bytes) + saved += 1 + return saved def enrich_flat_sync(flat_id: str) -> None: - """Run the full enrichment pipeline for one flat. Blocking.""" flat = db.get_flat(flat_id) if not flat: return url = flat["link"] - logger.info("enrich start flat=%s url=%s", flat_id, url) + logger.info("enrich start flat=%s", flat_id) try: listing = _fetch_listing(url) - html = listing.get("html") or "" - final_url = listing.get("final_url") or url - if not html.strip(): - raise EnrichmentError("fetch", "apply returned empty HTML") - - details = llm.extract_flat_details(html, final_url) - if details is None: - raise EnrichmentError("llm", "model returned no tool_use or call failed (see web logs)") - - image_urls = listing.get("image_urls") or [] - image_count = _download_images(flat_id, image_urls, referer=url) + candidates = listing.get("image_urls") or [] + if candidates: + candidates = llm.select_flat_image_urls(candidates, listing.get("final_url") or url) + image_count = _download_images(flat_id, candidates, referer=url) except EnrichmentError as e: _record_failure(flat_id, e.step, e.reason) return @@ -159,7 +189,7 @@ def enrich_flat_sync(flat_id: str) -> None: _record_failure(flat_id, "crash", f"{type(e).__name__}: {e}") return - db.set_flat_enrichment(flat_id, "ok", enrichment=details, image_count=image_count) + db.set_flat_enrichment(flat_id, "ok", enrichment=None, image_count=image_count) logger.info("enrich done flat=%s images=%d", flat_id, image_count) @@ -179,7 +209,6 @@ def _record_failure(flat_id: str, step: str, reason: str) -> None: def kick(flat_id: str) -> None: - """Fire-and-forget enrichment in a background thread.""" asyncio.create_task(asyncio.to_thread(enrich_flat_sync, flat_id)) diff --git a/web/llm.py b/web/llm.py index 491f0d3..73afa88 100644 --- a/web/llm.py +++ b/web/llm.py @@ -1,12 +1,14 @@ -"""Minimal Anthropic Messages API wrapper for flat enrichment. +"""Anthropic Haiku helper — used only to pick which `` URLs on a +listing page are actual photos of the flat (vs. nav icons, badges, ads…). -Uses tool-use forced output so Haiku returns structured JSON instead of free -text we'd have to regex. No SDK — plain `requests` is enough here. +If the API key is missing or the call fails, the caller passes the original +candidates straight through, so this is a soft enhancement, not a +dependency. """ from __future__ import annotations import logging -from typing import Any, Optional +from typing import Optional import requests @@ -17,68 +19,41 @@ logger = logging.getLogger("web.llm") API_URL = "https://api.anthropic.com/v1/messages" API_VERSION = "2023-06-01" -TOOL_NAME = "record_flat_details" -TOOL_SCHEMA: dict[str, Any] = { +TOOL_NAME = "select_flat_images" +TOOL_SCHEMA = { "type": "object", "properties": { - "address": {"type": ["string", "null"], - "description": "Full street address incl. postcode+city if present"}, - "rooms": {"type": ["number", "null"], "description": "Number of rooms (decimal ok)"}, - "size_sqm": {"type": ["number", "null"], "description": "Size in m²"}, - "rent_cold": {"type": ["number", "null"], "description": "Kaltmiete in €"}, - "rent_total": {"type": ["number", "null"], "description": "Warm/Gesamtmiete in €"}, - "utilities": {"type": ["number", "null"], "description": "Nebenkosten in €"}, - "deposit": {"type": ["number", "null"], "description": "Kaution in €"}, - "available_from": {"type": ["string", "null"], "description": "Bezugsfrei ab (text)"}, - "floor": {"type": ["string", "null"], "description": "Etage (text, z.B. '3. OG')"}, - "heating": {"type": ["string", "null"]}, - "energy_certificate": {"type": ["string", "null"]}, - "energy_value": {"type": ["string", "null"]}, - "year_built": {"type": ["string", "null"]}, - "wbs_required": {"type": ["boolean", "null"]}, - "wbs_type": {"type": ["string", "null"], "description": "WBS-Typ, z.B. '160' oder null"}, - "description": { - "type": ["string", "null"], - "description": "Kurze 2–3-Satz-Beschreibung der Wohnung auf Deutsch. Fakten, keine Werbesprache.", - }, - "features": { - "type": "array", "items": {"type": "string"}, - "description": "Ausstattungsmerkmale (z.B. 'Balkon', 'Einbauküche', 'Parkett')", - }, - "pros": { - "type": "array", "items": {"type": "string"}, - "description": "2–4 konkrete Vorteile aus Bewerbersicht (keine Werbung)", - }, - "cons": { - "type": "array", "items": {"type": "string"}, - "description": "2–4 mögliche Nachteile / Punkte zum Beachten", + "urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Subset of the candidate URLs that show the actual flat — " + "interior, exterior, floorplan. Keep ordering of input.", }, }, - "required": [], + "required": ["urls"], "additionalProperties": False, } SYSTEM_PROMPT = ( - "Du extrahierst strukturierte Wohnungsdaten aus deutschem HTML-Quelltext von " - "Berliner Wohnungsbaugesellschaften (howoge, gewobag, degewo, gesobau, wbm, " - "stadt-und-land). Antworte AUSSCHLIESSLICH über den bereitgestellten Tool-Call. " - "Fehlende Werte → null. Keine Erfindungen — wenn etwas nicht klar aus dem HTML " - "hervorgeht, lass das Feld null. Zahlen bitte als Zahlen (nicht als String), " - "Beschreibung/Pros/Cons auf Deutsch." + "Du bekommst eine Liste von Bild-URLs einer Wohnungsanzeige. Wähle nur " + "die URLs aus, die ein Foto der Wohnung zeigen (Innenraum, Außenansicht " + "des Gebäudes, Grundriss). Verwerfe Logos, Icons, Banner, Ads, " + "Bewertungs-Sterne, Karten/Stadtpläne, Mitarbeiter-Portraits, Tracking-" + "Pixel. Behalte die Reihenfolge der Input-Liste bei. Antworte " + "ausschließlich über den Tool-Call." ) -def extract_flat_details(html: str, url: str, - max_html_chars: int = 60_000, - timeout: int = 60) -> Optional[dict]: - """Call Haiku; return the structured dict or None on failure.""" - if not ANTHROPIC_API_KEY: - logger.info("skipping enrichment: ANTHROPIC_API_KEY not set") - return None +def select_flat_image_urls(candidates: list[str], page_url: str, + timeout: int = 30) -> list[str]: + """Return the LLM-filtered subset, or the original list on any failure.""" + if not ANTHROPIC_API_KEY or not candidates: + return candidates - user_content = ( - f"URL: {url}\n\n" - f"HTML-Quellcode (ggf. gekürzt):\n---\n{html[:max_html_chars]}\n---" + user_text = ( + f"Seite: {page_url}\n\n" + "Kandidaten-URLs (nummeriert):\n" + + "\n".join(f"{i+1}. {u}" for i, u in enumerate(candidates)) ) body = { "model": ANTHROPIC_MODEL, @@ -86,11 +61,11 @@ def extract_flat_details(html: str, url: str, "system": SYSTEM_PROMPT, "tools": [{ "name": TOOL_NAME, - "description": "Persist the extracted flat details.", + "description": "Persist the selected flat-photo URLs.", "input_schema": TOOL_SCHEMA, }], "tool_choice": {"type": "tool", "name": TOOL_NAME}, - "messages": [{"role": "user", "content": user_content}], + "messages": [{"role": "user", "content": user_text}], } try: r = requests.post( @@ -104,16 +79,18 @@ def extract_flat_details(html: str, url: str, timeout=timeout, ) except requests.RequestException as e: - logger.warning("anthropic request failed: %s", e) - return None - + logger.warning("anthropic image-select request failed: %s", e) + return candidates if r.status_code >= 400: - logger.warning("anthropic %s: %s", r.status_code, r.text[:300]) - return None + logger.warning("anthropic image-select %s: %s", r.status_code, r.text[:300]) + return candidates data = r.json() for block in data.get("content", []): if block.get("type") == "tool_use" and block.get("name") == TOOL_NAME: - return block.get("input") or {} - logger.warning("anthropic returned no tool_use block: %s", data) - return None + urls = (block.get("input") or {}).get("urls") or [] + # Constrain to the original candidate set so the model can't + # invent URLs (it sometimes lightly rewrites them otherwise). + allowed = set(candidates) + return [u for u in urls if u in allowed] + return candidates diff --git a/web/requirements.txt b/web/requirements.txt index 56293ef..a54f003 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -6,3 +6,5 @@ itsdangerous==2.2.0 python-multipart==0.0.17 python-dotenv==1.0.1 requests==2.32.5 +Pillow==11.0.0 +ImageHash==4.3.1 diff --git a/web/templates/_wohnung_detail.html b/web/templates/_wohnung_detail.html index b8d9152..e3ea633 100644 --- a/web/templates/_wohnung_detail.html +++ b/web/templates/_wohnung_detail.html @@ -1,14 +1,13 @@ -{# Expanded detail for a single flat, loaded into #flat-detail- via HTMX. #} +{# Expanded detail for a single flat — only shows downloaded images. #} {% if enrichment_status == 'pending' %} -
Analyse läuft – kommt in wenigen Augenblicken zurück…
+
Bilder werden abgerufen…
{% elif enrichment_status == 'failed' %}
- Detail-Analyse konnte nicht abgerufen werden. - Zur Original-Anzeige → + Bilder konnten nicht geladen werden. + Zur Original-Anzeige →
-{% else %} -
- {% if image_urls %} +{% elif image_urls %} +
{% for src in image_urls %} @@ -16,67 +15,13 @@ {% endfor %}
- {% endif %} - - {% if enrichment and enrichment.description %} -

{{ enrichment.description }}

- {% endif %} - - {% if enrichment %} -
- {% macro kv(label, value) %} - {% if value is not none and value != '' %} -
- {{ label }} - {{ value }} -
- {% endif %} - {% endmacro %} - {{ kv('Adresse', enrichment.address) }} - {{ kv('Zimmer', enrichment.rooms) }} - {{ kv('Größe', enrichment.size_sqm ~ ' m²' if enrichment.size_sqm else none) }} - {{ kv('Kaltmiete', enrichment.rent_cold ~ ' €' if enrichment.rent_cold else none) }} - {{ kv('Nebenkosten', enrichment.utilities ~ ' €' if enrichment.utilities else none) }} - {{ kv('Gesamtmiete', enrichment.rent_total ~ ' €' if enrichment.rent_total else none) }} - {{ kv('Kaution', enrichment.deposit ~ ' €' if enrichment.deposit else none) }} - {{ kv('Bezugsfrei ab', enrichment.available_from) }} - {{ kv('Etage', enrichment.floor) }} - {{ kv('Heizung', enrichment.heating) }} - {{ kv('Energieausweis', enrichment.energy_certificate) }} - {{ kv('Energiewert', enrichment.energy_value) }} - {{ kv('Baujahr', enrichment.year_built) }} - {{ kv('WBS', 'erforderlich' if enrichment.wbs_required else ('nicht erforderlich' if enrichment.wbs_required == false else none)) }} - {{ kv('WBS-Typ', enrichment.wbs_type) }} -
- {% endif %} - - {% if enrichment and enrichment.features %} -
- {% for f in enrichment.features %}{{ f }}{% endfor %} -
- {% endif %} - -
- {% if enrichment and enrichment.pros %} -
-
Pro
-
    - {% for p in enrichment.pros %}
  • + {{ p }}
    • {% endfor %} -
-
- {% endif %} - {% if enrichment and enrichment.cons %} -
-
Contra
-
    - {% for c in enrichment.cons %}
  • − {{ c }}
    • {% endfor %} -
-
- {% endif %} -
-
Zur Original-Anzeige →
+{% else %} +
+ Keine Bilder gefunden. + Zur Original-Anzeige → +
{% endif %} diff --git a/web/templates/_wohnungen_body.html b/web/templates/_wohnungen_body.html index fda8748..cc14ba6 100644 --- a/web/templates/_wohnungen_body.html +++ b/web/templates/_wohnungen_body.html @@ -90,8 +90,8 @@ hx-post="/actions/enrich-all" hx-target="#wohnungen-body" hx-swap="outerHTML"> {% endif %} @@ -133,52 +133,16 @@ {% elif item.last and item.last.success == 1 %}beworben {% elif item.last and item.last.success == 0 %}fehlgeschlagen {% endif %} - {% if f.enrichment_status == 'pending' %}analysiert… - {% elif f.enrichment_status == 'failed' %}? - {% endif %}
- {% if f.enrichment_status == 'pending' %} - Infos werden abgerufen… - · … - {% elif f.enrichment_status == 'failed' %} - {% set err = (item.enrichment or {}).get('_error') or 'unbekannt' %} - Fehler beim Abrufen der Infos - {% if is_admin %} -
- - - -
- {% endif %} - · … - {% else %} - {# LLM first, scraper as fallback. The scraper data - from inberlinwohnen.de is reliable; we only - replace it when the LLM has a concrete value. #} - {% set e = item.enrichment or {} %} - {% set rooms = e.rooms if e.rooms is not none else f.rooms %} - {% set size = e.size_sqm if e.size_sqm is not none else f.size %} - {% set rent = e.rent_total or e.rent_cold or f.total_rent %} - {% if e.wbs_required is sameas true %} - {% set wbs_label = 'WBS: ' ~ (e.wbs_type or 'erforderlich') %} - {% elif e.wbs_required is sameas false %} - {% set wbs_label = 'ohne WBS' %} - {% elif f.wbs == 'erforderlich' %} - {% set wbs_label = 'WBS: erforderlich' %} - {% elif f.wbs == 'nicht erforderlich' %} - {% set wbs_label = 'ohne WBS' %} - {% else %} - {% set wbs_label = '' %} - {% endif %} - {% set parts = [] %} - {% if rooms %}{% set _ = parts.append('%g Z'|format(rooms)) %}{% endif %} - {% if size %}{% set _ = parts.append('%.0f m²'|format(size)) %}{% endif %} - {% if rent %}{% set _ = parts.append('%.0f €'|format(rent)) %}{% endif %} - {% if wbs_label %}{% set _ = parts.append(wbs_label) %}{% endif %} - {{ parts|join(' · ') }}{% if parts %} · {% endif %}… + {% set parts = [] %} + {% if f.rooms %}{% set _ = parts.append('%g Z'|format(f.rooms)) %}{% endif %} + {% if f.size %}{% set _ = parts.append('%.0f m²'|format(f.size)) %}{% endif %} + {% if f.total_rent %}{% set _ = parts.append('%.0f €'|format(f.total_rent)) %}{% endif %} + {% if f.wbs == 'erforderlich' %}{% set _ = parts.append('WBS: erforderlich') %} + {% elif f.wbs == 'nicht erforderlich' %}{% set _ = parts.append('ohne WBS') %} {% endif %} + {{ parts|join(' · ') }}{% if parts %} · {% endif %}…