From a32cc84a9013ea1ce0ff6af3ba831872021f4dc5 Mon Sep 17 00:00:00 2001 From: Vysyr Date: Mon, 31 Aug 2026 20:57:14 +0200 Subject: [PATCH] sync --- app.py | 47 ++++++++++++++++++++++--- scraper.py | 11 +++++- static/app.js | 83 ++++++++++++++++++++++++++++++++++---------- static/style.css | 12 ++++++- templates/index.html | 9 ++++- tests/test_app.py | 23 ++++++++++++ 6 files changed, 159 insertions(+), 26 deletions(-) diff --git a/app.py b/app.py index 2522b47..382feca 100644 --- a/app.py +++ b/app.py @@ -1,7 +1,8 @@ import os +import secrets import time from collections import defaultdict, deque -from threading import Lock +from threading import Lock, Thread from flask import Flask, jsonify, render_template, request from scraper import ScrapeError, get_rating_distribution, validate_maps_url @@ -11,6 +12,8 @@ app.config["MAX_CONTENT_LENGTH"] = 4096 _scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi _rate_lock = Lock() _requests = defaultdict(deque) +_jobs = {} +_jobs_lock = Lock() def _rate_limited(ip, limit=6, window=60): @@ -60,16 +63,52 @@ def reviews(): return jsonify(error=str(exc)), 400 if not _scrape_lock.acquire(blocking=False): return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429 + + job_id = secrets.token_urlsafe(24) + with _jobs_lock: + cutoff = time.monotonic() - 600 + for old_id in [key for key, job in _jobs.items() if job["created"] < cutoff]: + del _jobs[old_id] + _jobs[job_id] = { + "status": "running", "progress": 0, + "message": "Rozpoczynam sprawdzanie", "created": time.monotonic(), + } + + Thread(target=_run_job, args=(job_id, url), daemon=True).start() + return jsonify(job_id=job_id), 202 + + +def _run_job(job_id, url): + def update(percent, message): + with _jobs_lock: + _jobs[job_id].update(progress=percent, message=message) + try: - return jsonify(get_rating_distribution(url)) + result = get_rating_distribution(url, progress=update) + with _jobs_lock: + _jobs[job_id].update(status="complete", progress=100, message="Gotowe", result=result) except ScrapeError as exc: - return jsonify(error=str(exc)), 422 + with _jobs_lock: + _jobs[job_id].update(status="error", message=str(exc)) except Exception: app.logger.exception("Unexpected scraper failure") - return jsonify(error="Google Maps nie odpowiedziało poprawnie. Spróbuj później."), 502 + with _jobs_lock: + _jobs[job_id].update( + status="error", message="Google Maps nie odpowiedziało poprawnie. Spróbuj później." + ) finally: _scrape_lock.release() +@app.get("/api/reviews/") +def review_status(job_id): + with _jobs_lock: + job = _jobs.get(job_id) + if not job: + return jsonify(error="Nie znaleziono tego sprawdzenia."), 404 + response = {key: value for key, value in job.items() if key != "created"} + return jsonify(response) + + if __name__ == "__main__": app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False) diff --git a/scraper.py b/scraper.py index 2c9cd78..045db51 100644 --- a/scraper.py +++ b/scraper.py @@ -1,5 +1,6 @@ import os import re +from collections.abc import Callable from urllib.parse import urlsplit, urlunsplit from playwright.sync_api import TimeoutError as PlaywrightTimeoutError @@ -98,8 +99,12 @@ def _open_reviews(page): pass -def get_rating_distribution(url: str) -> dict: +def get_rating_distribution( + url: str, + progress: Callable[[int, str], None] | None = None, +) -> dict: """Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane.""" + report = progress or (lambda _percent, _message: None) try: with sync_playwright() as playwright: headless = os.environ.get("CHROMIUM_HEADLESS", "1").lower() not in {"0", "false", "no"} @@ -131,13 +136,16 @@ def get_rating_distribution(url: str) -> dict: browser = playwright.chromium.launch(**launch) context = browser.new_context(locale="pl-PL", viewport={"width": 1100, "height": 800}) try: + report(50, "Otwarto Chromium") page = context.pages[0] if context.pages else context.new_page() context.add_init_script( "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" ) page.goto(url, wait_until="domcontentloaded", timeout=30_000) _accept_cookies(page) + report(75, "Załadowano Google Maps") _open_reviews(page) + report(90, "Otwarto panel opinii") page.wait_for_timeout(700) labels = page.locator("[aria-label]").evaluate_all( "els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)" @@ -148,6 +156,7 @@ def get_rating_distribution(url: str) -> dict: if "ograniczonego widoku" in body_text or "limited view" in body_text: raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.") raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.") + report(99, "Pojawiły się liczniki") ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)} total = sum(ratings.values()) average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0 diff --git a/static/app.js b/static/app.js index 0eab8de..f6b224d 100644 --- a/static/app.js +++ b/static/app.js @@ -1,28 +1,73 @@ const form = document.querySelector('#review-form'); const statusBox = document.querySelector('#status'); const resultBox = document.querySelector('#result'); +const progressBox = document.querySelector('#progress-box'); +const progressBar = document.querySelector('#progress-bar'); +const progressLabel = document.querySelector('#progress-label'); +const progressValue = document.querySelector('#progress-value'); const button = form.querySelector('button'); +const wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); + +function setProgress(percent, message) { + progressBar.value = percent; + progressBar.textContent = `${percent}%`; + progressValue.textContent = `${percent}%`; + progressLabel.textContent = message; +} + +function showResult(data) { + document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL'); + document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2}); + const max = Math.max(...Object.values(data.ratings), 1); + const rows = Object.entries(data.ratings).map(([stars, count]) => { + const row = document.createElement('div'); row.className = 'rating-row'; + const label = document.createElement('span'); label.textContent = `${stars} ★`; + const bar = document.createElement('progress'); + bar.className = 'rating-bar'; bar.max = max; bar.value = count; + const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL'); + row.append(label, bar, value); return row; + }); + document.querySelector('#ratings').replaceChildren(...rows); + resultBox.hidden = false; +} + form.addEventListener('submit', async (event) => { - event.preventDefault(); button.disabled = true; resultBox.hidden = true; - statusBox.className = 'status'; statusBox.textContent = 'Odczytuję dokładne liczniki…'; + event.preventDefault(); + button.disabled = true; + resultBox.hidden = true; + statusBox.textContent = ''; + statusBox.className = 'status'; + progressBox.hidden = false; + setProgress(0, 'Rozpoczynam sprawdzanie'); + try { - const response = await fetch('/api/reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({url: document.querySelector('#maps-url').value})}); - const data = await response.json(); - if (!response.ok) throw new Error(data.error || 'Nie udało się pobrać danych.'); - document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL'); - document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2}); - const max = Math.max(...Object.values(data.ratings), 1); - const rows = Object.entries(data.ratings).map(([stars, count]) => { - const row = document.createElement('div'); row.className = 'rating-row'; - const label = document.createElement('span'); label.textContent = `${stars} ★`; - const bar = document.createElement('div'); bar.className = 'bar'; - const fill = document.createElement('i'); fill.style.width = `${count / max * 100}%`; bar.append(fill); - const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL'); - row.append(label, bar, value); return row; + const startResponse = await fetch('/api/reviews', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({url: document.querySelector('#maps-url').value}) }); - document.querySelector('#ratings').replaceChildren(...rows); - statusBox.textContent = ''; resultBox.hidden = false; - } catch (error) { statusBox.className = 'status error'; statusBox.textContent = error.message; } - finally { button.disabled = false; } + const startData = await startResponse.json(); + if (!startResponse.ok) throw new Error(startData.error || 'Nie udało się rozpocząć sprawdzania.'); + + while (true) { + const response = await fetch(`/api/reviews/${encodeURIComponent(startData.job_id)}`, {cache: 'no-store'}); + const job = await response.json(); + if (!response.ok) throw new Error(job.error || 'Nie udało się sprawdzić postępu.'); + setProgress(job.progress, job.message); + if (job.status === 'complete') { + showResult(job.result); + progressBox.hidden = true; + break; + } + if (job.status === 'error') throw new Error(job.message); + await wait(350); + } + } catch (error) { + progressBox.hidden = true; + statusBox.className = 'status error'; + statusBox.textContent = error.message; + } finally { + button.disabled = false; + } }); diff --git a/static/style.css b/static/style.css index db2d9ef..2f77c24 100644 --- a/static/style.css +++ b/static/style.css @@ -10,9 +10,19 @@ input:focus { outline: 3px solid #1a73e822; border-color: #1a73e8; } button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; } button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; } .status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; } +.progress-box { margin-top: 22px; } +.progress-caption { display: flex; justify-content: space-between; gap: 16px; margin-bottom: 9px; color: #59636e; font-size: .9rem; } +.progress-caption strong { color: #1a73e8; font-variant-numeric: tabular-nums; } +#progress-bar, .rating-bar { display: block; width: 100%; overflow: hidden; border: 0; border-radius: 99px; background: #edf0f2; appearance: none; } +#progress-bar { height: 12px; } +#progress-bar::-webkit-progress-bar, .rating-bar::-webkit-progress-bar { background: #edf0f2; border-radius: 99px; } +#progress-bar::-webkit-progress-value { background: #1a73e8; border-radius: 99px; transition: width .35s ease; } +#progress-bar::-moz-progress-bar { background: #1a73e8; border-radius: 99px; transition: width .35s ease; } .result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; } .summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; } .rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; } .rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; } -.bar { height: 10px; overflow: hidden; border-radius: 99px; background: #edf0f2; } .bar i { display: block; height: 100%; border-radius: inherit; background: #fbbc04; } +.rating-bar { height: 10px; } +.rating-bar::-webkit-progress-value { background: #fbbc04; border-radius: 99px; } +.rating-bar::-moz-progress-bar { background: #fbbc04; border-radius: 99px; } @media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } } diff --git a/templates/index.html b/templates/index.html index f0ea5a9..f86cdde 100644 --- a/templates/index.html +++ b/templates/index.html @@ -18,7 +18,14 @@ -

+ +