This commit is contained in:
2026-08-31 20:57:14 +02:00
parent 94fb6a8568
commit a32cc84a90
6 changed files with 159 additions and 26 deletions
+43 -4
View File
@@ -1,7 +1,8 @@
import os import os
import secrets
import time import time
from collections import defaultdict, deque from collections import defaultdict, deque
from threading import Lock from threading import Lock, Thread
from flask import Flask, jsonify, render_template, request from flask import Flask, jsonify, render_template, request
from scraper import ScrapeError, get_rating_distribution, validate_maps_url from scraper import ScrapeError, get_rating_distribution, validate_maps_url
@@ -11,6 +12,8 @@ app.config["MAX_CONTENT_LENGTH"] = 4096
_scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi _scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi
_rate_lock = Lock() _rate_lock = Lock()
_requests = defaultdict(deque) _requests = defaultdict(deque)
_jobs = {}
_jobs_lock = Lock()
def _rate_limited(ip, limit=6, window=60): def _rate_limited(ip, limit=6, window=60):
@@ -60,16 +63,52 @@ def reviews():
return jsonify(error=str(exc)), 400 return jsonify(error=str(exc)), 400
if not _scrape_lock.acquire(blocking=False): if not _scrape_lock.acquire(blocking=False):
return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429 return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429
job_id = secrets.token_urlsafe(24)
with _jobs_lock:
cutoff = time.monotonic() - 600
for old_id in [key for key, job in _jobs.items() if job["created"] < cutoff]:
del _jobs[old_id]
_jobs[job_id] = {
"status": "running", "progress": 0,
"message": "Rozpoczynam sprawdzanie", "created": time.monotonic(),
}
Thread(target=_run_job, args=(job_id, url), daemon=True).start()
return jsonify(job_id=job_id), 202
def _run_job(job_id, url):
def update(percent, message):
with _jobs_lock:
_jobs[job_id].update(progress=percent, message=message)
try: try:
return jsonify(get_rating_distribution(url)) result = get_rating_distribution(url, progress=update)
with _jobs_lock:
_jobs[job_id].update(status="complete", progress=100, message="Gotowe", result=result)
except ScrapeError as exc: except ScrapeError as exc:
return jsonify(error=str(exc)), 422 with _jobs_lock:
_jobs[job_id].update(status="error", message=str(exc))
except Exception: except Exception:
app.logger.exception("Unexpected scraper failure") app.logger.exception("Unexpected scraper failure")
return jsonify(error="Google Maps nie odpowiedziało poprawnie. Spróbuj później."), 502 with _jobs_lock:
_jobs[job_id].update(
status="error", message="Google Maps nie odpowiedziało poprawnie. Spróbuj później."
)
finally: finally:
_scrape_lock.release() _scrape_lock.release()
@app.get("/api/reviews/<job_id>")
def review_status(job_id):
with _jobs_lock:
job = _jobs.get(job_id)
if not job:
return jsonify(error="Nie znaleziono tego sprawdzenia."), 404
response = {key: value for key, value in job.items() if key != "created"}
return jsonify(response)
if __name__ == "__main__": if __name__ == "__main__":
app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False) app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False)
+10 -1
View File
@@ -1,5 +1,6 @@
import os import os
import re import re
from collections.abc import Callable
from urllib.parse import urlsplit, urlunsplit from urllib.parse import urlsplit, urlunsplit
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
@@ -98,8 +99,12 @@ def _open_reviews(page):
pass pass
def get_rating_distribution(url: str) -> dict: def get_rating_distribution(
url: str,
progress: Callable[[int, str], None] | None = None,
) -> dict:
"""Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane.""" """Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane."""
report = progress or (lambda _percent, _message: None)
try: try:
with sync_playwright() as playwright: with sync_playwright() as playwright:
headless = os.environ.get("CHROMIUM_HEADLESS", "1").lower() not in {"0", "false", "no"} headless = os.environ.get("CHROMIUM_HEADLESS", "1").lower() not in {"0", "false", "no"}
@@ -131,13 +136,16 @@ def get_rating_distribution(url: str) -> dict:
browser = playwright.chromium.launch(**launch) browser = playwright.chromium.launch(**launch)
context = browser.new_context(locale="pl-PL", viewport={"width": 1100, "height": 800}) context = browser.new_context(locale="pl-PL", viewport={"width": 1100, "height": 800})
try: try:
report(50, "Otwarto Chromium")
page = context.pages[0] if context.pages else context.new_page() page = context.pages[0] if context.pages else context.new_page()
context.add_init_script( context.add_init_script(
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
) )
page.goto(url, wait_until="domcontentloaded", timeout=30_000) page.goto(url, wait_until="domcontentloaded", timeout=30_000)
_accept_cookies(page) _accept_cookies(page)
report(75, "Załadowano Google Maps")
_open_reviews(page) _open_reviews(page)
report(90, "Otwarto panel opinii")
page.wait_for_timeout(700) page.wait_for_timeout(700)
labels = page.locator("[aria-label]").evaluate_all( labels = page.locator("[aria-label]").evaluate_all(
"els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)" "els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)"
@@ -148,6 +156,7 @@ def get_rating_distribution(url: str) -> dict:
if "ograniczonego widoku" in body_text or "limited view" in body_text: if "ograniczonego widoku" in body_text or "limited view" in body_text:
raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.") raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.")
raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.") raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.")
report(99, "Pojawiły się liczniki")
ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)} ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)}
total = sum(ratings.values()) total = sum(ratings.values())
average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0 average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0
+57 -12
View File
@@ -1,28 +1,73 @@
const form = document.querySelector('#review-form'); const form = document.querySelector('#review-form');
const statusBox = document.querySelector('#status'); const statusBox = document.querySelector('#status');
const resultBox = document.querySelector('#result'); const resultBox = document.querySelector('#result');
const progressBox = document.querySelector('#progress-box');
const progressBar = document.querySelector('#progress-bar');
const progressLabel = document.querySelector('#progress-label');
const progressValue = document.querySelector('#progress-value');
const button = form.querySelector('button'); const button = form.querySelector('button');
form.addEventListener('submit', async (event) => { const wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
event.preventDefault(); button.disabled = true; resultBox.hidden = true;
statusBox.className = 'status'; statusBox.textContent = 'Odczytuję dokładne liczniki…'; function setProgress(percent, message) {
try { progressBar.value = percent;
const response = await fetch('/api/reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({url: document.querySelector('#maps-url').value})}); progressBar.textContent = `${percent}%`;
const data = await response.json(); progressValue.textContent = `${percent}%`;
if (!response.ok) throw new Error(data.error || 'Nie udało się pobrać danych.'); progressLabel.textContent = message;
}
function showResult(data) {
document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL'); document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL');
document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2}); document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2});
const max = Math.max(...Object.values(data.ratings), 1); const max = Math.max(...Object.values(data.ratings), 1);
const rows = Object.entries(data.ratings).map(([stars, count]) => { const rows = Object.entries(data.ratings).map(([stars, count]) => {
const row = document.createElement('div'); row.className = 'rating-row'; const row = document.createElement('div'); row.className = 'rating-row';
const label = document.createElement('span'); label.textContent = `${stars}`; const label = document.createElement('span'); label.textContent = `${stars}`;
const bar = document.createElement('div'); bar.className = 'bar'; const bar = document.createElement('progress');
const fill = document.createElement('i'); fill.style.width = `${count / max * 100}%`; bar.append(fill); bar.className = 'rating-bar'; bar.max = max; bar.value = count;
const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL'); const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL');
row.append(label, bar, value); return row; row.append(label, bar, value); return row;
}); });
document.querySelector('#ratings').replaceChildren(...rows); document.querySelector('#ratings').replaceChildren(...rows);
statusBox.textContent = ''; resultBox.hidden = false; resultBox.hidden = false;
} catch (error) { statusBox.className = 'status error'; statusBox.textContent = error.message; } }
finally { button.disabled = false; }
form.addEventListener('submit', async (event) => {
event.preventDefault();
button.disabled = true;
resultBox.hidden = true;
statusBox.textContent = '';
statusBox.className = 'status';
progressBox.hidden = false;
setProgress(0, 'Rozpoczynam sprawdzanie');
try {
const startResponse = await fetch('/api/reviews', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({url: document.querySelector('#maps-url').value})
});
const startData = await startResponse.json();
if (!startResponse.ok) throw new Error(startData.error || 'Nie udało się rozpocząć sprawdzania.');
while (true) {
const response = await fetch(`/api/reviews/${encodeURIComponent(startData.job_id)}`, {cache: 'no-store'});
const job = await response.json();
if (!response.ok) throw new Error(job.error || 'Nie udało się sprawdzić postępu.');
setProgress(job.progress, job.message);
if (job.status === 'complete') {
showResult(job.result);
progressBox.hidden = true;
break;
}
if (job.status === 'error') throw new Error(job.message);
await wait(350);
}
} catch (error) {
progressBox.hidden = true;
statusBox.className = 'status error';
statusBox.textContent = error.message;
} finally {
button.disabled = false;
}
}); });
+11 -1
View File
@@ -10,9 +10,19 @@ input:focus { outline: 3px solid #1a73e822; border-color: #1a73e8; }
button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; } button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; }
button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; } button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; }
.status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; } .status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; }
.progress-box { margin-top: 22px; }
.progress-caption { display: flex; justify-content: space-between; gap: 16px; margin-bottom: 9px; color: #59636e; font-size: .9rem; }
.progress-caption strong { color: #1a73e8; font-variant-numeric: tabular-nums; }
#progress-bar, .rating-bar { display: block; width: 100%; overflow: hidden; border: 0; border-radius: 99px; background: #edf0f2; appearance: none; }
#progress-bar { height: 12px; }
#progress-bar::-webkit-progress-bar, .rating-bar::-webkit-progress-bar { background: #edf0f2; border-radius: 99px; }
#progress-bar::-webkit-progress-value { background: #1a73e8; border-radius: 99px; transition: width .35s ease; }
#progress-bar::-moz-progress-bar { background: #1a73e8; border-radius: 99px; transition: width .35s ease; }
.result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; } .result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; }
.summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; } .summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; }
.rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; } .rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; }
.rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; } .rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; }
.bar { height: 10px; overflow: hidden; border-radius: 99px; background: #edf0f2; } .bar i { display: block; height: 100%; border-radius: inherit; background: #fbbc04; } .rating-bar { height: 10px; }
.rating-bar::-webkit-progress-value { background: #fbbc04; border-radius: 99px; }
.rating-bar::-moz-progress-bar { background: #fbbc04; border-radius: 99px; }
@media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } } @media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } }
+8 -1
View File
@@ -18,7 +18,14 @@
<button type="submit">Sprawdź</button> <button type="submit">Sprawdź</button>
</div> </div>
</form> </form>
<p id="status" class="status" role="status" aria-live="polite"></p> <div id="progress-box" class="progress-box" role="status" aria-live="polite" hidden>
<div class="progress-caption">
<span id="progress-label">Rozpoczynam sprawdzanie</span>
<strong id="progress-value">0%</strong>
</div>
<progress id="progress-bar" max="100" value="0">0%</progress>
</div>
<p id="status" class="status" role="alert" aria-live="assertive"></p>
<section id="result" class="result" hidden> <section id="result" class="result" hidden>
<div class="summary"> <div class="summary">
<div><span id="total">0</span><small>opinii łącznie</small></div> <div><span id="total">0</span><small>opinii łącznie</small></div>
+23
View File
@@ -1,4 +1,6 @@
import unittest import unittest
import time
from unittest.mock import patch
from app import app from app import app
from scraper import parse_distribution_labels, validate_maps_url from scraper import parse_distribution_labels, validate_maps_url
@@ -49,6 +51,27 @@ class ApiTests(unittest.TestCase):
response = app.test_client().post("/api/reviews", data=b"x" * 5000, content_type="application/json") response = app.test_client().post("/api/reviews", data=b"x" * 5000, content_type="application/json")
self.assertEqual(response.status_code, 413) self.assertEqual(response.status_code, 413)
def test_reports_job_progress_and_result(self):
expected = {"total": 3, "average": 4.0, "ratings": {"5": 2, "4": 0, "3": 0, "2": 0, "1": 1}}
def fake_scrape(_url, progress):
progress(50, "Otwarto Chromium")
progress(99, "Pojawiły się liczniki")
return expected
client = app.test_client()
with patch("app.get_rating_distribution", side_effect=fake_scrape):
start = client.post("/api/reviews", json={"url": "https://maps.app.goo.gl/USRCto35jhoVtmj4A"})
self.assertEqual(start.status_code, 202)
job_id = start.get_json()["job_id"]
for _ in range(100):
job = client.get(f"/api/reviews/{job_id}").get_json()
if job["status"] == "complete":
break
time.sleep(0.005)
self.assertEqual(job["progress"], 100)
self.assertEqual(job["result"], expected)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()