sync
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from threading import Lock
|
||||
from threading import Lock, Thread
|
||||
|
||||
from flask import Flask, jsonify, render_template, request
|
||||
from scraper import ScrapeError, get_rating_distribution, validate_maps_url
|
||||
@@ -11,6 +12,8 @@ app.config["MAX_CONTENT_LENGTH"] = 4096
|
||||
_scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi
|
||||
_rate_lock = Lock()
|
||||
_requests = defaultdict(deque)
|
||||
_jobs = {}
|
||||
_jobs_lock = Lock()
|
||||
|
||||
|
||||
def _rate_limited(ip, limit=6, window=60):
|
||||
@@ -60,16 +63,52 @@ def reviews():
|
||||
return jsonify(error=str(exc)), 400
|
||||
if not _scrape_lock.acquire(blocking=False):
|
||||
return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429
|
||||
|
||||
job_id = secrets.token_urlsafe(24)
|
||||
with _jobs_lock:
|
||||
cutoff = time.monotonic() - 600
|
||||
for old_id in [key for key, job in _jobs.items() if job["created"] < cutoff]:
|
||||
del _jobs[old_id]
|
||||
_jobs[job_id] = {
|
||||
"status": "running", "progress": 0,
|
||||
"message": "Rozpoczynam sprawdzanie", "created": time.monotonic(),
|
||||
}
|
||||
|
||||
Thread(target=_run_job, args=(job_id, url), daemon=True).start()
|
||||
return jsonify(job_id=job_id), 202
|
||||
|
||||
|
||||
def _run_job(job_id, url):
|
||||
def update(percent, message):
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update(progress=percent, message=message)
|
||||
|
||||
try:
|
||||
return jsonify(get_rating_distribution(url))
|
||||
result = get_rating_distribution(url, progress=update)
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update(status="complete", progress=100, message="Gotowe", result=result)
|
||||
except ScrapeError as exc:
|
||||
return jsonify(error=str(exc)), 422
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update(status="error", message=str(exc))
|
||||
except Exception:
|
||||
app.logger.exception("Unexpected scraper failure")
|
||||
return jsonify(error="Google Maps nie odpowiedziało poprawnie. Spróbuj później."), 502
|
||||
with _jobs_lock:
|
||||
_jobs[job_id].update(
|
||||
status="error", message="Google Maps nie odpowiedziało poprawnie. Spróbuj później."
|
||||
)
|
||||
finally:
|
||||
_scrape_lock.release()
|
||||
|
||||
|
||||
@app.get("/api/reviews/<job_id>")
|
||||
def review_status(job_id):
|
||||
with _jobs_lock:
|
||||
job = _jobs.get(job_id)
|
||||
if not job:
|
||||
return jsonify(error="Nie znaleziono tego sprawdzenia."), 404
|
||||
response = {key: value for key, value in job.items() if key != "created"}
|
||||
return jsonify(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False)
|
||||
|
||||
+10
-1
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
@@ -98,8 +99,12 @@ def _open_reviews(page):
|
||||
pass
|
||||
|
||||
|
||||
def get_rating_distribution(url: str) -> dict:
|
||||
def get_rating_distribution(
|
||||
url: str,
|
||||
progress: Callable[[int, str], None] | None = None,
|
||||
) -> dict:
|
||||
"""Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane."""
|
||||
report = progress or (lambda _percent, _message: None)
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
headless = os.environ.get("CHROMIUM_HEADLESS", "1").lower() not in {"0", "false", "no"}
|
||||
@@ -131,13 +136,16 @@ def get_rating_distribution(url: str) -> dict:
|
||||
browser = playwright.chromium.launch(**launch)
|
||||
context = browser.new_context(locale="pl-PL", viewport={"width": 1100, "height": 800})
|
||||
try:
|
||||
report(50, "Otwarto Chromium")
|
||||
page = context.pages[0] if context.pages else context.new_page()
|
||||
context.add_init_script(
|
||||
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
||||
)
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=30_000)
|
||||
_accept_cookies(page)
|
||||
report(75, "Załadowano Google Maps")
|
||||
_open_reviews(page)
|
||||
report(90, "Otwarto panel opinii")
|
||||
page.wait_for_timeout(700)
|
||||
labels = page.locator("[aria-label]").evaluate_all(
|
||||
"els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)"
|
||||
@@ -148,6 +156,7 @@ def get_rating_distribution(url: str) -> dict:
|
||||
if "ograniczonego widoku" in body_text or "limited view" in body_text:
|
||||
raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.")
|
||||
raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.")
|
||||
report(99, "Pojawiły się liczniki")
|
||||
ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)}
|
||||
total = sum(ratings.values())
|
||||
average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0
|
||||
|
||||
+64
-19
@@ -1,28 +1,73 @@
|
||||
const form = document.querySelector('#review-form');
|
||||
const statusBox = document.querySelector('#status');
|
||||
const resultBox = document.querySelector('#result');
|
||||
const progressBox = document.querySelector('#progress-box');
|
||||
const progressBar = document.querySelector('#progress-bar');
|
||||
const progressLabel = document.querySelector('#progress-label');
|
||||
const progressValue = document.querySelector('#progress-value');
|
||||
const button = form.querySelector('button');
|
||||
|
||||
const wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
function setProgress(percent, message) {
|
||||
progressBar.value = percent;
|
||||
progressBar.textContent = `${percent}%`;
|
||||
progressValue.textContent = `${percent}%`;
|
||||
progressLabel.textContent = message;
|
||||
}
|
||||
|
||||
function showResult(data) {
|
||||
document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL');
|
||||
document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2});
|
||||
const max = Math.max(...Object.values(data.ratings), 1);
|
||||
const rows = Object.entries(data.ratings).map(([stars, count]) => {
|
||||
const row = document.createElement('div'); row.className = 'rating-row';
|
||||
const label = document.createElement('span'); label.textContent = `${stars} ★`;
|
||||
const bar = document.createElement('progress');
|
||||
bar.className = 'rating-bar'; bar.max = max; bar.value = count;
|
||||
const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL');
|
||||
row.append(label, bar, value); return row;
|
||||
});
|
||||
document.querySelector('#ratings').replaceChildren(...rows);
|
||||
resultBox.hidden = false;
|
||||
}
|
||||
|
||||
form.addEventListener('submit', async (event) => {
|
||||
event.preventDefault(); button.disabled = true; resultBox.hidden = true;
|
||||
statusBox.className = 'status'; statusBox.textContent = 'Odczytuję dokładne liczniki…';
|
||||
event.preventDefault();
|
||||
button.disabled = true;
|
||||
resultBox.hidden = true;
|
||||
statusBox.textContent = '';
|
||||
statusBox.className = 'status';
|
||||
progressBox.hidden = false;
|
||||
setProgress(0, 'Rozpoczynam sprawdzanie');
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({url: document.querySelector('#maps-url').value})});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Nie udało się pobrać danych.');
|
||||
document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL');
|
||||
document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2});
|
||||
const max = Math.max(...Object.values(data.ratings), 1);
|
||||
const rows = Object.entries(data.ratings).map(([stars, count]) => {
|
||||
const row = document.createElement('div'); row.className = 'rating-row';
|
||||
const label = document.createElement('span'); label.textContent = `${stars} ★`;
|
||||
const bar = document.createElement('div'); bar.className = 'bar';
|
||||
const fill = document.createElement('i'); fill.style.width = `${count / max * 100}%`; bar.append(fill);
|
||||
const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL');
|
||||
row.append(label, bar, value); return row;
|
||||
const startResponse = await fetch('/api/reviews', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({url: document.querySelector('#maps-url').value})
|
||||
});
|
||||
document.querySelector('#ratings').replaceChildren(...rows);
|
||||
statusBox.textContent = ''; resultBox.hidden = false;
|
||||
} catch (error) { statusBox.className = 'status error'; statusBox.textContent = error.message; }
|
||||
finally { button.disabled = false; }
|
||||
const startData = await startResponse.json();
|
||||
if (!startResponse.ok) throw new Error(startData.error || 'Nie udało się rozpocząć sprawdzania.');
|
||||
|
||||
while (true) {
|
||||
const response = await fetch(`/api/reviews/${encodeURIComponent(startData.job_id)}`, {cache: 'no-store'});
|
||||
const job = await response.json();
|
||||
if (!response.ok) throw new Error(job.error || 'Nie udało się sprawdzić postępu.');
|
||||
setProgress(job.progress, job.message);
|
||||
if (job.status === 'complete') {
|
||||
showResult(job.result);
|
||||
progressBox.hidden = true;
|
||||
break;
|
||||
}
|
||||
if (job.status === 'error') throw new Error(job.message);
|
||||
await wait(350);
|
||||
}
|
||||
} catch (error) {
|
||||
progressBox.hidden = true;
|
||||
statusBox.className = 'status error';
|
||||
statusBox.textContent = error.message;
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
+11
-1
@@ -10,9 +10,19 @@ input:focus { outline: 3px solid #1a73e822; border-color: #1a73e8; }
|
||||
button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; }
|
||||
button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; }
|
||||
.status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; }
|
||||
.progress-box { margin-top: 22px; }
|
||||
.progress-caption { display: flex; justify-content: space-between; gap: 16px; margin-bottom: 9px; color: #59636e; font-size: .9rem; }
|
||||
.progress-caption strong { color: #1a73e8; font-variant-numeric: tabular-nums; }
|
||||
#progress-bar, .rating-bar { display: block; width: 100%; overflow: hidden; border: 0; border-radius: 99px; background: #edf0f2; appearance: none; }
|
||||
#progress-bar { height: 12px; }
|
||||
#progress-bar::-webkit-progress-bar, .rating-bar::-webkit-progress-bar { background: #edf0f2; border-radius: 99px; }
|
||||
#progress-bar::-webkit-progress-value { background: #1a73e8; border-radius: 99px; transition: width .35s ease; }
|
||||
#progress-bar::-moz-progress-bar { background: #1a73e8; border-radius: 99px; transition: width .35s ease; }
|
||||
.result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; }
|
||||
.summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; }
|
||||
.rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; }
|
||||
.rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
.bar { height: 10px; overflow: hidden; border-radius: 99px; background: #edf0f2; } .bar i { display: block; height: 100%; border-radius: inherit; background: #fbbc04; }
|
||||
.rating-bar { height: 10px; }
|
||||
.rating-bar::-webkit-progress-value { background: #fbbc04; border-radius: 99px; }
|
||||
.rating-bar::-moz-progress-bar { background: #fbbc04; border-radius: 99px; }
|
||||
@media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } }
|
||||
|
||||
@@ -18,7 +18,14 @@
|
||||
<button type="submit">Sprawdź</button>
|
||||
</div>
|
||||
</form>
|
||||
<p id="status" class="status" role="status" aria-live="polite"></p>
|
||||
<div id="progress-box" class="progress-box" role="status" aria-live="polite" hidden>
|
||||
<div class="progress-caption">
|
||||
<span id="progress-label">Rozpoczynam sprawdzanie</span>
|
||||
<strong id="progress-value">0%</strong>
|
||||
</div>
|
||||
<progress id="progress-bar" max="100" value="0">0%</progress>
|
||||
</div>
|
||||
<p id="status" class="status" role="alert" aria-live="assertive"></p>
|
||||
<section id="result" class="result" hidden>
|
||||
<div class="summary">
|
||||
<div><span id="total">0</span><small>opinii łącznie</small></div>
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import unittest
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
from app import app
|
||||
from scraper import parse_distribution_labels, validate_maps_url
|
||||
@@ -49,6 +51,27 @@ class ApiTests(unittest.TestCase):
|
||||
response = app.test_client().post("/api/reviews", data=b"x" * 5000, content_type="application/json")
|
||||
self.assertEqual(response.status_code, 413)
|
||||
|
||||
def test_reports_job_progress_and_result(self):
|
||||
expected = {"total": 3, "average": 4.0, "ratings": {"5": 2, "4": 0, "3": 0, "2": 0, "1": 1}}
|
||||
|
||||
def fake_scrape(_url, progress):
|
||||
progress(50, "Otwarto Chromium")
|
||||
progress(99, "Pojawiły się liczniki")
|
||||
return expected
|
||||
|
||||
client = app.test_client()
|
||||
with patch("app.get_rating_distribution", side_effect=fake_scrape):
|
||||
start = client.post("/api/reviews", json={"url": "https://maps.app.goo.gl/USRCto35jhoVtmj4A"})
|
||||
self.assertEqual(start.status_code, 202)
|
||||
job_id = start.get_json()["job_id"]
|
||||
for _ in range(100):
|
||||
job = client.get(f"/api/reviews/{job_id}").get_json()
|
||||
if job["status"] == "complete":
|
||||
break
|
||||
time.sleep(0.005)
|
||||
self.assertEqual(job["progress"], 100)
|
||||
self.assertEqual(job["result"], expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user