web app
This commit is contained in:
@@ -174,3 +174,7 @@ cython_debug/
|
||||
# PyPI configuration file
|
||||
.pypirc
|
||||
|
||||
.venv/
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
*.py[cod]
|
||||
|
||||
@@ -1,2 +1,36 @@
|
||||
# opinie
|
||||
# Licznik opinii Google Maps
|
||||
|
||||
Lekka strona pokazująca dokładne liczniki opinii 1–5. Odczytuje tekstowe wartości dostępności histogramu Google — nie szacuje ich z długości pasków, nie pobiera treści opinii i niczego nie zapisuje.
|
||||
|
||||
## Lokalnie
|
||||
|
||||
```bash
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
playwright install chromium
|
||||
python app.py
|
||||
```
|
||||
|
||||
Otwórz `http://127.0.0.1:8080`.
|
||||
|
||||
## Raspberry Pi OS (bez Dockera)
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install chromium python3-venv
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
CHROMIUM_PATH=/usr/bin/chromium gunicorn --workers 1 --threads 2 --bind 127.0.0.1:8080 app:app
|
||||
```
|
||||
|
||||
Na świat wystaw aplikację przez nginx/Caddy z HTTPS. Zostaw jeden worker: aplikacja celowo dopuszcza tylko jedno sprawdzenie naraz, by chronić RAM i CPU RPi.
|
||||
|
||||
Google może czasem pokazać niezalogowanej przeglądarce „ograniczony widok” i ukryć wszystkie opinie. Wtedy aplikacja zwraca błąd — nigdy nie zgaduje wartości z długości pasków i nie wymaga przechowywania konta Google.
|
||||
|
||||
## Testy
|
||||
|
||||
```bash
|
||||
python -m unittest discover -s tests -v
|
||||
```
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from threading import Lock
|
||||
|
||||
from flask import Flask, jsonify, render_template, request
|
||||
from scraper import ScrapeError, get_rating_distribution, validate_maps_url
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config["MAX_CONTENT_LENGTH"] = 4096
|
||||
_scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi
|
||||
_rate_lock = Lock()
|
||||
_requests = defaultdict(deque)
|
||||
|
||||
|
||||
def _rate_limited(ip, limit=6, window=60):
|
||||
now = time.monotonic()
|
||||
with _rate_lock:
|
||||
attempts = _requests[ip]
|
||||
while attempts and attempts[0] < now - window:
|
||||
attempts.popleft()
|
||||
if len(attempts) >= limit:
|
||||
return True
|
||||
attempts.append(now)
|
||||
return False
|
||||
|
||||
|
||||
@app.after_request
|
||||
def headers(response):
|
||||
response.headers["Content-Security-Policy"] = (
|
||||
"default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; "
|
||||
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
||||
)
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["Referrer-Policy"] = "no-referrer"
|
||||
response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
|
||||
return response
|
||||
|
||||
|
||||
@app.errorhandler(413)
|
||||
def too_large(_error):
|
||||
return jsonify(error="Żądanie jest zbyt duże."), 413
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def index():
|
||||
return render_template("index.html")
|
||||
|
||||
|
||||
@app.post("/api/reviews")
|
||||
def reviews():
|
||||
if _rate_limited(request.remote_addr or "unknown"):
|
||||
return jsonify(error="Za dużo prób. Poczekaj minutę."), 429
|
||||
data = request.get_json(silent=True)
|
||||
if not isinstance(data, dict) or not isinstance(data.get("url"), str):
|
||||
return jsonify(error="Podaj link do miejsca w Google Maps."), 400
|
||||
try:
|
||||
url = validate_maps_url(data["url"])
|
||||
except ValueError as exc:
|
||||
return jsonify(error=str(exc)), 400
|
||||
if not _scrape_lock.acquire(blocking=False):
|
||||
return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429
|
||||
try:
|
||||
return jsonify(get_rating_distribution(url))
|
||||
except ScrapeError as exc:
|
||||
return jsonify(error=str(exc)), 422
|
||||
except Exception:
|
||||
app.logger.exception("Unexpected scraper failure")
|
||||
return jsonify(error="Google Maps nie odpowiedziało poprawnie. Spróbuj później."), 502
|
||||
finally:
|
||||
_scrape_lock.release()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False)
|
||||
@@ -0,0 +1,3 @@
|
||||
Flask==3.1.2
|
||||
playwright==1.55.0
|
||||
gunicorn==23.0.0
|
||||
+3
-581
@@ -1,585 +1,7 @@
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
"""Zgodny wstecznie punkt startowy: python reviews.py."""
|
||||
|
||||
import pandas as pd
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
|
||||
# ============================================================
|
||||
# USTAWIENIA
|
||||
# ============================================================
|
||||
|
||||
URL = "https://maps.app.goo.gl/USRCto35jhoVtmj4A"
|
||||
|
||||
HEADLESS = False
|
||||
|
||||
# Ile kolejnych scrolli bez nowych opinii oznacza koniec
|
||||
MAX_EMPTY_SCROLLS = 8
|
||||
|
||||
# Czas między scrollami
|
||||
SCROLL_DELAY = 1.2
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PARSOWANIE DATY GOOGLE
|
||||
# ============================================================
|
||||
|
||||
def parse_google_date(text: str):
|
||||
if not text:
|
||||
return None
|
||||
|
||||
text = text.strip().lower()
|
||||
now = datetime.now()
|
||||
|
||||
if text == "dzisiaj":
|
||||
return now.date()
|
||||
|
||||
if text == "wczoraj":
|
||||
return (now - timedelta(days=1)).date()
|
||||
|
||||
patterns = [
|
||||
(r"(\d+)\s+dni?\s+temu", lambda n: now - timedelta(days=n)),
|
||||
(r"(\d+)\s+dzień\s+temu", lambda n: now - timedelta(days=n)),
|
||||
(r"(\d+)\s+tygodni?\s+temu", lambda n: now - timedelta(weeks=n)),
|
||||
(
|
||||
r"(\d+)\s+miesi(?:ąc|ące|ęcy)\s+temu",
|
||||
lambda n: now - relativedelta(months=n)
|
||||
),
|
||||
(
|
||||
r"(\d+)\s+lat(?:a)?\s+temu",
|
||||
lambda n: now - relativedelta(years=n)
|
||||
),
|
||||
]
|
||||
|
||||
for pattern, func in patterns:
|
||||
match = re.search(pattern, text)
|
||||
|
||||
if match:
|
||||
return func(int(match.group(1))).date()
|
||||
|
||||
if "tydzień temu" in text:
|
||||
return (now - timedelta(weeks=1)).date()
|
||||
|
||||
if "miesiąc temu" in text:
|
||||
return (now - relativedelta(months=1)).date()
|
||||
|
||||
if "rok temu" in text:
|
||||
return (now - relativedelta(years=1)).date()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# COOKIES
|
||||
# ============================================================
|
||||
|
||||
def accept_cookies(page):
|
||||
labels = [
|
||||
"Zaakceptuj wszystko",
|
||||
"Accept all",
|
||||
"Odrzuć wszystko",
|
||||
"Reject all",
|
||||
]
|
||||
|
||||
for label in labels:
|
||||
try:
|
||||
button = page.get_by_role(
|
||||
"button",
|
||||
name=re.compile(label, re.I)
|
||||
)
|
||||
|
||||
if button.count() and button.first.is_visible():
|
||||
button.first.click(timeout=3000)
|
||||
time.sleep(2)
|
||||
return
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
# ============================================================
|
||||
# OTWIERANIE PANELU OPINII
|
||||
# ============================================================
|
||||
|
||||
def open_reviews_panel(page):
|
||||
print("Szukam właściwego przycisku z opiniami...")
|
||||
|
||||
buttons = page.locator("button")
|
||||
|
||||
for i in range(buttons.count()):
|
||||
try:
|
||||
button = buttons.nth(i)
|
||||
|
||||
if not button.is_visible():
|
||||
continue
|
||||
|
||||
aria = button.get_attribute("aria-label") or ""
|
||||
|
||||
try:
|
||||
text = button.inner_text(timeout=500)
|
||||
except:
|
||||
text = ""
|
||||
|
||||
label = f"{aria} {text}".strip()
|
||||
|
||||
# Musi zawierać liczbę oraz opini/recenz/review
|
||||
if (
|
||||
re.search(r"\d+", label)
|
||||
and re.search(r"(opini|recenz|reviews?)", label, re.I)
|
||||
):
|
||||
print("Kandydat:", repr(label[:200]))
|
||||
|
||||
try:
|
||||
button.scroll_into_view_if_needed()
|
||||
time.sleep(0.5)
|
||||
button.click(timeout=3000)
|
||||
|
||||
print("Kliknięto:", repr(label[:200]))
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
cards = page.locator('div[data-review-id]')
|
||||
|
||||
if cards.count() > 5:
|
||||
print("Panel opinii otwarty.")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print("Nie udało się kliknąć:", e)
|
||||
|
||||
except:
|
||||
continue
|
||||
|
||||
# Fallback dla role=button
|
||||
print("Próbuję alternatywnego selektora...")
|
||||
|
||||
candidates = page.locator(
|
||||
'[role="button"][aria-label*="opini"], '
|
||||
'[role="button"][aria-label*="recenz"], '
|
||||
'[role="button"][aria-label*="review"]'
|
||||
)
|
||||
|
||||
for i in range(candidates.count()):
|
||||
try:
|
||||
el = candidates.nth(i)
|
||||
|
||||
if not el.is_visible():
|
||||
continue
|
||||
|
||||
aria = el.get_attribute("aria-label") or ""
|
||||
|
||||
if not re.search(r"\d+", aria):
|
||||
continue
|
||||
|
||||
print("Klikam:", repr(aria))
|
||||
|
||||
el.scroll_into_view_if_needed()
|
||||
el.click(timeout=3000)
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
cards = page.locator('div[data-review-id]')
|
||||
|
||||
if cards.count() > 5:
|
||||
print("Panel opinii otwarty.")
|
||||
return True
|
||||
|
||||
except:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ZNAJDOWANIE SCROLLOWANEGO PANELU
|
||||
# ============================================================
|
||||
|
||||
def get_scrollable_parent(card):
|
||||
try:
|
||||
return card.evaluate_handle("""
|
||||
el => {
|
||||
let p = el.parentElement;
|
||||
|
||||
while (p) {
|
||||
const style = getComputedStyle(p);
|
||||
|
||||
if (
|
||||
(style.overflowY === 'auto' ||
|
||||
style.overflowY === 'scroll') &&
|
||||
p.scrollHeight > p.clientHeight
|
||||
) {
|
||||
return p;
|
||||
}
|
||||
|
||||
p = p.parentElement;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
""")
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# OCENA GWIAZDKOWA
|
||||
# ============================================================
|
||||
|
||||
def extract_rating(card):
|
||||
selectors = [
|
||||
'[role="img"][aria-label*="gwiazd"]',
|
||||
'[aria-label*="gwiazd"]',
|
||||
'[role="img"][aria-label*="star"]',
|
||||
'[aria-label*="star"]',
|
||||
'span.kvMYJc[aria-label]',
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
try:
|
||||
el = card.locator(selector).first
|
||||
|
||||
if not el.count():
|
||||
continue
|
||||
|
||||
label = el.get_attribute("aria-label") or ""
|
||||
|
||||
match = re.search(r"([1-5](?:[.,]\d+)?)", label)
|
||||
|
||||
if match:
|
||||
value = match.group(1).replace(",", ".")
|
||||
return int(float(value))
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# DATA OPINII
|
||||
# ============================================================
|
||||
|
||||
def extract_date_text(card):
|
||||
selectors = [
|
||||
".rsqaWe",
|
||||
"span.rsqaWe",
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
try:
|
||||
el = card.locator(selector).first
|
||||
|
||||
if el.count():
|
||||
text = el.inner_text().strip()
|
||||
|
||||
if text:
|
||||
return text
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ŁADOWANIE WSZYSTKICH OPINII
|
||||
# ============================================================
|
||||
|
||||
def count_unique_reviews(cards):
|
||||
"""Policz opinie, a nie zduplikowane elementy DOM Google Maps."""
|
||||
try:
|
||||
return cards.evaluate_all("""
|
||||
elements => new Set(
|
||||
elements
|
||||
.map(el => el.getAttribute('data-review-id'))
|
||||
.filter(Boolean)
|
||||
).size
|
||||
""")
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def load_all_reviews(page):
|
||||
previous_count = 0
|
||||
empty_scrolls = 0
|
||||
|
||||
print("\nŁadowanie opinii...")
|
||||
|
||||
while empty_scrolls < MAX_EMPTY_SCROLLS:
|
||||
cards = page.locator('div[data-review-id]')
|
||||
count = count_unique_reviews(cards)
|
||||
|
||||
print(
|
||||
f"\rZaładowane opinie: {count}",
|
||||
end="",
|
||||
flush=True
|
||||
)
|
||||
|
||||
if count > previous_count:
|
||||
previous_count = count
|
||||
empty_scrolls = 0
|
||||
else:
|
||||
empty_scrolls += 1
|
||||
|
||||
if count == 0:
|
||||
page.mouse.wheel(0, 5000)
|
||||
time.sleep(SCROLL_DELAY)
|
||||
continue
|
||||
|
||||
last_card = cards.last
|
||||
|
||||
scrollable = get_scrollable_parent(last_card)
|
||||
|
||||
if scrollable:
|
||||
try:
|
||||
scrollable.evaluate("""
|
||||
el => {
|
||||
el.scrollTop = el.scrollHeight;
|
||||
}
|
||||
""")
|
||||
except:
|
||||
page.mouse.wheel(0, 5000)
|
||||
else:
|
||||
try:
|
||||
last_card.scroll_into_view_if_needed()
|
||||
except:
|
||||
page.mouse.wheel(0, 5000)
|
||||
|
||||
time.sleep(SCROLL_DELAY)
|
||||
|
||||
print()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ZBIERANIE DANYCH
|
||||
# ============================================================
|
||||
|
||||
def extract_reviews(page):
|
||||
results = {}
|
||||
|
||||
cards = page.locator('div[data-review-id]')
|
||||
|
||||
unique_count = count_unique_reviews(cards)
|
||||
print(f"\nZbieram dane z {unique_count} unikalnych opinii...")
|
||||
|
||||
for i in range(cards.count()):
|
||||
card = cards.nth(i)
|
||||
|
||||
try:
|
||||
review_id = card.get_attribute("data-review-id")
|
||||
|
||||
if not review_id:
|
||||
continue
|
||||
|
||||
# Deduplikacja
|
||||
if review_id in results:
|
||||
continue
|
||||
|
||||
rating = extract_rating(card)
|
||||
|
||||
if rating is None:
|
||||
continue
|
||||
|
||||
date_raw = extract_date_text(card)
|
||||
date_approx = parse_google_date(date_raw)
|
||||
|
||||
results[review_id] = {
|
||||
"review_id": review_id,
|
||||
"rating": rating,
|
||||
"date_raw": date_raw,
|
||||
"date_approx": date_approx,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"Błąd przy opinii #{i}: {e}")
|
||||
|
||||
return pd.DataFrame(results.values())
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PODSUMOWANIE
|
||||
# ============================================================
|
||||
|
||||
def print_summary(df):
|
||||
print("\n==============================")
|
||||
print("ŁĄCZNIE")
|
||||
print("==============================")
|
||||
print(len(df))
|
||||
|
||||
print("\n==============================")
|
||||
print("ROZKŁAD GWIAZDEK")
|
||||
print("==============================")
|
||||
|
||||
distribution = (
|
||||
df["rating"]
|
||||
.value_counts()
|
||||
.reindex([5, 4, 3, 2, 1], fill_value=0)
|
||||
)
|
||||
|
||||
total = len(df)
|
||||
|
||||
for stars, count in distribution.items():
|
||||
percent = (count / total * 100) if total else 0
|
||||
|
||||
print(
|
||||
f"{stars}★: "
|
||||
f"{count:>4} "
|
||||
f"({percent:>5.1f}%)"
|
||||
)
|
||||
|
||||
print("\n==============================")
|
||||
print("ŚREDNIA")
|
||||
print("==============================")
|
||||
|
||||
if len(df):
|
||||
print(round(df["rating"].mean(), 3))
|
||||
else:
|
||||
print("Brak danych")
|
||||
|
||||
|
||||
# ============================================================
|
||||
# FILTROWANIE PO OKRESIE
|
||||
# ============================================================
|
||||
|
||||
def print_period_summary(df, date_from=None, date_to=None):
|
||||
temp = df.copy()
|
||||
|
||||
temp["date_approx"] = pd.to_datetime(
|
||||
temp["date_approx"],
|
||||
errors="coerce"
|
||||
)
|
||||
|
||||
if date_from:
|
||||
temp = temp[
|
||||
temp["date_approx"] >= pd.to_datetime(date_from)
|
||||
]
|
||||
|
||||
if date_to:
|
||||
temp = temp[
|
||||
temp["date_approx"] <= pd.to_datetime(date_to)
|
||||
]
|
||||
|
||||
print("\n==============================")
|
||||
print(f"OKRES: {date_from} -> {date_to}")
|
||||
print("==============================")
|
||||
|
||||
if temp.empty:
|
||||
print("Brak opinii w tym okresie.")
|
||||
return
|
||||
|
||||
distribution = (
|
||||
temp["rating"]
|
||||
.value_counts()
|
||||
.reindex([5, 4, 3, 2, 1], fill_value=0)
|
||||
)
|
||||
|
||||
print(f"Liczba opinii: {len(temp)}")
|
||||
|
||||
for stars, count in distribution.items():
|
||||
print(f"{stars}★: {count}")
|
||||
|
||||
print("Średnia:", round(temp["rating"].mean(), 3))
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MAIN
|
||||
# ============================================================
|
||||
|
||||
def main():
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=HEADLESS
|
||||
)
|
||||
|
||||
context = browser.new_context(
|
||||
locale="pl-PL",
|
||||
viewport={
|
||||
"width": 1400,
|
||||
"height": 900,
|
||||
},
|
||||
)
|
||||
|
||||
page = context.new_page()
|
||||
|
||||
print("Otwieram Google Maps...")
|
||||
|
||||
page.goto(
|
||||
URL,
|
||||
wait_until="domcontentloaded",
|
||||
timeout=60_000
|
||||
)
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
accept_cookies(page)
|
||||
|
||||
opened = open_reviews_panel(page)
|
||||
|
||||
if not opened:
|
||||
print(
|
||||
"\nNie udało się automatycznie otworzyć panelu opinii."
|
||||
)
|
||||
print(
|
||||
"Kliknij ręcznie liczbę opinii / „Więcej opinii”."
|
||||
)
|
||||
|
||||
input(
|
||||
"Gdy panel będzie otwarty, naciśnij ENTER..."
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
cards = page.locator('div[data-review-id]')
|
||||
|
||||
print(
|
||||
"Opinie widoczne po otwarciu panelu:",
|
||||
count_unique_reviews(cards)
|
||||
)
|
||||
|
||||
if count_unique_reviews(cards) < 5:
|
||||
print(
|
||||
"\nUWAGA: panel opinii prawdopodobnie "
|
||||
"nie został poprawnie otwarty."
|
||||
)
|
||||
|
||||
input(
|
||||
"Otwórz ręcznie wszystkie opinie "
|
||||
"i naciśnij ENTER..."
|
||||
)
|
||||
|
||||
load_all_reviews(page)
|
||||
|
||||
df = extract_reviews(page)
|
||||
|
||||
browser.close()
|
||||
|
||||
if df.empty:
|
||||
print("\nNie znaleziono żadnych opinii.")
|
||||
return
|
||||
|
||||
df.to_csv(
|
||||
"google_reviews_ratings.csv",
|
||||
index=False,
|
||||
encoding="utf-8-sig"
|
||||
)
|
||||
|
||||
print_summary(df)
|
||||
|
||||
print("\nZapisano:")
|
||||
print("google_reviews_ratings.csv")
|
||||
|
||||
# ========================================================
|
||||
# PRZYKŁADOWE FILTROWANIE
|
||||
# Odkomentuj jeśli chcesz
|
||||
# ========================================================
|
||||
|
||||
# print_period_summary(
|
||||
# df,
|
||||
# "2025-01-01",
|
||||
# "2025-12-31"
|
||||
# )
|
||||
from app import app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
app.run(host="127.0.0.1", port=8080, debug=False)
|
||||
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
|
||||
ALLOWED_HOSTS = {
|
||||
"maps.app.goo.gl", "goo.gl", "google.com", "www.google.com",
|
||||
"maps.google.com", "google.pl", "www.google.pl", "maps.google.pl",
|
||||
}
|
||||
MAX_URL_LENGTH = 1200
|
||||
|
||||
|
||||
class ScrapeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def validate_maps_url(raw_url: str) -> str:
|
||||
"""Przepuszcza wyłącznie zwykłe linki HTTPS do Google Maps."""
|
||||
url = raw_url.strip()
|
||||
if not url:
|
||||
raise ValueError("Wklej link do miejsca w Google Maps.")
|
||||
if len(url) > MAX_URL_LENGTH or any(ord(char) < 32 for char in url):
|
||||
raise ValueError("Link jest nieprawidłowy albo zbyt długi.")
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
host = (parsed.hostname or "").lower().rstrip(".")
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
raise ValueError("Link ma nieprawidłowy format.") from None
|
||||
if parsed.scheme.lower() != "https" or not host or parsed.username or parsed.password:
|
||||
raise ValueError("Dozwolony jest wyłącznie bezpieczny link HTTPS do Google Maps.")
|
||||
if port not in (None, 443) or host not in ALLOWED_HOSTS:
|
||||
raise ValueError("To nie jest obsługiwany link Google Maps.")
|
||||
|
||||
path = parsed.path or "/"
|
||||
if host == "maps.app.goo.gl":
|
||||
valid_path = re.fullmatch(r"/[A-Za-z0-9_-]{5,100}/?", path)
|
||||
elif host == "goo.gl":
|
||||
valid_path = re.fullmatch(r"/maps/[A-Za-z0-9_-]{5,100}/?", path)
|
||||
else:
|
||||
valid_path = path == "/maps" or path.startswith("/maps/")
|
||||
if not valid_path:
|
||||
raise ValueError("Link musi prowadzić do miejsca w Google Maps.")
|
||||
return urlunsplit(("https", host, path, parsed.query, ""))
|
||||
|
||||
|
||||
def _number(text: str) -> int | None:
|
||||
compact = re.sub(r"[\s\u00a0\u202f.,]", "", text)
|
||||
return int(compact) if compact.isdigit() else None
|
||||
|
||||
|
||||
def parse_distribution_labels(labels: list[str]) -> dict[int, int]:
|
||||
"""Czyta np. '5 gwiazdek, 123 opinie' — nie mierzy szerokości paska."""
|
||||
result = {}
|
||||
for label in labels:
|
||||
clean = " ".join(label.lower().split())
|
||||
if not re.search(r"gwiazd|star", clean) or not re.search(r"opini|recenz|review", clean):
|
||||
continue
|
||||
rating_match = re.search(r"(?:^|\D)([1-5])(?:\D|$)", clean)
|
||||
if not rating_match:
|
||||
continue
|
||||
candidates = re.findall(r"\d[\d\s\u00a0\u202f.,]*", clean[rating_match.end():])
|
||||
values = [_number(item) for item in candidates]
|
||||
values = [item for item in values if item is not None]
|
||||
if values:
|
||||
result[int(rating_match.group(1))] = values[-1]
|
||||
return result
|
||||
|
||||
|
||||
def _accept_cookies(page):
|
||||
for text in ("Zaakceptuj wszystko", "Odrzuć wszystko", "Accept all", "Reject all"):
|
||||
try:
|
||||
button = page.get_by_role("button", name=re.compile(f"^{re.escape(text)}$", re.I)).first
|
||||
if button.is_visible(timeout=700):
|
||||
button.click(timeout=2_000)
|
||||
page.wait_for_timeout(500)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _open_reviews(page):
|
||||
candidates = page.locator("button, [role=button]")
|
||||
for index in range(min(candidates.count(), 250)):
|
||||
element = candidates.nth(index)
|
||||
try:
|
||||
label = " ".join(filter(None, [
|
||||
element.get_attribute("aria-label"), element.inner_text(timeout=100)
|
||||
]))
|
||||
if re.search(r"\d", label) and re.search(r"opini|recenz|review", label, re.I):
|
||||
element.click(timeout=2_000)
|
||||
page.wait_for_timeout(800)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_rating_distribution(url: str) -> dict:
|
||||
"""Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane."""
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
launch = {"headless": True, "args": ["--disable-dev-shm-usage", "--no-first-run", "--disable-gpu"]}
|
||||
if os.environ.get("CHROMIUM_PATH"):
|
||||
launch["executable_path"] = os.environ["CHROMIUM_PATH"]
|
||||
browser = playwright.chromium.launch(**launch)
|
||||
try:
|
||||
page = browser.new_page(locale="pl-PL", viewport={"width": 1100, "height": 800})
|
||||
page.goto(url, wait_until="domcontentloaded", timeout=30_000)
|
||||
_accept_cookies(page)
|
||||
_open_reviews(page)
|
||||
page.wait_for_timeout(700)
|
||||
labels = page.locator("[aria-label]").evaluate_all(
|
||||
"els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)"
|
||||
)
|
||||
counts = parse_distribution_labels(labels)
|
||||
if len(counts) != 5:
|
||||
body_text = page.locator("body").inner_text().lower()
|
||||
if "ograniczonego widoku" in body_text or "limited view" in body_text:
|
||||
raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.")
|
||||
raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.")
|
||||
ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)}
|
||||
total = sum(ratings.values())
|
||||
average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0
|
||||
return {"total": total, "average": average, "ratings": ratings}
|
||||
finally:
|
||||
browser.close()
|
||||
except ScrapeError:
|
||||
raise
|
||||
except PlaywrightTimeoutError:
|
||||
raise ScrapeError("Google Maps nie odpowiedziało w wyznaczonym czasie.") from None
|
||||
except Exception as exc:
|
||||
if "Executable doesn't exist" in str(exc):
|
||||
raise ScrapeError("Chromium nie jest zainstalowane na serwerze.") from None
|
||||
raise
|
||||
@@ -0,0 +1,28 @@
|
||||
const form = document.querySelector('#review-form');
|
||||
const statusBox = document.querySelector('#status');
|
||||
const resultBox = document.querySelector('#result');
|
||||
const button = form.querySelector('button');
|
||||
|
||||
form.addEventListener('submit', async (event) => {
|
||||
event.preventDefault(); button.disabled = true; resultBox.hidden = true;
|
||||
statusBox.className = 'status'; statusBox.textContent = 'Odczytuję dokładne liczniki…';
|
||||
try {
|
||||
const response = await fetch('/api/reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({url: document.querySelector('#maps-url').value})});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Nie udało się pobrać danych.');
|
||||
document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL');
|
||||
document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2});
|
||||
const max = Math.max(...Object.values(data.ratings), 1);
|
||||
const rows = Object.entries(data.ratings).map(([stars, count]) => {
|
||||
const row = document.createElement('div'); row.className = 'rating-row';
|
||||
const label = document.createElement('span'); label.textContent = `${stars} ★`;
|
||||
const bar = document.createElement('div'); bar.className = 'bar';
|
||||
const fill = document.createElement('i'); fill.style.width = `${count / max * 100}%`; bar.append(fill);
|
||||
const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL');
|
||||
row.append(label, bar, value); return row;
|
||||
});
|
||||
document.querySelector('#ratings').replaceChildren(...rows);
|
||||
statusBox.textContent = ''; resultBox.hidden = false;
|
||||
} catch (error) { statusBox.className = 'status error'; statusBox.textContent = error.message; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
:root { font-family: Inter, ui-sans-serif, system-ui, sans-serif; color: #202124; background: #f3f5f7; }
|
||||
* { box-sizing: border-box; } body { margin: 0; min-height: 100vh; }
|
||||
main { min-height: 100vh; display: grid; place-items: center; padding: 24px; }
|
||||
.card { width: min(680px, 100%); padding: clamp(24px, 6vw, 52px); background: #fff; border: 1px solid #e2e6ea; border-radius: 24px; box-shadow: 0 18px 55px #1f293714; }
|
||||
.eyebrow { margin: 0 0 8px; color: #1967d2; font-size: .78rem; font-weight: 800; letter-spacing: .12em; text-transform: uppercase; }
|
||||
h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.65rem); line-height: 1.08; } .intro { margin: 14px 0 30px; color: #697079; }
|
||||
label { display: block; margin-bottom: 8px; font-size: .9rem; font-weight: 700; } .input-row { display: flex; gap: 10px; }
|
||||
input { min-width: 0; flex: 1; padding: 14px 16px; border: 1px solid #c8cdd2; border-radius: 12px; font: inherit; }
|
||||
input:focus { outline: 3px solid #1a73e822; border-color: #1a73e8; }
|
||||
button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; }
|
||||
button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; }
|
||||
.status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; }
|
||||
.result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; }
|
||||
.summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; }
|
||||
.rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; }
|
||||
.rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
.bar { height: 10px; overflow: hidden; border-radius: 99px; background: #edf0f2; } .bar i { display: block; height: 100%; border-radius: inherit; background: #fbbc04; }
|
||||
@media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } }
|
||||
@@ -0,0 +1,32 @@
|
||||
<!doctype html>
|
||||
<html lang="pl">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Rozkład opinii Google</title>
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body>
|
||||
<main><section class="card">
|
||||
<p class="eyebrow">Google Maps</p>
|
||||
<h1>Policz opinie według oceny</h1>
|
||||
<p class="intro">Wklej link do firmy lub miejsca. Aplikacja niczego nie zapisuje.</p>
|
||||
<form id="review-form">
|
||||
<label for="maps-url">Link Google Maps</label>
|
||||
<div class="input-row">
|
||||
<input id="maps-url" type="url" maxlength="1200" placeholder="https://maps.app.goo.gl/…" required autocomplete="off">
|
||||
<button type="submit">Sprawdź</button>
|
||||
</div>
|
||||
</form>
|
||||
<p id="status" class="status" role="status" aria-live="polite"></p>
|
||||
<section id="result" class="result" hidden>
|
||||
<div class="summary">
|
||||
<div><span id="total">0</span><small>opinii łącznie</small></div>
|
||||
<div><span id="average">0</span><small>średnia</small></div>
|
||||
</div>
|
||||
<div id="ratings" class="ratings"></div>
|
||||
</section>
|
||||
</section></main>
|
||||
<script src="/static/app.js" defer></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,54 @@
|
||||
import unittest
|
||||
|
||||
from app import app
|
||||
from scraper import parse_distribution_labels, validate_maps_url
|
||||
|
||||
|
||||
class ValidationTests(unittest.TestCase):
|
||||
def test_accepts_google_maps_links(self):
|
||||
urls = [
|
||||
"https://maps.app.goo.gl/USRCto35jhoVtmj4A",
|
||||
"https://www.google.com/maps/place/Test/@50,20,15z",
|
||||
"https://maps.google.pl/maps?q=test",
|
||||
]
|
||||
for url in urls:
|
||||
with self.subTest(url=url):
|
||||
self.assertTrue(validate_maps_url(url).startswith("https://"))
|
||||
|
||||
def test_rejects_unsafe_links(self):
|
||||
urls = [
|
||||
"http://maps.app.goo.gl/USRCto35jhoVtmj4A",
|
||||
"https://evil.example/maps/x",
|
||||
"https://google.com.evil.example/maps/x",
|
||||
"https://user:pass@www.google.com/maps/x",
|
||||
"https://www.google.com:444/maps/x",
|
||||
"https://www.google.com/search?q=x",
|
||||
"file:///etc/passwd",
|
||||
]
|
||||
for url in urls:
|
||||
with self.subTest(url=url), self.assertRaises(ValueError):
|
||||
validate_maps_url(url)
|
||||
|
||||
def test_parses_exact_accessibility_labels(self):
|
||||
labels = [
|
||||
"5 gwiazdek, 1 234 opinie", "4 gwiazdki, 56 opinii",
|
||||
"3 stars, 7 reviews", "2 stars, 1 review", "1 gwiazdka, 0 opinii",
|
||||
]
|
||||
self.assertEqual(parse_distribution_labels(labels), {5: 1234, 4: 56, 3: 7, 2: 1, 1: 0})
|
||||
|
||||
|
||||
class ApiTests(unittest.TestCase):
|
||||
def test_home_page(self):
|
||||
self.assertEqual(app.test_client().get("/").status_code, 200)
|
||||
|
||||
def test_rejects_non_google_url(self):
|
||||
response = app.test_client().post("/api/reviews", json={"url": "https://example.com/"})
|
||||
self.assertEqual(response.status_code, 400)
|
||||
|
||||
def test_rejects_large_request(self):
|
||||
response = app.test_client().post("/api/reviews", data=b"x" * 5000, content_type="application/json")
|
||||
self.assertEqual(response.status_code, 413)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user