web app
This commit is contained in:
@@ -174,3 +174,7 @@ cython_debug/
|
|||||||
# PyPI configuration file
|
# PyPI configuration file
|
||||||
.pypirc
|
.pypirc
|
||||||
|
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
.pytest_cache/
|
||||||
|
*.py[cod]
|
||||||
|
|||||||
@@ -1,2 +1,36 @@
|
|||||||
# opinie
|
# Licznik opinii Google Maps
|
||||||
|
|
||||||
|
Lekka strona pokazująca dokładne liczniki opinii 1–5. Odczytuje tekstowe wartości dostępności histogramu Google — nie szacuje ich z długości pasków, nie pobiera treści opinii i niczego nie zapisuje.
|
||||||
|
|
||||||
|
## Lokalnie
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
playwright install chromium
|
||||||
|
python app.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Otwórz `http://127.0.0.1:8080`.
|
||||||
|
|
||||||
|
## Raspberry Pi OS (bez Dockera)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo apt update
|
||||||
|
sudo apt install chromium python3-venv
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
CHROMIUM_PATH=/usr/bin/chromium gunicorn --workers 1 --threads 2 --bind 127.0.0.1:8080 app:app
|
||||||
|
```
|
||||||
|
|
||||||
|
Na świat wystaw aplikację przez nginx/Caddy z HTTPS. Zostaw jeden worker: aplikacja celowo dopuszcza tylko jedno sprawdzenie naraz, by chronić RAM i CPU RPi.
|
||||||
|
|
||||||
|
Google może czasem pokazać niezalogowanej przeglądarce „ograniczony widok” i ukryć wszystkie opinie. Wtedy aplikacja zwraca błąd — nigdy nie zgaduje wartości z długości pasków i nie wymaga przechowywania konta Google.
|
||||||
|
|
||||||
|
## Testy
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m unittest discover -s tests -v
|
||||||
|
```
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
from collections import defaultdict, deque
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
|
from flask import Flask, jsonify, render_template, request
|
||||||
|
from scraper import ScrapeError, get_rating_distribution, validate_maps_url
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.config["MAX_CONTENT_LENGTH"] = 4096
|
||||||
|
_scrape_lock = Lock() # jeden Chromium naraz, żeby nie zamęczyć RPi
|
||||||
|
_rate_lock = Lock()
|
||||||
|
_requests = defaultdict(deque)
|
||||||
|
|
||||||
|
|
||||||
|
def _rate_limited(ip, limit=6, window=60):
|
||||||
|
now = time.monotonic()
|
||||||
|
with _rate_lock:
|
||||||
|
attempts = _requests[ip]
|
||||||
|
while attempts and attempts[0] < now - window:
|
||||||
|
attempts.popleft()
|
||||||
|
if len(attempts) >= limit:
|
||||||
|
return True
|
||||||
|
attempts.append(now)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
@app.after_request
|
||||||
|
def headers(response):
|
||||||
|
response.headers["Content-Security-Policy"] = (
|
||||||
|
"default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; "
|
||||||
|
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'"
|
||||||
|
)
|
||||||
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||||
|
response.headers["Referrer-Policy"] = "no-referrer"
|
||||||
|
response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()"
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@app.errorhandler(413)
|
||||||
|
def too_large(_error):
|
||||||
|
return jsonify(error="Żądanie jest zbyt duże."), 413
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/")
|
||||||
|
def index():
|
||||||
|
return render_template("index.html")
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/reviews")
|
||||||
|
def reviews():
|
||||||
|
if _rate_limited(request.remote_addr or "unknown"):
|
||||||
|
return jsonify(error="Za dużo prób. Poczekaj minutę."), 429
|
||||||
|
data = request.get_json(silent=True)
|
||||||
|
if not isinstance(data, dict) or not isinstance(data.get("url"), str):
|
||||||
|
return jsonify(error="Podaj link do miejsca w Google Maps."), 400
|
||||||
|
try:
|
||||||
|
url = validate_maps_url(data["url"])
|
||||||
|
except ValueError as exc:
|
||||||
|
return jsonify(error=str(exc)), 400
|
||||||
|
if not _scrape_lock.acquire(blocking=False):
|
||||||
|
return jsonify(error="Trwa już inne sprawdzanie. Spróbuj za chwilę."), 429
|
||||||
|
try:
|
||||||
|
return jsonify(get_rating_distribution(url))
|
||||||
|
except ScrapeError as exc:
|
||||||
|
return jsonify(error=str(exc)), 422
|
||||||
|
except Exception:
|
||||||
|
app.logger.exception("Unexpected scraper failure")
|
||||||
|
return jsonify(error="Google Maps nie odpowiedziało poprawnie. Spróbuj później."), 502
|
||||||
|
finally:
|
||||||
|
_scrape_lock.release()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
app.run(host=os.environ.get("HOST", "127.0.0.1"), port=int(os.environ.get("PORT", 8080)), debug=False)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
Flask==3.1.2
|
||||||
|
playwright==1.55.0
|
||||||
|
gunicorn==23.0.0
|
||||||
+3
-581
@@ -1,585 +1,7 @@
|
|||||||
import re
|
"""Zgodny wstecznie punkt startowy: python reviews.py."""
|
||||||
import time
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
|
|
||||||
import pandas as pd
|
from app import app
|
||||||
from dateutil.relativedelta import relativedelta
|
|
||||||
from playwright.sync_api import sync_playwright
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# USTAWIENIA
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
URL = "https://maps.app.goo.gl/USRCto35jhoVtmj4A"
|
|
||||||
|
|
||||||
HEADLESS = False
|
|
||||||
|
|
||||||
# Ile kolejnych scrolli bez nowych opinii oznacza koniec
|
|
||||||
MAX_EMPTY_SCROLLS = 8
|
|
||||||
|
|
||||||
# Czas między scrollami
|
|
||||||
SCROLL_DELAY = 1.2
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# PARSOWANIE DATY GOOGLE
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def parse_google_date(text: str):
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
|
|
||||||
text = text.strip().lower()
|
|
||||||
now = datetime.now()
|
|
||||||
|
|
||||||
if text == "dzisiaj":
|
|
||||||
return now.date()
|
|
||||||
|
|
||||||
if text == "wczoraj":
|
|
||||||
return (now - timedelta(days=1)).date()
|
|
||||||
|
|
||||||
patterns = [
|
|
||||||
(r"(\d+)\s+dni?\s+temu", lambda n: now - timedelta(days=n)),
|
|
||||||
(r"(\d+)\s+dzień\s+temu", lambda n: now - timedelta(days=n)),
|
|
||||||
(r"(\d+)\s+tygodni?\s+temu", lambda n: now - timedelta(weeks=n)),
|
|
||||||
(
|
|
||||||
r"(\d+)\s+miesi(?:ąc|ące|ęcy)\s+temu",
|
|
||||||
lambda n: now - relativedelta(months=n)
|
|
||||||
),
|
|
||||||
(
|
|
||||||
r"(\d+)\s+lat(?:a)?\s+temu",
|
|
||||||
lambda n: now - relativedelta(years=n)
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
for pattern, func in patterns:
|
|
||||||
match = re.search(pattern, text)
|
|
||||||
|
|
||||||
if match:
|
|
||||||
return func(int(match.group(1))).date()
|
|
||||||
|
|
||||||
if "tydzień temu" in text:
|
|
||||||
return (now - timedelta(weeks=1)).date()
|
|
||||||
|
|
||||||
if "miesiąc temu" in text:
|
|
||||||
return (now - relativedelta(months=1)).date()
|
|
||||||
|
|
||||||
if "rok temu" in text:
|
|
||||||
return (now - relativedelta(years=1)).date()
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# COOKIES
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def accept_cookies(page):
|
|
||||||
labels = [
|
|
||||||
"Zaakceptuj wszystko",
|
|
||||||
"Accept all",
|
|
||||||
"Odrzuć wszystko",
|
|
||||||
"Reject all",
|
|
||||||
]
|
|
||||||
|
|
||||||
for label in labels:
|
|
||||||
try:
|
|
||||||
button = page.get_by_role(
|
|
||||||
"button",
|
|
||||||
name=re.compile(label, re.I)
|
|
||||||
)
|
|
||||||
|
|
||||||
if button.count() and button.first.is_visible():
|
|
||||||
button.first.click(timeout=3000)
|
|
||||||
time.sleep(2)
|
|
||||||
return
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# OTWIERANIE PANELU OPINII
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def open_reviews_panel(page):
|
|
||||||
print("Szukam właściwego przycisku z opiniami...")
|
|
||||||
|
|
||||||
buttons = page.locator("button")
|
|
||||||
|
|
||||||
for i in range(buttons.count()):
|
|
||||||
try:
|
|
||||||
button = buttons.nth(i)
|
|
||||||
|
|
||||||
if not button.is_visible():
|
|
||||||
continue
|
|
||||||
|
|
||||||
aria = button.get_attribute("aria-label") or ""
|
|
||||||
|
|
||||||
try:
|
|
||||||
text = button.inner_text(timeout=500)
|
|
||||||
except:
|
|
||||||
text = ""
|
|
||||||
|
|
||||||
label = f"{aria} {text}".strip()
|
|
||||||
|
|
||||||
# Musi zawierać liczbę oraz opini/recenz/review
|
|
||||||
if (
|
|
||||||
re.search(r"\d+", label)
|
|
||||||
and re.search(r"(opini|recenz|reviews?)", label, re.I)
|
|
||||||
):
|
|
||||||
print("Kandydat:", repr(label[:200]))
|
|
||||||
|
|
||||||
try:
|
|
||||||
button.scroll_into_view_if_needed()
|
|
||||||
time.sleep(0.5)
|
|
||||||
button.click(timeout=3000)
|
|
||||||
|
|
||||||
print("Kliknięto:", repr(label[:200]))
|
|
||||||
|
|
||||||
time.sleep(3)
|
|
||||||
|
|
||||||
cards = page.locator('div[data-review-id]')
|
|
||||||
|
|
||||||
if cards.count() > 5:
|
|
||||||
print("Panel opinii otwarty.")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print("Nie udało się kliknąć:", e)
|
|
||||||
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Fallback dla role=button
|
|
||||||
print("Próbuję alternatywnego selektora...")
|
|
||||||
|
|
||||||
candidates = page.locator(
|
|
||||||
'[role="button"][aria-label*="opini"], '
|
|
||||||
'[role="button"][aria-label*="recenz"], '
|
|
||||||
'[role="button"][aria-label*="review"]'
|
|
||||||
)
|
|
||||||
|
|
||||||
for i in range(candidates.count()):
|
|
||||||
try:
|
|
||||||
el = candidates.nth(i)
|
|
||||||
|
|
||||||
if not el.is_visible():
|
|
||||||
continue
|
|
||||||
|
|
||||||
aria = el.get_attribute("aria-label") or ""
|
|
||||||
|
|
||||||
if not re.search(r"\d+", aria):
|
|
||||||
continue
|
|
||||||
|
|
||||||
print("Klikam:", repr(aria))
|
|
||||||
|
|
||||||
el.scroll_into_view_if_needed()
|
|
||||||
el.click(timeout=3000)
|
|
||||||
|
|
||||||
time.sleep(3)
|
|
||||||
|
|
||||||
cards = page.locator('div[data-review-id]')
|
|
||||||
|
|
||||||
if cards.count() > 5:
|
|
||||||
print("Panel opinii otwarty.")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except:
|
|
||||||
continue
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# ZNAJDOWANIE SCROLLOWANEGO PANELU
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def get_scrollable_parent(card):
|
|
||||||
try:
|
|
||||||
return card.evaluate_handle("""
|
|
||||||
el => {
|
|
||||||
let p = el.parentElement;
|
|
||||||
|
|
||||||
while (p) {
|
|
||||||
const style = getComputedStyle(p);
|
|
||||||
|
|
||||||
if (
|
|
||||||
(style.overflowY === 'auto' ||
|
|
||||||
style.overflowY === 'scroll') &&
|
|
||||||
p.scrollHeight > p.clientHeight
|
|
||||||
) {
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
p = p.parentElement;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
except:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# OCENA GWIAZDKOWA
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def extract_rating(card):
|
|
||||||
selectors = [
|
|
||||||
'[role="img"][aria-label*="gwiazd"]',
|
|
||||||
'[aria-label*="gwiazd"]',
|
|
||||||
'[role="img"][aria-label*="star"]',
|
|
||||||
'[aria-label*="star"]',
|
|
||||||
'span.kvMYJc[aria-label]',
|
|
||||||
]
|
|
||||||
|
|
||||||
for selector in selectors:
|
|
||||||
try:
|
|
||||||
el = card.locator(selector).first
|
|
||||||
|
|
||||||
if not el.count():
|
|
||||||
continue
|
|
||||||
|
|
||||||
label = el.get_attribute("aria-label") or ""
|
|
||||||
|
|
||||||
match = re.search(r"([1-5](?:[.,]\d+)?)", label)
|
|
||||||
|
|
||||||
if match:
|
|
||||||
value = match.group(1).replace(",", ".")
|
|
||||||
return int(float(value))
|
|
||||||
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# DATA OPINII
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def extract_date_text(card):
|
|
||||||
selectors = [
|
|
||||||
".rsqaWe",
|
|
||||||
"span.rsqaWe",
|
|
||||||
]
|
|
||||||
|
|
||||||
for selector in selectors:
|
|
||||||
try:
|
|
||||||
el = card.locator(selector).first
|
|
||||||
|
|
||||||
if el.count():
|
|
||||||
text = el.inner_text().strip()
|
|
||||||
|
|
||||||
if text:
|
|
||||||
return text
|
|
||||||
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# ŁADOWANIE WSZYSTKICH OPINII
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def count_unique_reviews(cards):
|
|
||||||
"""Policz opinie, a nie zduplikowane elementy DOM Google Maps."""
|
|
||||||
try:
|
|
||||||
return cards.evaluate_all("""
|
|
||||||
elements => new Set(
|
|
||||||
elements
|
|
||||||
.map(el => el.getAttribute('data-review-id'))
|
|
||||||
.filter(Boolean)
|
|
||||||
).size
|
|
||||||
""")
|
|
||||||
except:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def load_all_reviews(page):
|
|
||||||
previous_count = 0
|
|
||||||
empty_scrolls = 0
|
|
||||||
|
|
||||||
print("\nŁadowanie opinii...")
|
|
||||||
|
|
||||||
while empty_scrolls < MAX_EMPTY_SCROLLS:
|
|
||||||
cards = page.locator('div[data-review-id]')
|
|
||||||
count = count_unique_reviews(cards)
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"\rZaładowane opinie: {count}",
|
|
||||||
end="",
|
|
||||||
flush=True
|
|
||||||
)
|
|
||||||
|
|
||||||
if count > previous_count:
|
|
||||||
previous_count = count
|
|
||||||
empty_scrolls = 0
|
|
||||||
else:
|
|
||||||
empty_scrolls += 1
|
|
||||||
|
|
||||||
if count == 0:
|
|
||||||
page.mouse.wheel(0, 5000)
|
|
||||||
time.sleep(SCROLL_DELAY)
|
|
||||||
continue
|
|
||||||
|
|
||||||
last_card = cards.last
|
|
||||||
|
|
||||||
scrollable = get_scrollable_parent(last_card)
|
|
||||||
|
|
||||||
if scrollable:
|
|
||||||
try:
|
|
||||||
scrollable.evaluate("""
|
|
||||||
el => {
|
|
||||||
el.scrollTop = el.scrollHeight;
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
except:
|
|
||||||
page.mouse.wheel(0, 5000)
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
last_card.scroll_into_view_if_needed()
|
|
||||||
except:
|
|
||||||
page.mouse.wheel(0, 5000)
|
|
||||||
|
|
||||||
time.sleep(SCROLL_DELAY)
|
|
||||||
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# ZBIERANIE DANYCH
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def extract_reviews(page):
|
|
||||||
results = {}
|
|
||||||
|
|
||||||
cards = page.locator('div[data-review-id]')
|
|
||||||
|
|
||||||
unique_count = count_unique_reviews(cards)
|
|
||||||
print(f"\nZbieram dane z {unique_count} unikalnych opinii...")
|
|
||||||
|
|
||||||
for i in range(cards.count()):
|
|
||||||
card = cards.nth(i)
|
|
||||||
|
|
||||||
try:
|
|
||||||
review_id = card.get_attribute("data-review-id")
|
|
||||||
|
|
||||||
if not review_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Deduplikacja
|
|
||||||
if review_id in results:
|
|
||||||
continue
|
|
||||||
|
|
||||||
rating = extract_rating(card)
|
|
||||||
|
|
||||||
if rating is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
date_raw = extract_date_text(card)
|
|
||||||
date_approx = parse_google_date(date_raw)
|
|
||||||
|
|
||||||
results[review_id] = {
|
|
||||||
"review_id": review_id,
|
|
||||||
"rating": rating,
|
|
||||||
"date_raw": date_raw,
|
|
||||||
"date_approx": date_approx,
|
|
||||||
}
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Błąd przy opinii #{i}: {e}")
|
|
||||||
|
|
||||||
return pd.DataFrame(results.values())
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# PODSUMOWANIE
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def print_summary(df):
|
|
||||||
print("\n==============================")
|
|
||||||
print("ŁĄCZNIE")
|
|
||||||
print("==============================")
|
|
||||||
print(len(df))
|
|
||||||
|
|
||||||
print("\n==============================")
|
|
||||||
print("ROZKŁAD GWIAZDEK")
|
|
||||||
print("==============================")
|
|
||||||
|
|
||||||
distribution = (
|
|
||||||
df["rating"]
|
|
||||||
.value_counts()
|
|
||||||
.reindex([5, 4, 3, 2, 1], fill_value=0)
|
|
||||||
)
|
|
||||||
|
|
||||||
total = len(df)
|
|
||||||
|
|
||||||
for stars, count in distribution.items():
|
|
||||||
percent = (count / total * 100) if total else 0
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"{stars}★: "
|
|
||||||
f"{count:>4} "
|
|
||||||
f"({percent:>5.1f}%)"
|
|
||||||
)
|
|
||||||
|
|
||||||
print("\n==============================")
|
|
||||||
print("ŚREDNIA")
|
|
||||||
print("==============================")
|
|
||||||
|
|
||||||
if len(df):
|
|
||||||
print(round(df["rating"].mean(), 3))
|
|
||||||
else:
|
|
||||||
print("Brak danych")
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# FILTROWANIE PO OKRESIE
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def print_period_summary(df, date_from=None, date_to=None):
|
|
||||||
temp = df.copy()
|
|
||||||
|
|
||||||
temp["date_approx"] = pd.to_datetime(
|
|
||||||
temp["date_approx"],
|
|
||||||
errors="coerce"
|
|
||||||
)
|
|
||||||
|
|
||||||
if date_from:
|
|
||||||
temp = temp[
|
|
||||||
temp["date_approx"] >= pd.to_datetime(date_from)
|
|
||||||
]
|
|
||||||
|
|
||||||
if date_to:
|
|
||||||
temp = temp[
|
|
||||||
temp["date_approx"] <= pd.to_datetime(date_to)
|
|
||||||
]
|
|
||||||
|
|
||||||
print("\n==============================")
|
|
||||||
print(f"OKRES: {date_from} -> {date_to}")
|
|
||||||
print("==============================")
|
|
||||||
|
|
||||||
if temp.empty:
|
|
||||||
print("Brak opinii w tym okresie.")
|
|
||||||
return
|
|
||||||
|
|
||||||
distribution = (
|
|
||||||
temp["rating"]
|
|
||||||
.value_counts()
|
|
||||||
.reindex([5, 4, 3, 2, 1], fill_value=0)
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Liczba opinii: {len(temp)}")
|
|
||||||
|
|
||||||
for stars, count in distribution.items():
|
|
||||||
print(f"{stars}★: {count}")
|
|
||||||
|
|
||||||
print("Średnia:", round(temp["rating"].mean(), 3))
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# MAIN
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def main():
|
|
||||||
with sync_playwright() as p:
|
|
||||||
browser = p.chromium.launch(
|
|
||||||
headless=HEADLESS
|
|
||||||
)
|
|
||||||
|
|
||||||
context = browser.new_context(
|
|
||||||
locale="pl-PL",
|
|
||||||
viewport={
|
|
||||||
"width": 1400,
|
|
||||||
"height": 900,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
page = context.new_page()
|
|
||||||
|
|
||||||
print("Otwieram Google Maps...")
|
|
||||||
|
|
||||||
page.goto(
|
|
||||||
URL,
|
|
||||||
wait_until="domcontentloaded",
|
|
||||||
timeout=60_000
|
|
||||||
)
|
|
||||||
|
|
||||||
time.sleep(3)
|
|
||||||
|
|
||||||
accept_cookies(page)
|
|
||||||
|
|
||||||
opened = open_reviews_panel(page)
|
|
||||||
|
|
||||||
if not opened:
|
|
||||||
print(
|
|
||||||
"\nNie udało się automatycznie otworzyć panelu opinii."
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"Kliknij ręcznie liczbę opinii / „Więcej opinii”."
|
|
||||||
)
|
|
||||||
|
|
||||||
input(
|
|
||||||
"Gdy panel będzie otwarty, naciśnij ENTER..."
|
|
||||||
)
|
|
||||||
|
|
||||||
time.sleep(2)
|
|
||||||
|
|
||||||
cards = page.locator('div[data-review-id]')
|
|
||||||
|
|
||||||
print(
|
|
||||||
"Opinie widoczne po otwarciu panelu:",
|
|
||||||
count_unique_reviews(cards)
|
|
||||||
)
|
|
||||||
|
|
||||||
if count_unique_reviews(cards) < 5:
|
|
||||||
print(
|
|
||||||
"\nUWAGA: panel opinii prawdopodobnie "
|
|
||||||
"nie został poprawnie otwarty."
|
|
||||||
)
|
|
||||||
|
|
||||||
input(
|
|
||||||
"Otwórz ręcznie wszystkie opinie "
|
|
||||||
"i naciśnij ENTER..."
|
|
||||||
)
|
|
||||||
|
|
||||||
load_all_reviews(page)
|
|
||||||
|
|
||||||
df = extract_reviews(page)
|
|
||||||
|
|
||||||
browser.close()
|
|
||||||
|
|
||||||
if df.empty:
|
|
||||||
print("\nNie znaleziono żadnych opinii.")
|
|
||||||
return
|
|
||||||
|
|
||||||
df.to_csv(
|
|
||||||
"google_reviews_ratings.csv",
|
|
||||||
index=False,
|
|
||||||
encoding="utf-8-sig"
|
|
||||||
)
|
|
||||||
|
|
||||||
print_summary(df)
|
|
||||||
|
|
||||||
print("\nZapisano:")
|
|
||||||
print("google_reviews_ratings.csv")
|
|
||||||
|
|
||||||
# ========================================================
|
|
||||||
# PRZYKŁADOWE FILTROWANIE
|
|
||||||
# Odkomentuj jeśli chcesz
|
|
||||||
# ========================================================
|
|
||||||
|
|
||||||
# print_period_summary(
|
|
||||||
# df,
|
|
||||||
# "2025-01-01",
|
|
||||||
# "2025-12-31"
|
|
||||||
# )
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
app.run(host="127.0.0.1", port=8080, debug=False)
|
||||||
|
|||||||
+137
@@ -0,0 +1,137 @@
|
|||||||
|
import os
|
||||||
|
import re
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
|
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||||
|
from playwright.sync_api import sync_playwright
|
||||||
|
|
||||||
|
|
||||||
|
ALLOWED_HOSTS = {
|
||||||
|
"maps.app.goo.gl", "goo.gl", "google.com", "www.google.com",
|
||||||
|
"maps.google.com", "google.pl", "www.google.pl", "maps.google.pl",
|
||||||
|
}
|
||||||
|
MAX_URL_LENGTH = 1200
|
||||||
|
|
||||||
|
|
||||||
|
class ScrapeError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def validate_maps_url(raw_url: str) -> str:
|
||||||
|
"""Przepuszcza wyłącznie zwykłe linki HTTPS do Google Maps."""
|
||||||
|
url = raw_url.strip()
|
||||||
|
if not url:
|
||||||
|
raise ValueError("Wklej link do miejsca w Google Maps.")
|
||||||
|
if len(url) > MAX_URL_LENGTH or any(ord(char) < 32 for char in url):
|
||||||
|
raise ValueError("Link jest nieprawidłowy albo zbyt długi.")
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
host = (parsed.hostname or "").lower().rstrip(".")
|
||||||
|
port = parsed.port
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError("Link ma nieprawidłowy format.") from None
|
||||||
|
if parsed.scheme.lower() != "https" or not host or parsed.username or parsed.password:
|
||||||
|
raise ValueError("Dozwolony jest wyłącznie bezpieczny link HTTPS do Google Maps.")
|
||||||
|
if port not in (None, 443) or host not in ALLOWED_HOSTS:
|
||||||
|
raise ValueError("To nie jest obsługiwany link Google Maps.")
|
||||||
|
|
||||||
|
path = parsed.path or "/"
|
||||||
|
if host == "maps.app.goo.gl":
|
||||||
|
valid_path = re.fullmatch(r"/[A-Za-z0-9_-]{5,100}/?", path)
|
||||||
|
elif host == "goo.gl":
|
||||||
|
valid_path = re.fullmatch(r"/maps/[A-Za-z0-9_-]{5,100}/?", path)
|
||||||
|
else:
|
||||||
|
valid_path = path == "/maps" or path.startswith("/maps/")
|
||||||
|
if not valid_path:
|
||||||
|
raise ValueError("Link musi prowadzić do miejsca w Google Maps.")
|
||||||
|
return urlunsplit(("https", host, path, parsed.query, ""))
|
||||||
|
|
||||||
|
|
||||||
|
def _number(text: str) -> int | None:
|
||||||
|
compact = re.sub(r"[\s\u00a0\u202f.,]", "", text)
|
||||||
|
return int(compact) if compact.isdigit() else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_distribution_labels(labels: list[str]) -> dict[int, int]:
|
||||||
|
"""Czyta np. '5 gwiazdek, 123 opinie' — nie mierzy szerokości paska."""
|
||||||
|
result = {}
|
||||||
|
for label in labels:
|
||||||
|
clean = " ".join(label.lower().split())
|
||||||
|
if not re.search(r"gwiazd|star", clean) or not re.search(r"opini|recenz|review", clean):
|
||||||
|
continue
|
||||||
|
rating_match = re.search(r"(?:^|\D)([1-5])(?:\D|$)", clean)
|
||||||
|
if not rating_match:
|
||||||
|
continue
|
||||||
|
candidates = re.findall(r"\d[\d\s\u00a0\u202f.,]*", clean[rating_match.end():])
|
||||||
|
values = [_number(item) for item in candidates]
|
||||||
|
values = [item for item in values if item is not None]
|
||||||
|
if values:
|
||||||
|
result[int(rating_match.group(1))] = values[-1]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _accept_cookies(page):
|
||||||
|
for text in ("Zaakceptuj wszystko", "Odrzuć wszystko", "Accept all", "Reject all"):
|
||||||
|
try:
|
||||||
|
button = page.get_by_role("button", name=re.compile(f"^{re.escape(text)}$", re.I)).first
|
||||||
|
if button.is_visible(timeout=700):
|
||||||
|
button.click(timeout=2_000)
|
||||||
|
page.wait_for_timeout(500)
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _open_reviews(page):
|
||||||
|
candidates = page.locator("button, [role=button]")
|
||||||
|
for index in range(min(candidates.count(), 250)):
|
||||||
|
element = candidates.nth(index)
|
||||||
|
try:
|
||||||
|
label = " ".join(filter(None, [
|
||||||
|
element.get_attribute("aria-label"), element.inner_text(timeout=100)
|
||||||
|
]))
|
||||||
|
if re.search(r"\d", label) and re.search(r"opini|recenz|review", label, re.I):
|
||||||
|
element.click(timeout=2_000)
|
||||||
|
page.wait_for_timeout(800)
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_rating_distribution(url: str) -> dict:
|
||||||
|
"""Pobiera pięć zbiorczych liczników. Treści opinii nie są odczytywane ani zapisywane."""
|
||||||
|
try:
|
||||||
|
with sync_playwright() as playwright:
|
||||||
|
launch = {"headless": True, "args": ["--disable-dev-shm-usage", "--no-first-run", "--disable-gpu"]}
|
||||||
|
if os.environ.get("CHROMIUM_PATH"):
|
||||||
|
launch["executable_path"] = os.environ["CHROMIUM_PATH"]
|
||||||
|
browser = playwright.chromium.launch(**launch)
|
||||||
|
try:
|
||||||
|
page = browser.new_page(locale="pl-PL", viewport={"width": 1100, "height": 800})
|
||||||
|
page.goto(url, wait_until="domcontentloaded", timeout=30_000)
|
||||||
|
_accept_cookies(page)
|
||||||
|
_open_reviews(page)
|
||||||
|
page.wait_for_timeout(700)
|
||||||
|
labels = page.locator("[aria-label]").evaluate_all(
|
||||||
|
"els => els.map(el => el.getAttribute('aria-label')).filter(Boolean)"
|
||||||
|
)
|
||||||
|
counts = parse_distribution_labels(labels)
|
||||||
|
if len(counts) != 5:
|
||||||
|
body_text = page.locator("body").inner_text().lower()
|
||||||
|
if "ograniczonego widoku" in body_text or "limited view" in body_text:
|
||||||
|
raise ScrapeError("Google włączyło ograniczony widok i ukryło opinie. Spróbuj później lub z innego łącza.")
|
||||||
|
raise ScrapeError("Nie udało się odczytać dokładnych pięciu liczników. Sprawdź link do miejsca.")
|
||||||
|
ratings = {str(stars): counts[stars] for stars in range(5, 0, -1)}
|
||||||
|
total = sum(ratings.values())
|
||||||
|
average = round(sum(int(stars) * count for stars, count in ratings.items()) / total, 2) if total else 0
|
||||||
|
return {"total": total, "average": average, "ratings": ratings}
|
||||||
|
finally:
|
||||||
|
browser.close()
|
||||||
|
except ScrapeError:
|
||||||
|
raise
|
||||||
|
except PlaywrightTimeoutError:
|
||||||
|
raise ScrapeError("Google Maps nie odpowiedziało w wyznaczonym czasie.") from None
|
||||||
|
except Exception as exc:
|
||||||
|
if "Executable doesn't exist" in str(exc):
|
||||||
|
raise ScrapeError("Chromium nie jest zainstalowane na serwerze.") from None
|
||||||
|
raise
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
const form = document.querySelector('#review-form');
|
||||||
|
const statusBox = document.querySelector('#status');
|
||||||
|
const resultBox = document.querySelector('#result');
|
||||||
|
const button = form.querySelector('button');
|
||||||
|
|
||||||
|
form.addEventListener('submit', async (event) => {
|
||||||
|
event.preventDefault(); button.disabled = true; resultBox.hidden = true;
|
||||||
|
statusBox.className = 'status'; statusBox.textContent = 'Odczytuję dokładne liczniki…';
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/reviews', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({url: document.querySelector('#maps-url').value})});
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok) throw new Error(data.error || 'Nie udało się pobrać danych.');
|
||||||
|
document.querySelector('#total').textContent = data.total.toLocaleString('pl-PL');
|
||||||
|
document.querySelector('#average').textContent = data.average.toLocaleString('pl-PL', {minimumFractionDigits: 2});
|
||||||
|
const max = Math.max(...Object.values(data.ratings), 1);
|
||||||
|
const rows = Object.entries(data.ratings).map(([stars, count]) => {
|
||||||
|
const row = document.createElement('div'); row.className = 'rating-row';
|
||||||
|
const label = document.createElement('span'); label.textContent = `${stars} ★`;
|
||||||
|
const bar = document.createElement('div'); bar.className = 'bar';
|
||||||
|
const fill = document.createElement('i'); fill.style.width = `${count / max * 100}%`; bar.append(fill);
|
||||||
|
const value = document.createElement('strong'); value.textContent = count.toLocaleString('pl-PL');
|
||||||
|
row.append(label, bar, value); return row;
|
||||||
|
});
|
||||||
|
document.querySelector('#ratings').replaceChildren(...rows);
|
||||||
|
statusBox.textContent = ''; resultBox.hidden = false;
|
||||||
|
} catch (error) { statusBox.className = 'status error'; statusBox.textContent = error.message; }
|
||||||
|
finally { button.disabled = false; }
|
||||||
|
});
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
:root { font-family: Inter, ui-sans-serif, system-ui, sans-serif; color: #202124; background: #f3f5f7; }
|
||||||
|
* { box-sizing: border-box; } body { margin: 0; min-height: 100vh; }
|
||||||
|
main { min-height: 100vh; display: grid; place-items: center; padding: 24px; }
|
||||||
|
.card { width: min(680px, 100%); padding: clamp(24px, 6vw, 52px); background: #fff; border: 1px solid #e2e6ea; border-radius: 24px; box-shadow: 0 18px 55px #1f293714; }
|
||||||
|
.eyebrow { margin: 0 0 8px; color: #1967d2; font-size: .78rem; font-weight: 800; letter-spacing: .12em; text-transform: uppercase; }
|
||||||
|
h1 { margin: 0; font-size: clamp(1.8rem, 5vw, 2.65rem); line-height: 1.08; } .intro { margin: 14px 0 30px; color: #697079; }
|
||||||
|
label { display: block; margin-bottom: 8px; font-size: .9rem; font-weight: 700; } .input-row { display: flex; gap: 10px; }
|
||||||
|
input { min-width: 0; flex: 1; padding: 14px 16px; border: 1px solid #c8cdd2; border-radius: 12px; font: inherit; }
|
||||||
|
input:focus { outline: 3px solid #1a73e822; border-color: #1a73e8; }
|
||||||
|
button { padding: 14px 22px; border: 0; border-radius: 12px; color: #fff; background: #1a73e8; font: inherit; font-weight: 750; cursor: pointer; }
|
||||||
|
button:hover { background: #155fc0; } button:disabled { opacity: .6; cursor: wait; }
|
||||||
|
.status { min-height: 24px; margin: 16px 0 0; color: #59636e; } .status.error { color: #b3261e; }
|
||||||
|
.result { margin-top: 26px; border-top: 1px solid #e6e9ec; padding-top: 26px; } .summary { display: flex; gap: 54px; margin-bottom: 26px; }
|
||||||
|
.summary span { display: block; font-size: 2rem; font-weight: 800; } .summary small { color: #737b84; }
|
||||||
|
.rating-row { display: grid; grid-template-columns: 42px 1fr 70px; align-items: center; gap: 12px; margin: 11px 0; }
|
||||||
|
.rating-row > span { color: #9a6700; font-weight: 700; } .rating-row strong { text-align: right; font-variant-numeric: tabular-nums; }
|
||||||
|
.bar { height: 10px; overflow: hidden; border-radius: 99px; background: #edf0f2; } .bar i { display: block; height: 100%; border-radius: inherit; background: #fbbc04; }
|
||||||
|
@media (max-width: 560px) { .input-row { flex-direction: column; } button { width: 100%; } }
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="pl">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Rozkład opinii Google</title>
|
||||||
|
<link rel="stylesheet" href="/static/style.css">
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main><section class="card">
|
||||||
|
<p class="eyebrow">Google Maps</p>
|
||||||
|
<h1>Policz opinie według oceny</h1>
|
||||||
|
<p class="intro">Wklej link do firmy lub miejsca. Aplikacja niczego nie zapisuje.</p>
|
||||||
|
<form id="review-form">
|
||||||
|
<label for="maps-url">Link Google Maps</label>
|
||||||
|
<div class="input-row">
|
||||||
|
<input id="maps-url" type="url" maxlength="1200" placeholder="https://maps.app.goo.gl/…" required autocomplete="off">
|
||||||
|
<button type="submit">Sprawdź</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
<p id="status" class="status" role="status" aria-live="polite"></p>
|
||||||
|
<section id="result" class="result" hidden>
|
||||||
|
<div class="summary">
|
||||||
|
<div><span id="total">0</span><small>opinii łącznie</small></div>
|
||||||
|
<div><span id="average">0</span><small>średnia</small></div>
|
||||||
|
</div>
|
||||||
|
<div id="ratings" class="ratings"></div>
|
||||||
|
</section>
|
||||||
|
</section></main>
|
||||||
|
<script src="/static/app.js" defer></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from app import app
|
||||||
|
from scraper import parse_distribution_labels, validate_maps_url
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationTests(unittest.TestCase):
|
||||||
|
def test_accepts_google_maps_links(self):
|
||||||
|
urls = [
|
||||||
|
"https://maps.app.goo.gl/USRCto35jhoVtmj4A",
|
||||||
|
"https://www.google.com/maps/place/Test/@50,20,15z",
|
||||||
|
"https://maps.google.pl/maps?q=test",
|
||||||
|
]
|
||||||
|
for url in urls:
|
||||||
|
with self.subTest(url=url):
|
||||||
|
self.assertTrue(validate_maps_url(url).startswith("https://"))
|
||||||
|
|
||||||
|
def test_rejects_unsafe_links(self):
|
||||||
|
urls = [
|
||||||
|
"http://maps.app.goo.gl/USRCto35jhoVtmj4A",
|
||||||
|
"https://evil.example/maps/x",
|
||||||
|
"https://google.com.evil.example/maps/x",
|
||||||
|
"https://user:pass@www.google.com/maps/x",
|
||||||
|
"https://www.google.com:444/maps/x",
|
||||||
|
"https://www.google.com/search?q=x",
|
||||||
|
"file:///etc/passwd",
|
||||||
|
]
|
||||||
|
for url in urls:
|
||||||
|
with self.subTest(url=url), self.assertRaises(ValueError):
|
||||||
|
validate_maps_url(url)
|
||||||
|
|
||||||
|
def test_parses_exact_accessibility_labels(self):
|
||||||
|
labels = [
|
||||||
|
"5 gwiazdek, 1 234 opinie", "4 gwiazdki, 56 opinii",
|
||||||
|
"3 stars, 7 reviews", "2 stars, 1 review", "1 gwiazdka, 0 opinii",
|
||||||
|
]
|
||||||
|
self.assertEqual(parse_distribution_labels(labels), {5: 1234, 4: 56, 3: 7, 2: 1, 1: 0})
|
||||||
|
|
||||||
|
|
||||||
|
class ApiTests(unittest.TestCase):
|
||||||
|
def test_home_page(self):
|
||||||
|
self.assertEqual(app.test_client().get("/").status_code, 200)
|
||||||
|
|
||||||
|
def test_rejects_non_google_url(self):
|
||||||
|
response = app.test_client().post("/api/reviews", json={"url": "https://example.com/"})
|
||||||
|
self.assertEqual(response.status_code, 400)
|
||||||
|
|
||||||
|
def test_rejects_large_request(self):
|
||||||
|
response = app.test_client().post("/api/reviews", data=b"x" * 5000, content_type="application/json")
|
||||||
|
self.assertEqual(response.status_code, 413)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user