silas.brito/leadradar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: extracao profunda de detalhes do Maps abrindo o painel lateral de cada estabelecimento (Telefone, Endereco, Website)

Browse Source
This commit is contained in:
Silas Andrade de Brito
2026-08-26 17:41:15 -03:00
parent 4c8f1f8299
commit fcb6468426
1 changed files with 160 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/scraper_service.py
+160 -72
View File
@@ -15,7 +15,9 @@ class ScraperService:
@staticmethod @staticmethod
def scrape_google_maps(search_query: str, max_results: int = 15, headless: bool = True) -> List[Dict[str, Any]]: def scrape_google_maps(search_query: str, max_results: int = 15, headless: bool = True) -> List[Dict[str, Any]]:
""" """
Executa o Playwright em modo headless para buscar no Google Maps com extração 100% real de estabelecimentos. Executa o Playwright em modo headless para buscar no Google Maps.
Clica em cada estabelecimento para abrir o painel lateral de detalhes
e extrair dados completos: Telefone, Endereço completo, Website e Rating.
""" """
results = [] results = []
try: try:
@@ -42,18 +44,16 @@ class ScraperService:
locale='pt-BR' locale='pt-BR'
) )
page = context.new_page() page = context.new_page()
# Prevenir detecção do webdriver
page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
encoded_query = search_query.replace(' ', '+') encoded_query = search_query.replace(' ', '+')
url = f"https://www.google.com/maps/search/{encoded_query}" url = f"https://www.google.com/maps/search/{encoded_query}"
logger.info(f"Prospecção no Maps: {url}") logger.info(f"Iniciando varredura detalhada no Maps: {url}")
page.goto(url, wait_until='domcontentloaded', timeout=40000) page.goto(url, wait_until='domcontentloaded', timeout=40000)
time.sleep(3) time.sleep(3)
# Fechar dialogs de consentimento de cookies do Google caso apareçam # Aceitar cookies se houver modal
try: try:
accept_btn = page.query_selector("button[aria-label*='Aceitar'], button[aria-label*='Accept']") accept_btn = page.query_selector("button[aria-label*='Aceitar'], button[aria-label*='Accept']")
if accept_btn: if accept_btn:
@@ -62,89 +62,175 @@ class ScraperService:
except Exception: except Exception:
pass pass
# Aguardar feed de resultados ou container principal de busca
feed_selector = "div[role='feed']" feed_selector = "div[role='feed']"
has_feed = True
try: try:
page.wait_for_selector(feed_selector, timeout=12000) page.wait_for_selector(feed_selector, timeout=8000)
except Exception: except Exception:
# Se for redirecionado direto para 1 único resultado has_feed = False
logger.info("Verificando se a busca caiu diretamente no painel de 1 lugar.") logger.info("Feed de múltiplos resultados não localizado. Verificando painel de resultado único.")
# Scroll progressivo no feed de resultados # Se houver feed de múltiplos resultados, fazer scroll progressivo
for scroll_step in range(min(8, (max_results // 2) + 2)): if has_feed:
page.evaluate(f""" for _ in range(min(6, (max_results // 3) + 2)):
const feed = document.querySelector("{feed_selector}"); page.evaluate(f"""
if (feed) {{ const feed = document.querySelector("{feed_selector}");
feed.scrollBy(0, 1200); if (feed) {{
}} feed.scrollBy(0, 1000);
""") }}
time.sleep(1.2) """)
time.sleep(1.0)
# Extrair os links dos lugares listados card_elements = page.query_selector_all("a[href*='/maps/place/']")
card_elements = page.query_selector_all("a[href*='/maps/place/']") seen_urls = set()
seen_urls = set()
for card in card_elements: for card in card_elements:
if len(results) >= max_results: if len(results) >= max_results:
break break
try: try:
href = card.get_attribute("href") href = card.get_attribute("href")
if not href or href in seen_urls: if not href or href in seen_urls:
continue continue
seen_urls.add(href) seen_urls.add(href)
# Extrair o nome da empresa via aria-label ou texto interno # Clicar no card para abrir o painel lateral de detalhes no Maps
aria_label = card.get_attribute("aria-label") or ""
card_text = card.inner_text() or ""
lines = [l.strip() for l in card_text.split('\n') if l.strip()]
nome_empresa = aria_label.strip() if aria_label else (lines[0] if lines else "")
if not nome_empresa:
continue
# Extrair avaliação e total de reviews
rating = 0.0
total_avaliacoes = 0
# Procura regex por padroes como "4,8 (150)" ou "4.8"
rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', card_text)
if rating_match:
try: try:
rating = float(rating_match.group(1).replace(',', '.')) card.click()
if rating_match.group(2): time.sleep(1.2)
total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2))) except Exception:
except ValueError:
pass pass
# Extrair número de telefone do card text se presente detail_panel = page.query_selector("div[role='main']") or page.query_selector("div.m6QEbd")
phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', card_text) detail_text = detail_panel.inner_text() if detail_panel else card.inner_text()
telefone_raw = phone_match.group(1) if phone_match else ""
# Extrair trecho de endereço ou bairro das linhas # 1. Nome da Empresa (h1 do painel ou aria-label do card)
endereco = "" h1_elem = page.query_selector("h1")
for line in lines: nome_empresa = ""
if any(term in line.lower() for term in ['r.', 'rua', 'av.', 'avenida', 'alameda', 'bairro', 'praça', 'nº', 'dr.', 'doutor', 'centro']): if h1_elem:
if line != nome_empresa: nome_empresa = h1_elem.inner_text().strip()
endereco = line if not nome_empresa:
break nome_empresa = (card.get_attribute("aria-label") or "").strip()
if not nome_empresa:
lines = [l.strip() for l in card.inner_text().split('\n') if l.strip()]
nome_empresa = lines[0] if lines else ""
results.append({ if not nome_empresa:
'nome_empresa': nome_empresa, continue
'telefone': telefone_raw,
'endereco': endereco, # 2. Telefone detalhado
'google_rating': rating, telefone_raw = ""
'total_avaliacoes': total_avaliacoes, phone_btn = page.query_selector("button[data-tooltip*='telefone'], button[data-item-id*='phone'], [aria-label*='Telefone'], [aria-label*='Ligar']")
'google_maps_url': href, if phone_btn:
'website': '' phone_attr = phone_btn.get_attribute("aria-label") or phone_btn.inner_text()
}) phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', phone_attr)
except Exception as item_err: if phone_match:
logger.error(f"Erro ao processar item do Maps: {item_err}") telefone_raw = phone_match.group(1)
continue
if not telefone_raw:
phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', detail_text)
if phone_match:
telefone_raw = phone_match.group(1)
# 3. Endereço Completo
endereco = ""
addr_btn = page.query_selector("button[data-tooltip*='endereço'], button[data-item-id*='address'], [aria-label*='Endereço']")
if addr_btn:
addr_text = addr_btn.get_attribute("aria-label") or addr_btn.inner_text()
addr_clean = re.sub(r'^[Ee]ndereço:\s*', '', addr_text).strip()
if addr_clean:
endereco = addr_clean
if not endereco:
for line in detail_text.split('\n'):
line_str = line.strip()
if any(term in line_str.lower() for term in ['r.', 'rua', 'av.', 'avenida', 'alameda', 'bairro', 'praça', 'nº', 'dr.', 'centro', 'sp', 'rj', 'mg']):
if line_str != nome_empresa and not re.search(r'^\d\.\d', line_str):
endereco = line_str
break
# 4. Website
website = ""
site_btn = page.query_selector("a[data-tooltip*='site'], a[data-item-id*='authority']")
if site_btn:
website = site_btn.get_attribute("href") or ""
# 5. Rating e Avaliações
rating = 0.0
total_avaliacoes = 0
rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', detail_text)
if rating_match:
try:
rating = float(rating_match.group(1).replace(',', '.'))
if rating_match.group(2):
total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2)))
except ValueError:
pass
results.append({
'nome_empresa': nome_empresa,
'telefone': telefone_raw,
'endereco': endereco,
'google_rating': rating,
'total_avaliacoes': total_avaliacoes,
'google_maps_url': href,
'website': website
})
except Exception as item_err:
logger.error(f"Erro ao detalhar lugar do Maps: {item_err}")
continue
else:
# Se for resultado único direto
try:
h1_elem = page.query_selector("h1")
if h1_elem:
nome_empresa = h1_elem.inner_text().strip()
detail_panel = page.query_selector("div[role='main']") or page.query_selector("div.m6QEbd")
detail_text = detail_panel.inner_text() if detail_panel else ""
telefone_raw = ""
phone_btn = page.query_selector("button[data-tooltip*='telefone'], button[data-item-id*='phone'], [aria-label*='Telefone']")
if phone_btn:
phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', phone_btn.get_attribute("aria-label") or phone_btn.inner_text())
if phone_match:
telefone_raw = phone_match.group(1)
endereco = ""
addr_btn = page.query_selector("button[data-tooltip*='endereço'], button[data-item-id*='address']")
if addr_btn:
endereco = re.sub(r'^[Ee]ndereço:\s*', '', addr_btn.get_attribute("aria-label") or addr_btn.inner_text()).strip()
website = ""
site_btn = page.query_selector("a[data-tooltip*='site'], a[data-item-id*='authority']")
if site_btn:
website = site_btn.get_attribute("href") or ""
rating = 0.0
total_avaliacoes = 0
rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', detail_text)
if rating_match:
try:
rating = float(rating_match.group(1).replace(',', '.'))
if rating_match.group(2):
total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2)))
except ValueError:
pass
results.append({
'nome_empresa': nome_empresa,
'telefone': telefone_raw,
'endereco': endereco,
'google_rating': rating,
'total_avaliacoes': total_avaliacoes,
'google_maps_url': page.url,
'website': website
})
except Exception as single_err:
logger.error(f"Erro ao processar resultado único no Maps: {single_err}")
browser.close() browser.close()
except Exception as err: except Exception as err:
logger.error(f"Erro na execução do Playwright Scraper real: {err}") logger.error(f"Erro na execução do Playwright Scraper detalhado: {err}")
return results return results
@@ -197,6 +283,8 @@ class ScraperService:
existing_lead.total_avaliacoes = item['total_avaliacoes'] existing_lead.total_avaliacoes = item['total_avaliacoes']
if item.get('google_maps_url'): if item.get('google_maps_url'):
existing_lead.google_maps_url = item['google_maps_url'] existing_lead.google_maps_url = item['google_maps_url']
if item.get('website') and not existing_lead.website:
existing_lead.website = item['website']
existing_lead.atualizado_em = db.func.now() existing_lead.atualizado_em = db.func.now()