From fcb64684262f628a542360a8b68c273bf0432238 Mon Sep 17 00:00:00 2001 From: Silas Brito Date: Wed, 26 Aug 2026 17:41:15 -0300 Subject: [PATCH] fix: extracao profunda de detalhes do Maps abrindo o painel lateral de cada estabelecimento (Telefone, Endereco, Website) --- backend/app/services/scraper_service.py | 232 ++++++++++++++++-------- 1 file changed, 160 insertions(+), 72 deletions(-) diff --git a/backend/app/services/scraper_service.py b/backend/app/services/scraper_service.py index fac7f6c..f00f374 100644 --- a/backend/app/services/scraper_service.py +++ b/backend/app/services/scraper_service.py @@ -15,7 +15,9 @@ class ScraperService: @staticmethod def scrape_google_maps(search_query: str, max_results: int = 15, headless: bool = True) -> List[Dict[str, Any]]: """ - Executa o Playwright em modo headless para buscar no Google Maps com extração 100% real de estabelecimentos. + Executa o Playwright em modo headless para buscar no Google Maps. + Clica em cada estabelecimento para abrir o painel lateral de detalhes + e extrair dados completos: Telefone, Endereço completo, Website e Rating. """ results = [] try: @@ -42,18 +44,16 @@ class ScraperService: locale='pt-BR' ) page = context.new_page() - - # Prevenir detecção do webdriver page.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") encoded_query = search_query.replace(' ', '+') url = f"https://www.google.com/maps/search/{encoded_query}" - logger.info(f"Prospecção no Maps: {url}") + logger.info(f"Iniciando varredura detalhada no Maps: {url}") page.goto(url, wait_until='domcontentloaded', timeout=40000) time.sleep(3) - # Fechar dialogs de consentimento de cookies do Google caso apareçam + # Aceitar cookies se houver modal try: accept_btn = page.query_selector("button[aria-label*='Aceitar'], button[aria-label*='Accept']") if accept_btn: @@ -62,89 +62,175 @@ class ScraperService: except Exception: pass - # Aguardar feed de resultados ou container principal de busca feed_selector = "div[role='feed']" + has_feed = True try: - page.wait_for_selector(feed_selector, timeout=12000) + page.wait_for_selector(feed_selector, timeout=8000) except Exception: - # Se for redirecionado direto para 1 único resultado - logger.info("Verificando se a busca caiu diretamente no painel de 1 lugar.") + has_feed = False + logger.info("Feed de múltiplos resultados não localizado. Verificando painel de resultado único.") - # Scroll progressivo no feed de resultados - for scroll_step in range(min(8, (max_results // 2) + 2)): - page.evaluate(f""" - const feed = document.querySelector("{feed_selector}"); - if (feed) {{ - feed.scrollBy(0, 1200); - }} - """) - time.sleep(1.2) + # Se houver feed de múltiplos resultados, fazer scroll progressivo + if has_feed: + for _ in range(min(6, (max_results // 3) + 2)): + page.evaluate(f""" + const feed = document.querySelector("{feed_selector}"); + if (feed) {{ + feed.scrollBy(0, 1000); + }} + """) + time.sleep(1.0) - # Extrair os links dos lugares listados - card_elements = page.query_selector_all("a[href*='/maps/place/']") - seen_urls = set() + card_elements = page.query_selector_all("a[href*='/maps/place/']") + seen_urls = set() - for card in card_elements: - if len(results) >= max_results: - break + for card in card_elements: + if len(results) >= max_results: + break - try: - href = card.get_attribute("href") - if not href or href in seen_urls: - continue - seen_urls.add(href) + try: + href = card.get_attribute("href") + if not href or href in seen_urls: + continue + seen_urls.add(href) - # Extrair o nome da empresa via aria-label ou texto interno - aria_label = card.get_attribute("aria-label") or "" - card_text = card.inner_text() or "" - lines = [l.strip() for l in card_text.split('\n') if l.strip()] - - nome_empresa = aria_label.strip() if aria_label else (lines[0] if lines else "") - if not nome_empresa: - continue - - # Extrair avaliação e total de reviews - rating = 0.0 - total_avaliacoes = 0 - - # Procura regex por padroes como "4,8 (150)" ou "4.8" - rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', card_text) - if rating_match: + # Clicar no card para abrir o painel lateral de detalhes no Maps try: - rating = float(rating_match.group(1).replace(',', '.')) - if rating_match.group(2): - total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2))) - except ValueError: + card.click() + time.sleep(1.2) + except Exception: pass - # Extrair número de telefone do card text se presente - phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', card_text) - telefone_raw = phone_match.group(1) if phone_match else "" + detail_panel = page.query_selector("div[role='main']") or page.query_selector("div.m6QEbd") + detail_text = detail_panel.inner_text() if detail_panel else card.inner_text() - # Extrair trecho de endereço ou bairro das linhas - endereco = "" - for line in lines: - if any(term in line.lower() for term in ['r.', 'rua', 'av.', 'avenida', 'alameda', 'bairro', 'praça', 'nº', 'dr.', 'doutor', 'centro']): - if line != nome_empresa: - endereco = line - break + # 1. Nome da Empresa (h1 do painel ou aria-label do card) + h1_elem = page.query_selector("h1") + nome_empresa = "" + if h1_elem: + nome_empresa = h1_elem.inner_text().strip() + if not nome_empresa: + nome_empresa = (card.get_attribute("aria-label") or "").strip() + if not nome_empresa: + lines = [l.strip() for l in card.inner_text().split('\n') if l.strip()] + nome_empresa = lines[0] if lines else "" - results.append({ - 'nome_empresa': nome_empresa, - 'telefone': telefone_raw, - 'endereco': endereco, - 'google_rating': rating, - 'total_avaliacoes': total_avaliacoes, - 'google_maps_url': href, - 'website': '' - }) - except Exception as item_err: - logger.error(f"Erro ao processar item do Maps: {item_err}") - continue + if not nome_empresa: + continue + + # 2. Telefone detalhado + telefone_raw = "" + phone_btn = page.query_selector("button[data-tooltip*='telefone'], button[data-item-id*='phone'], [aria-label*='Telefone'], [aria-label*='Ligar']") + if phone_btn: + phone_attr = phone_btn.get_attribute("aria-label") or phone_btn.inner_text() + phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', phone_attr) + if phone_match: + telefone_raw = phone_match.group(1) + + if not telefone_raw: + phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', detail_text) + if phone_match: + telefone_raw = phone_match.group(1) + + # 3. Endereço Completo + endereco = "" + addr_btn = page.query_selector("button[data-tooltip*='endereço'], button[data-item-id*='address'], [aria-label*='Endereço']") + if addr_btn: + addr_text = addr_btn.get_attribute("aria-label") or addr_btn.inner_text() + addr_clean = re.sub(r'^[Ee]ndereço:\s*', '', addr_text).strip() + if addr_clean: + endereco = addr_clean + + if not endereco: + for line in detail_text.split('\n'): + line_str = line.strip() + if any(term in line_str.lower() for term in ['r.', 'rua', 'av.', 'avenida', 'alameda', 'bairro', 'praça', 'nº', 'dr.', 'centro', 'sp', 'rj', 'mg']): + if line_str != nome_empresa and not re.search(r'^\d\.\d', line_str): + endereco = line_str + break + + # 4. Website + website = "" + site_btn = page.query_selector("a[data-tooltip*='site'], a[data-item-id*='authority']") + if site_btn: + website = site_btn.get_attribute("href") or "" + + # 5. Rating e Avaliações + rating = 0.0 + total_avaliacoes = 0 + rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', detail_text) + if rating_match: + try: + rating = float(rating_match.group(1).replace(',', '.')) + if rating_match.group(2): + total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2))) + except ValueError: + pass + + results.append({ + 'nome_empresa': nome_empresa, + 'telefone': telefone_raw, + 'endereco': endereco, + 'google_rating': rating, + 'total_avaliacoes': total_avaliacoes, + 'google_maps_url': href, + 'website': website + }) + except Exception as item_err: + logger.error(f"Erro ao detalhar lugar do Maps: {item_err}") + continue + else: + # Se for resultado único direto + try: + h1_elem = page.query_selector("h1") + if h1_elem: + nome_empresa = h1_elem.inner_text().strip() + detail_panel = page.query_selector("div[role='main']") or page.query_selector("div.m6QEbd") + detail_text = detail_panel.inner_text() if detail_panel else "" + + telefone_raw = "" + phone_btn = page.query_selector("button[data-tooltip*='telefone'], button[data-item-id*='phone'], [aria-label*='Telefone']") + if phone_btn: + phone_match = re.search(r'(\(?\d{2}\)?\s*9?\d{4}[-.\s]?\d{4})', phone_btn.get_attribute("aria-label") or phone_btn.inner_text()) + if phone_match: + telefone_raw = phone_match.group(1) + + endereco = "" + addr_btn = page.query_selector("button[data-tooltip*='endereço'], button[data-item-id*='address']") + if addr_btn: + endereco = re.sub(r'^[Ee]ndereço:\s*', '', addr_btn.get_attribute("aria-label") or addr_btn.inner_text()).strip() + + website = "" + site_btn = page.query_selector("a[data-tooltip*='site'], a[data-item-id*='authority']") + if site_btn: + website = site_btn.get_attribute("href") or "" + + rating = 0.0 + total_avaliacoes = 0 + rating_match = re.search(r'([1-5][.,]\d)\s*(?:\(([\d.,]+)\))?', detail_text) + if rating_match: + try: + rating = float(rating_match.group(1).replace(',', '.')) + if rating_match.group(2): + total_avaliacoes = int(re.sub(r'\D', '', rating_match.group(2))) + except ValueError: + pass + + results.append({ + 'nome_empresa': nome_empresa, + 'telefone': telefone_raw, + 'endereco': endereco, + 'google_rating': rating, + 'total_avaliacoes': total_avaliacoes, + 'google_maps_url': page.url, + 'website': website + }) + except Exception as single_err: + logger.error(f"Erro ao processar resultado único no Maps: {single_err}") browser.close() except Exception as err: - logger.error(f"Erro na execução do Playwright Scraper real: {err}") + logger.error(f"Erro na execução do Playwright Scraper detalhado: {err}") return results @@ -197,6 +283,8 @@ class ScraperService: existing_lead.total_avaliacoes = item['total_avaliacoes'] if item.get('google_maps_url'): existing_lead.google_maps_url = item['google_maps_url'] + if item.get('website') and not existing_lead.website: + existing_lead.website = item['website'] existing_lead.atualizado_em = db.func.now()