feat: add baidu lvlin crawler
This commit is contained in:
hello-dd-code
@@ -0,0 +1,473 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
from urllib.parse import urlencode
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_root = os.path.dirname(current_dir)
|
||||
request_dir = os.path.join(project_root, "request")
|
||||
if request_dir not in sys.path:
|
||||
sys.path.insert(0, request_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.append(project_root)
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from Db import Db
|
||||
from request.proxy_config import get_proxies, report_proxy_status
|
||||
|
||||
|
||||
DOMAIN = "百度法行宝"
|
||||
BASE_URL = "https://lvlin.baidu.com"
|
||||
CITY_API = f"{BASE_URL}/pc/api/law/sync/city"
|
||||
LIST_API = f"{BASE_URL}/pc/api/law/api/lawyerlist"
|
||||
DETAIL_API = f"{BASE_URL}/pc/api/law/api/lawyerhome"
|
||||
DEFAULT_PAGE_SIZE = 16
|
||||
DEFAULT_MAX_PAGES = 30
|
||||
DEFAULT_STOP_ZERO_NEW_PAGES = 3
|
||||
DEFAULT_SLEEP_SECONDS = 0.1
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="采集百度法行宝律师信息并落库")
|
||||
parser.add_argument("--province", default="", help="仅采集指定省份，例如：山东")
|
||||
parser.add_argument("--city", default="", help="仅采集指定城市，例如：聊城 / 聊城市")
|
||||
parser.add_argument(
|
||||
"--areas",
|
||||
default="",
|
||||
help="指定案件类型，逗号分隔；不传时自动发现顶级类型并追加不限",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit-cities",
|
||||
type=int,
|
||||
default=0,
|
||||
help="仅处理前 N 个城市，0 表示不限",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-size",
|
||||
type=int,
|
||||
default=DEFAULT_PAGE_SIZE,
|
||||
help=f"每次列表请求条数，默认 {DEFAULT_PAGE_SIZE}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-pages-per-query",
|
||||
type=int,
|
||||
default=DEFAULT_MAX_PAGES,
|
||||
help=f"单城市单类型最大翻页数，默认 {DEFAULT_MAX_PAGES}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stop-zero-new-pages",
|
||||
type=int,
|
||||
default=DEFAULT_STOP_ZERO_NEW_PAGES,
|
||||
help=f"连续多少页无新增就停止当前查询，默认 {DEFAULT_STOP_ZERO_NEW_PAGES}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sleep-seconds",
|
||||
type=float,
|
||||
default=DEFAULT_SLEEP_SECONDS,
|
||||
help=f"请求间隔秒数，默认 {DEFAULT_SLEEP_SECONDS}",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
class BaiduLvlinSpider:
|
||||
def __init__(self, db_connection: Db, args: argparse.Namespace):
|
||||
self.db = db_connection
|
||||
self.args = args
|
||||
self.page_size = max(1, int(args.page_size or DEFAULT_PAGE_SIZE))
|
||||
self.max_pages_per_query = max(1, int(args.max_pages_per_query or DEFAULT_MAX_PAGES))
|
||||
self.stop_zero_new_pages = max(1, int(args.stop_zero_new_pages or DEFAULT_STOP_ZERO_NEW_PAGES))
|
||||
self.sleep_seconds = max(0.0, float(args.sleep_seconds or 0.0))
|
||||
self.proxy_enabled = False
|
||||
self.session = self._build_session()
|
||||
self.existing_urls = self._load_existing_urls()
|
||||
self.cities = self._load_cities()
|
||||
self.areas = self._load_areas()
|
||||
self.inserted_count = 0
|
||||
|
||||
def _build_session(self) -> requests.Session:
|
||||
report_proxy_status()
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
proxies = get_proxies()
|
||||
if proxies:
|
||||
session.proxies.update(proxies)
|
||||
self.proxy_enabled = True
|
||||
else:
|
||||
session.proxies.clear()
|
||||
self.proxy_enabled = False
|
||||
|
||||
retries = Retry(
|
||||
total=3,
|
||||
backoff_factor=1,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset(["GET"]),
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retries)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/123.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Referer": f"{BASE_URL}/pc/r?vn=law",
|
||||
"Connection": "close",
|
||||
}
|
||||
)
|
||||
return session
|
||||
|
||||
def _disable_proxy(self) -> None:
|
||||
if not self.proxy_enabled:
|
||||
return
|
||||
self.session.proxies.clear()
|
||||
self.proxy_enabled = False
|
||||
print(f"[{DOMAIN}] 代理不可用，已切换直连")
|
||||
|
||||
def _sleep(self) -> None:
|
||||
if self.sleep_seconds > 0:
|
||||
time.sleep(self.sleep_seconds)
|
||||
|
||||
def _get_json(self, url: str, params: Optional[Dict[str, object]] = None, referer: str = "") -> Dict:
|
||||
headers = {}
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
try:
|
||||
resp = self.session.get(url, params=params or {}, timeout=20, headers=headers)
|
||||
except requests.exceptions.ProxyError:
|
||||
self._disable_proxy()
|
||||
resp = self.session.get(url, params=params or {}, timeout=20, headers=headers)
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
finally:
|
||||
resp.close()
|
||||
|
||||
def _load_existing_urls(self) -> Set[str]:
|
||||
urls: Set[str] = set()
|
||||
cursor = self.db.db.cursor()
|
||||
try:
|
||||
cursor.execute("SELECT url FROM lawyer WHERE domain=%s AND url IS NOT NULL", (DOMAIN,))
|
||||
for row in cursor.fetchall():
|
||||
url = (row[0] or "").strip()
|
||||
if url:
|
||||
urls.add(url)
|
||||
finally:
|
||||
cursor.close()
|
||||
print(f"[{DOMAIN}] 已存在 URL 数: {len(urls)}")
|
||||
return urls
|
||||
|
||||
def _normalize_city_name(self, city_name: str) -> str:
|
||||
text = str(city_name or "").strip()
|
||||
if text.endswith("市"):
|
||||
return text[:-1]
|
||||
return text
|
||||
|
||||
def _city_matches(self, expected_city: str, actual_city: str) -> bool:
|
||||
left = self._normalize_city_name(expected_city)
|
||||
right = self._normalize_city_name(actual_city)
|
||||
if not left or not right:
|
||||
return False
|
||||
return left == right
|
||||
|
||||
def _load_cities(self) -> List[Dict[str, str]]:
|
||||
payload = self._get_json(CITY_API, params={"vn": "law"}, referer=f"{BASE_URL}/pc/r?vn=law")
|
||||
all_city_list = payload.get("data", {}).get("AllCityList", []) or []
|
||||
cities: List[Dict[str, str]] = []
|
||||
province_filter = self.args.province.strip()
|
||||
city_filter = self._normalize_city_name(self.args.city)
|
||||
|
||||
for block in all_city_list:
|
||||
for item in block.get("cityList", []) or []:
|
||||
city_name = str(item.get("name") or "").strip()
|
||||
province = str(item.get("province") or "").strip()
|
||||
city_code = str(item.get("code") or "").strip()
|
||||
if not city_name or not province or not city_code:
|
||||
continue
|
||||
if province_filter and province != province_filter:
|
||||
continue
|
||||
if city_filter and self._normalize_city_name(city_name) != city_filter:
|
||||
continue
|
||||
cities.append(
|
||||
{
|
||||
"province": province,
|
||||
"city": city_name,
|
||||
"city_code": city_code,
|
||||
}
|
||||
)
|
||||
|
||||
cities.sort(key=lambda item: (item["province"], item["city"]))
|
||||
if self.args.limit_cities and self.args.limit_cities > 0:
|
||||
cities = cities[: self.args.limit_cities]
|
||||
print(f"[{DOMAIN}] 本次待采城市数: {len(cities)}")
|
||||
return cities
|
||||
|
||||
def _discover_top_level_areas(self) -> List[str]:
|
||||
sample_city = self.cities[0]["city"] if self.cities else "北京"
|
||||
payload = self._get_json(
|
||||
LIST_API,
|
||||
params={
|
||||
"city_name": sample_city,
|
||||
"page_num": 1,
|
||||
"page_size": self.page_size,
|
||||
"ts": int(time.time()),
|
||||
"clientType": "pc",
|
||||
"list_type": 1,
|
||||
},
|
||||
referer=f"{BASE_URL}/pc/r?vn=law",
|
||||
)
|
||||
filters = payload.get("data", {}).get("filters", []) or []
|
||||
areas: List[str] = ["不限"]
|
||||
seen = {"不限"}
|
||||
for item in filters:
|
||||
if item.get("key") != "type":
|
||||
continue
|
||||
for option in item.get("options", []) or []:
|
||||
value = str(option.get("value") or "").strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
areas.append(value)
|
||||
return areas
|
||||
|
||||
def _load_areas(self) -> List[str]:
|
||||
if self.args.areas.strip():
|
||||
areas = [part.strip() for part in self.args.areas.split(",") if part.strip()]
|
||||
unique: List[str] = []
|
||||
seen = set()
|
||||
for area in areas:
|
||||
if area not in seen:
|
||||
seen.add(area)
|
||||
unique.append(area)
|
||||
print(f"[{DOMAIN}] 使用指定案件类型: {unique}")
|
||||
return unique
|
||||
|
||||
areas = self._discover_top_level_areas()
|
||||
print(f"[{DOMAIN}] 自动发现案件类型: {areas}")
|
||||
return areas
|
||||
|
||||
def _build_pc_detail_url(self, qc_no: str, rs_id: str) -> str:
|
||||
return f"{BASE_URL}/pc/lawyer?vn=law&qc_no={qc_no}&rs_id={rs_id}"
|
||||
|
||||
def _build_list_page_url(self, city_name: str, area_name: str) -> str:
|
||||
params = {"city": city_name, "vn": "law"}
|
||||
if area_name and area_name != "不限":
|
||||
params["expertiseArea"] = area_name
|
||||
return f"{BASE_URL}/pc/r?{urlencode(params)}"
|
||||
|
||||
def _fetch_list(self, city_name: str, area_name: str, page_num: int) -> List[Dict]:
|
||||
params: Dict[str, object] = {
|
||||
"city_name": city_name,
|
||||
"page_num": page_num,
|
||||
"page_size": self.page_size,
|
||||
"ts": int(time.time()),
|
||||
"clientType": "pc",
|
||||
"list_type": 1,
|
||||
}
|
||||
if area_name and area_name != "不限":
|
||||
params["expertiseArea"] = area_name
|
||||
payload = self._get_json(
|
||||
LIST_API,
|
||||
params=params,
|
||||
referer=self._build_list_page_url(city_name, area_name),
|
||||
)
|
||||
return payload.get("data", {}).get("lawyer_list", []) or []
|
||||
|
||||
def _fetch_detail(self, qc_no: str, rs_id: str) -> Dict:
|
||||
payload = self._get_json(
|
||||
DETAIL_API,
|
||||
params={"vn": "law", "qc_no": qc_no, "rs_id": rs_id},
|
||||
referer=self._build_pc_detail_url(qc_no, rs_id),
|
||||
)
|
||||
return payload.get("data", {}).get("lawyer", {}) or {}
|
||||
|
||||
def _extract_phone(self, detail: Dict) -> Optional[str]:
|
||||
for service in detail.get("lawyer_service", []) or []:
|
||||
phone = str(service.get("phone_num") or "").strip()
|
||||
if phone:
|
||||
return phone
|
||||
for service in detail.get("lawyer_service_new", []) or []:
|
||||
phone = str(service.get("phone_num") or "").strip()
|
||||
if phone:
|
||||
return phone
|
||||
return None
|
||||
|
||||
def _safe_json(self, payload: Dict) -> str:
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
def _build_record(
|
||||
self,
|
||||
city_info: Dict[str, str],
|
||||
area_name: str,
|
||||
page_num: int,
|
||||
list_item: Dict,
|
||||
detail: Dict,
|
||||
) -> Dict[str, object]:
|
||||
qc_no = str(list_item.get("qc_no") or detail.get("qc_no") or "").strip()
|
||||
rs_id = str(list_item.get("rs_id") or detail.get("rs_id") or "").strip()
|
||||
detail_url = self._build_pc_detail_url(qc_no, rs_id)
|
||||
name = str(detail.get("lawyer_name") or list_item.get("lawyer_name") or "").strip()
|
||||
law_firm = str(detail.get("practice_company") or list_item.get("practice_company") or "").strip()
|
||||
city_name = str(list_item.get("city") or city_info.get("city") or "").strip()
|
||||
avatar_url = str(detail.get("lawyer_avatar_big") or detail.get("lawyer_avatar") or list_item.get("lawyer_avatar_big") or list_item.get("lawyer_avatar") or "").strip()
|
||||
phone = self._extract_phone(detail)
|
||||
|
||||
params = {
|
||||
"source": {
|
||||
"site": "baidu_lvlin",
|
||||
"city_name": city_info.get("city"),
|
||||
"city_code": city_info.get("city_code"),
|
||||
"province": city_info.get("province"),
|
||||
"expertise_area": area_name,
|
||||
"page_num": page_num,
|
||||
"list_url": self._build_list_page_url(city_info.get("city", ""), area_name),
|
||||
"detail_url": detail_url,
|
||||
"list_api": LIST_API,
|
||||
"detail_api": DETAIL_API,
|
||||
},
|
||||
"list_item": list_item,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
return {
|
||||
"name": name or None,
|
||||
"phone": phone or None,
|
||||
"law_firm": law_firm or None,
|
||||
"province": city_info.get("province") or None,
|
||||
"city": city_name or city_info.get("city") or None,
|
||||
"url": detail_url,
|
||||
"avatar_url": avatar_url or None,
|
||||
"domain": DOMAIN,
|
||||
"create_time": int(time.time()),
|
||||
"site_time": None,
|
||||
"params": self._safe_json(params),
|
||||
}
|
||||
|
||||
def _insert_record(self, record: Dict[str, object]) -> bool:
|
||||
url = str(record.get("url") or "").strip()
|
||||
if not url or url in self.existing_urls:
|
||||
return False
|
||||
self.db.insert_data("lawyer", record)
|
||||
self.existing_urls.add(url)
|
||||
self.inserted_count += 1
|
||||
return True
|
||||
|
||||
def _iter_city_area(self, city_info: Dict[str, str], area_name: str) -> Tuple[int, int]:
|
||||
inserted = 0
|
||||
pages = 0
|
||||
zero_new_pages = 0
|
||||
city_name = city_info["city"]
|
||||
|
||||
for page_num in range(1, self.max_pages_per_query + 1):
|
||||
pages = page_num
|
||||
try:
|
||||
items = self._fetch_list(city_name, area_name, page_num)
|
||||
except Exception as exc:
|
||||
print(f"[{DOMAIN}] 列表请求失败 {city_name}-{area_name}-p{page_num}: {exc}")
|
||||
break
|
||||
|
||||
if not items:
|
||||
print(f"[{DOMAIN}] {city_name}-{area_name} 第 {page_num} 页无数据，停止")
|
||||
break
|
||||
|
||||
page_new = 0
|
||||
for item in items:
|
||||
qc_no = str(item.get("qc_no") or "").strip()
|
||||
rs_id = str(item.get("rs_id") or "").strip()
|
||||
actual_city = str(item.get("city") or "").strip()
|
||||
if not qc_no or not rs_id:
|
||||
continue
|
||||
if actual_city and not self._city_matches(city_name, actual_city):
|
||||
continue
|
||||
|
||||
detail_url = self._build_pc_detail_url(qc_no, rs_id)
|
||||
if detail_url in self.existing_urls:
|
||||
continue
|
||||
|
||||
detail: Dict = {}
|
||||
try:
|
||||
detail = self._fetch_detail(qc_no, rs_id)
|
||||
except Exception as exc:
|
||||
print(f"[{DOMAIN}] 详情请求失败 {qc_no}-{rs_id}: {exc}")
|
||||
|
||||
record = self._build_record(city_info, area_name, page_num, item, detail)
|
||||
try:
|
||||
if self._insert_record(record):
|
||||
page_new += 1
|
||||
inserted += 1
|
||||
print(
|
||||
f"[{DOMAIN}] -> 新增 {record.get('name') or qc_no} "
|
||||
f"| {city_name} | {area_name} | p{page_num}"
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[{DOMAIN}] 插入失败 {record.get('url')}: {exc}")
|
||||
self._sleep()
|
||||
|
||||
print(
|
||||
f"[{DOMAIN}] {city_name} | {area_name} | p{page_num} "
|
||||
f"| 列表 {len(items)} | 新增 {page_new}"
|
||||
)
|
||||
|
||||
if len(items) < self.page_size:
|
||||
break
|
||||
if page_new == 0:
|
||||
zero_new_pages += 1
|
||||
if zero_new_pages >= self.stop_zero_new_pages:
|
||||
print(
|
||||
f"[{DOMAIN}] {city_name}-{area_name} 连续 {zero_new_pages} 页无新增，停止"
|
||||
)
|
||||
break
|
||||
else:
|
||||
zero_new_pages = 0
|
||||
|
||||
self._sleep()
|
||||
|
||||
return inserted, pages
|
||||
|
||||
def run(self) -> None:
|
||||
print(f"[{DOMAIN}] 启动采集")
|
||||
if not self.cities:
|
||||
print(f"[{DOMAIN}] 无可采城市")
|
||||
return
|
||||
if not self.areas:
|
||||
print(f"[{DOMAIN}] 无可采案件类型")
|
||||
return
|
||||
|
||||
total_queries = len(self.cities) * len(self.areas)
|
||||
query_index = 0
|
||||
for city_info in self.cities:
|
||||
city_inserted = 0
|
||||
for area_name in self.areas:
|
||||
query_index += 1
|
||||
print(
|
||||
f"[{DOMAIN}] 进度 {query_index}/{total_queries} | "
|
||||
f"{city_info['province']}-{city_info['city']} | {area_name}"
|
||||
)
|
||||
inserted, pages = self._iter_city_area(city_info, area_name)
|
||||
city_inserted += inserted
|
||||
print(
|
||||
f"[{DOMAIN}] 完成 {city_info['city']} | {area_name} "
|
||||
f"| 翻页 {pages} | 新增 {inserted}"
|
||||
)
|
||||
print(
|
||||
f"[{DOMAIN}] 城市完成 {city_info['province']}-{city_info['city']} "
|
||||
f"| 本城新增 {city_inserted} | 总新增 {self.inserted_count}"
|
||||
)
|
||||
print(f"[{DOMAIN}] 采集完成，总新增 {self.inserted_count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli_args = parse_args()
|
||||
with Db() as db:
|
||||
spider = BaiduLvlinSpider(db, cli_args)
|
||||
spider.run()
|
||||
Reference in New Issue
Block a user