Add maxlaw PC spider and shared proxy limiter
This commit is contained in:
@@ -16,6 +16,7 @@ if project_root not in sys.path:
|
||||
import requests
|
||||
from request.proxy_config import get_proxies, report_proxy_status
|
||||
from Db import Db
|
||||
from utils.rate_limiter import request_slot
|
||||
|
||||
DOMAIN = "找法网"
|
||||
LIST_TEMPLATE = "https://m.findlaw.cn/{pinyin}/q_lawyer/p{page}?ajax=1&order=0&sex=-1"
|
||||
@@ -59,7 +60,8 @@ class FindlawSpider:
|
||||
headers = {"Referer": referer}
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
resp = self.session.get(url, timeout=15, verify=verify, headers=headers)
|
||||
with request_slot():
|
||||
resp = self.session.get(url, timeout=15, verify=verify, headers=headers)
|
||||
status_code = resp.status_code
|
||||
text = resp.text
|
||||
resp.close()
|
||||
|
||||
Reference in New Issue
Block a user