365 lines
12 KiB
Python
365 lines
12 KiB
Python
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from typing import Any, List, Dict, Union, Optional, NamedTuple
|
||
import time
|
||
|
||
import requests
|
||
import structlog
|
||
from sqlalchemy import desc
|
||
|
||
from core.db import get_db_session
|
||
from models import LotteryDLT
|
||
|
||
logger = structlog.getLogger(__name__)
|
||
|
||
|
||
def parse_draw_numbers(result_str: str) -> List[str]:
|
||
"""解析开奖号码，支持空格或逗号分隔"""
|
||
if not result_str or not isinstance(result_str, str):
|
||
return []
|
||
|
||
# 先尝试用空格分割
|
||
if ' ' in result_str:
|
||
numbers = result_str.strip().split()
|
||
elif ',' in result_str:
|
||
numbers = [num.strip() for num in result_str.split(',')]
|
||
else:
|
||
# 如果没有分隔符，尝试按每2个字符分割（如 "0208172933"）
|
||
numbers = [result_str[i:i + 2] for i in range(0, len(result_str), 2)]
|
||
|
||
# 过滤掉空字符串并确保是2位数字
|
||
numbers = [num.zfill(2) for num in numbers if num]
|
||
return numbers
|
||
|
||
|
||
def get_safe_idx(arr: List[str], idx: int, default: str = "0") -> str:
|
||
"""安全获取列表元素"""
|
||
try:
|
||
if arr and idx < len(arr):
|
||
return arr[idx].zfill(2)
|
||
return default.zfill(2)
|
||
except (IndexError, TypeError):
|
||
return default.zfill(2)
|
||
|
||
|
||
def normalize_unsort_result(un_result: Any, sort_result_str: str, issue: str) -> str:
|
||
"""
|
||
标准化 lotteryUnsortDrawresult 字段
|
||
|
||
Args:
|
||
un_result: 原始 unsort 值
|
||
sort_result_str: 排序后的结果
|
||
issue: 期号（用于日志）
|
||
|
||
Returns:
|
||
标准化后的字符串
|
||
"""
|
||
# 处理各种情况
|
||
if isinstance(un_result, str):
|
||
if un_result == "current" or un_result == "":
|
||
logger.debug(f"期号 {issue}: unsort 为 '{un_result}'，使用 lotteryDrawResult")
|
||
return sort_result_str
|
||
return un_result
|
||
|
||
if isinstance(un_result, dict):
|
||
# 字典类型（包括空字典）
|
||
logger.debug(f"期号 {issue}: lotteryUnsortDrawresult 是字典，使用 lotteryDrawResult")
|
||
return sort_result_str
|
||
|
||
if un_result is None:
|
||
return sort_result_str
|
||
|
||
# 其他类型，尝试转换为字符串
|
||
try:
|
||
return str(un_result)
|
||
except:
|
||
logger.warning(f"期号 {issue}: 无法转换 unsort 结果，使用 lotteryDrawResult")
|
||
return sort_result_str
|
||
|
||
|
||
def result_handle(data: dict) -> Optional[LotteryDLT]:
|
||
"""将爬取的数据格式化为当前服务的数据格式"""
|
||
try:
|
||
# 获取基本字段
|
||
issue = data.get('lotteryDrawNum')
|
||
if not issue:
|
||
logger.warning("跳过无效数据: 缺少 lotteryDrawNum")
|
||
return None
|
||
|
||
# 获取排序结果（必须存在）
|
||
sort_result_str = data.get('lotteryDrawResult', '')
|
||
if not sort_result_str or not isinstance(sort_result_str, str):
|
||
logger.warning(f"跳过期号 {issue}: lotteryDrawResult 无效")
|
||
return None
|
||
|
||
# 标准化 unsort 结果
|
||
un_result = normalize_unsort_result(
|
||
data.get('lotteryUnsortDrawresult'),
|
||
sort_result_str,
|
||
str(issue)
|
||
)
|
||
|
||
# 解析号码
|
||
un_sort_result = parse_draw_numbers(un_result)
|
||
sort_result = parse_draw_numbers(sort_result_str)
|
||
|
||
# 验证解析结果
|
||
if len(sort_result) < 7:
|
||
logger.warning(f"期号 {issue}: 排序号码解析失败，原始数据: {sort_result_str}")
|
||
return None
|
||
|
||
if len(un_sort_result) < 7:
|
||
logger.debug(f"期号 {issue}: 未排序号码解析失败，使用排序结果")
|
||
un_sort_result = sort_result
|
||
|
||
# 解析时间
|
||
dt_str = data.get('lotteryDrawTime')
|
||
if not dt_str or not isinstance(dt_str, str):
|
||
logger.warning(f"期号 {issue}: 无效的时间格式")
|
||
return None
|
||
|
||
try:
|
||
dt = datetime.fromisoformat(dt_str.replace(' ', 'T'))
|
||
except ValueError:
|
||
# 尝试其他格式
|
||
dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
||
|
||
record = LotteryDLT(
|
||
issue=str(issue),
|
||
game_name=str(data.get('lotteryGameName', '超级大乐透')),
|
||
draw_date=dt.date(),
|
||
draw_time=dt,
|
||
front_sort_1=get_safe_idx(sort_result, 0),
|
||
front_sort_2=get_safe_idx(sort_result, 1),
|
||
front_sort_3=get_safe_idx(sort_result, 2),
|
||
front_sort_4=get_safe_idx(sort_result, 3),
|
||
front_sort_5=get_safe_idx(sort_result, 4),
|
||
back_sort_1=get_safe_idx(sort_result, 5),
|
||
back_sort_2=get_safe_idx(sort_result, 6),
|
||
front_1=get_safe_idx(un_sort_result, 0),
|
||
front_2=get_safe_idx(un_sort_result, 1),
|
||
front_3=get_safe_idx(un_sort_result, 2),
|
||
front_4=get_safe_idx(un_sort_result, 3),
|
||
front_5=get_safe_idx(un_sort_result, 4),
|
||
back_1=get_safe_idx(un_sort_result, 5),
|
||
back_2=get_safe_idx(un_sort_result, 6)
|
||
)
|
||
return record
|
||
|
||
except Exception as e:
|
||
logger.error(f"处理期号 {data.get('lotteryDrawNum', '未知')} 失败: {e}")
|
||
return None
|
||
|
||
@dataclass
|
||
class ResultCount:
|
||
"""成功失败数据类"""
|
||
success_count: int
|
||
fail_count: int
|
||
|
||
class DLTSpider:
|
||
"""大乐透爬虫"""
|
||
|
||
def __init__(self):
|
||
self.headers = {
|
||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
|
||
"Referer": "https://www.sporttery.cn/"
|
||
}
|
||
self.session = requests.Session()
|
||
self.session.headers.update(self.headers)
|
||
|
||
def _fetch_data(self, url: str) -> Optional[dict]:
|
||
"""通用的数据获取方法"""
|
||
try:
|
||
response = self.session.get(url, timeout=15)
|
||
response.raise_for_status()
|
||
return response.json()
|
||
except requests.exceptions.RequestException as e:
|
||
logger.error(f"请求失败: {url}, 错误: {e}")
|
||
return None
|
||
|
||
def _save_record(self, db, record: LotteryDLT) -> bool:
|
||
"""保存单条记录到数据库"""
|
||
try:
|
||
existing = db.query(LotteryDLT).filter(
|
||
LotteryDLT.issue == record.issue
|
||
).first()
|
||
|
||
if existing:
|
||
logger.debug(f"期号 {record.issue} 已存在，跳过")
|
||
return False
|
||
|
||
db.add(record)
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"保存期号 {record.issue} 失败: {e}")
|
||
return False
|
||
|
||
def fetch_latest(self):
|
||
"""获取最新一期大乐透数据"""
|
||
url = "https://webapi.sporttery.cn/gateway/lottery/getDigitalDrawInfoV1.qry?param=85,0&isVerify=1"
|
||
|
||
data = self._fetch_data(url)
|
||
if not data:
|
||
logger.error("获取最新数据失败")
|
||
return
|
||
|
||
dlt_data = dict(data.get("value", {}).get("dlt", {}))
|
||
if not dlt_data:
|
||
logger.error("数据格式错误")
|
||
return
|
||
|
||
record = result_handle(dlt_data)
|
||
if not record:
|
||
logger.error("数据转换失败")
|
||
return
|
||
|
||
with get_db_session() as db:
|
||
if self._save_record(db, record):
|
||
db.commit()
|
||
logger.info(f"成功保存最新数据: 期号 {record.issue}")
|
||
else:
|
||
logger.info(f"期号 {record.issue} 已存在")
|
||
|
||
def fetch_by_year(self, year: int) -> tuple[List[LotteryDLT], ResultCount]:
|
||
"""获取指定年份的所有数据"""
|
||
logger.info(f"开始爬取 {year} 年的数据")
|
||
|
||
year_prefix = str(year)[-2:]
|
||
all_records = []
|
||
success_count = 0
|
||
fail_count = 0
|
||
|
||
# 分两段获取：001-100 和 101-200
|
||
segments = [
|
||
(f"{year_prefix}001", f"{year_prefix}100"),
|
||
(f"{year_prefix}101", f"{year_prefix}200")
|
||
]
|
||
|
||
for start_term, end_term in segments:
|
||
url = (
|
||
f"https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry"
|
||
f"?gameNo=85&provinceId=0&isVerify=1"
|
||
f"&pageNo=1&pageSize=100"
|
||
f"&startTerm={start_term}&endTerm={end_term}"
|
||
)
|
||
|
||
data = self._fetch_data(url)
|
||
if not data:
|
||
logger.warning(f"获取 {start_term}-{end_term} 数据失败")
|
||
continue
|
||
|
||
dlt_list = data.get("value", {}).get("list", [])
|
||
if not dlt_list:
|
||
logger.warning(f"{start_term}-{end_term} 没有数据")
|
||
continue
|
||
|
||
for item_data in dlt_list:
|
||
record = result_handle(item_data)
|
||
if record:
|
||
all_records.append(record)
|
||
success_count += 1
|
||
else:
|
||
fail_count += 1
|
||
|
||
# 避免请求过快
|
||
time.sleep(0.5)
|
||
|
||
# 按期号排序
|
||
all_records.sort(key=lambda x: int(x.issue))
|
||
logger.info(f"{year} 年成功获取 {success_count} 条数据, 失败 {fail_count} 条数据")
|
||
return all_records, ResultCount(success_count, fail_count)
|
||
|
||
def fetch_range(self, start_year: int = 2007, end_year: Optional[int] = None):
|
||
"""
|
||
获取年限范围内历史所有的大乐透数据
|
||
|
||
Args:
|
||
start_year: 开始年份（大乐透从2007年开始）
|
||
end_year: 结束年份，默认为当前年份
|
||
"""
|
||
if end_year is None:
|
||
end_year = datetime.now().year + 1
|
||
|
||
logger.info(f"开始爬取 {start_year} 到 {end_year} 年的数据")
|
||
|
||
total_count = 0
|
||
success_count = 0
|
||
fail_count = 0
|
||
for year in range(start_year, end_year):
|
||
records, state = self.fetch_by_year(year)
|
||
success_count += state.success_count
|
||
fail_count += state.fail_count
|
||
total_count += (success_count + fail_count)
|
||
|
||
# 批量保存到数据库
|
||
if records:
|
||
with get_db_session() as db:
|
||
saved_count = 0
|
||
for record in records:
|
||
if self._save_record(db, record):
|
||
saved_count += 1
|
||
|
||
if saved_count > 0:
|
||
logger.info(f"{year} 年成功保存 {saved_count} 条新数据")
|
||
else:
|
||
db.rollback()
|
||
|
||
# 避免请求过快
|
||
time.sleep(1)
|
||
|
||
logger.info(f"爬取完成！总共 {total_count} 条数据, 成功: {success_count}, 失败: {fail_count}")
|
||
|
||
def fetch_missing(self, target_year: int = 2026):
|
||
"""
|
||
补全指定年份缺失的数据
|
||
|
||
Args:
|
||
target_year: 要补全的年份
|
||
"""
|
||
logger.info(f"开始补全 {target_year} 年的数据")
|
||
|
||
# 获取数据库中已存在的期号
|
||
with get_db_session() as db:
|
||
existing_issues = [
|
||
r[0] for r in db.query(LotteryDLT.issue)
|
||
.filter(LotteryDLT.issue.startswith(str(target_year)[-2:]))
|
||
.all()
|
||
]
|
||
|
||
# 获取该年份所有数据
|
||
records, state = self.fetch_by_year(target_year)
|
||
|
||
# 过滤出缺失的
|
||
missing_records = [r for r in records if r.issue not in existing_issues]
|
||
success_count = 0
|
||
if missing_records:
|
||
logger.info(f"发现 {len(missing_records)} 条缺失数据")
|
||
with get_db_session() as db:
|
||
for record in missing_records:
|
||
if self._save_record(db, record):
|
||
success_count += 1
|
||
db.commit()
|
||
logger.info(f"成功补全 {success_count} 条数据")
|
||
else:
|
||
logger.info(f"{target_year} 年数据完整，无需补全")
|
||
|
||
def get_latest_issue(self) -> Optional[str]:
|
||
"""获取数据库中最新的期号"""
|
||
with get_db_session() as db:
|
||
record = db.query(LotteryDLT).order_by(
|
||
desc(LotteryDLT.issue)
|
||
).first()
|
||
return record.issue if record else None
|
||
|
||
|
||
if __name__ == "__main__":
|
||
spider = DLTSpider()
|
||
|
||
# 爬取所有历史数据（从2007年开始）
|
||
# spider.fetch_range(start_year=2007)
|
||
|
||
# 或者只补全2026年的数据
|
||
spider.fetch_missing(2017)
|
||
|
||
# 或者只获取最新一期
|
||
# spider.fetch_latest() |