import logging from dataclasses import dataclass from datetime import datetime from typing import Any, List, Dict, Union, Optional, NamedTuple import time import requests from pydantic import BaseModel, field_validator from sqlalchemy import desc from core.db import get_db_session from models import LotteryDLT logger = logging.getLogger(__name__) def parse_draw_numbers(result_str: str) -> List[str]: """解析开奖号码,支持空格或逗号分隔""" if not result_str or not isinstance(result_str, str): return [] # 先尝试用空格分割 if ' ' in result_str: numbers = result_str.strip().split() elif ',' in result_str: numbers = [num.strip() for num in result_str.split(',')] else: # 如果没有分隔符,尝试按每2个字符分割(如 "0208172933") numbers = [result_str[i:i + 2] for i in range(0, len(result_str), 2)] # 过滤掉空字符串并确保是2位数字 numbers = [num.zfill(2) for num in numbers if num] return numbers def get_safe_idx(arr: List[str], idx: int, default: str = "0") -> str: """安全获取列表元素""" try: if arr and idx < len(arr): return arr[idx].zfill(2) return default.zfill(2) except (IndexError, TypeError): return default.zfill(2) def normalize_unsort_result(un_result: Any, sort_result_str: str, issue: str) -> str: """ 标准化 lotteryUnsortDrawresult 字段 Args: un_result: 原始 unsort 值 sort_result_str: 排序后的结果 issue: 期号(用于日志) Returns: 标准化后的字符串 """ # 处理各种情况 if isinstance(un_result, str): if un_result == "current" or un_result == "": logger.debug(f"期号 {issue}: unsort 为 '{un_result}',使用 lotteryDrawResult") return sort_result_str return un_result if isinstance(un_result, dict): # 字典类型(包括空字典) logger.debug(f"期号 {issue}: lotteryUnsortDrawresult 是字典,使用 lotteryDrawResult") return sort_result_str if un_result is None: return sort_result_str # 其他类型,尝试转换为字符串 try: return str(un_result) except: logger.warning(f"期号 {issue}: 无法转换 unsort 结果,使用 lotteryDrawResult") return sort_result_str def result_handle(data: dict) -> Optional[LotteryDLT]: """将爬取的数据格式化为当前服务的数据格式""" try: # 获取基本字段 issue = data.get('lotteryDrawNum') if not issue: logger.warning("跳过无效数据: 缺少 lotteryDrawNum") return None # 获取排序结果(必须存在) sort_result_str = data.get('lotteryDrawResult', '') if not sort_result_str or not isinstance(sort_result_str, str): logger.warning(f"跳过期号 {issue}: lotteryDrawResult 无效") return None # 标准化 unsort 结果 un_result = normalize_unsort_result( data.get('lotteryUnsortDrawresult'), sort_result_str, str(issue) ) # 解析号码 un_sort_result = parse_draw_numbers(un_result) sort_result = parse_draw_numbers(sort_result_str) # 验证解析结果 if len(sort_result) < 7: logger.warning(f"期号 {issue}: 排序号码解析失败,原始数据: {sort_result_str}") return None if len(un_sort_result) < 7: logger.debug(f"期号 {issue}: 未排序号码解析失败,使用排序结果") un_sort_result = sort_result # 解析时间 dt_str = data.get('lotteryDrawTime') if not dt_str or not isinstance(dt_str, str): logger.warning(f"期号 {issue}: 无效的时间格式") return None try: dt = datetime.fromisoformat(dt_str.replace(' ', 'T')) except ValueError: # 尝试其他格式 dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S") record = LotteryDLT( issue=str(issue), game_name=str(data.get('lotteryGameName', '超级大乐透')), draw_date=dt.date(), draw_time=dt, front_sort_1=get_safe_idx(sort_result, 0), front_sort_2=get_safe_idx(sort_result, 1), front_sort_3=get_safe_idx(sort_result, 2), front_sort_4=get_safe_idx(sort_result, 3), front_sort_5=get_safe_idx(sort_result, 4), back_sort_1=get_safe_idx(sort_result, 5), back_sort_2=get_safe_idx(sort_result, 6), front_1=get_safe_idx(un_sort_result, 0), front_2=get_safe_idx(un_sort_result, 1), front_3=get_safe_idx(un_sort_result, 2), front_4=get_safe_idx(un_sort_result, 3), front_5=get_safe_idx(un_sort_result, 4), back_1=get_safe_idx(un_sort_result, 5), back_2=get_safe_idx(un_sort_result, 6) ) return record except Exception as e: logger.error(f"处理期号 {data.get('lotteryDrawNum', '未知')} 失败: {e}") return None @dataclass class ResultCount: """成功失败数据类""" success_count: int fail_count: int class DLTSpider: """大乐透爬虫""" def __init__(self): self.headers = { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36", "Referer": "https://www.sporttery.cn/" } self.session = requests.Session() self.session.headers.update(self.headers) def _fetch_data(self, url: str) -> Optional[dict]: """通用的数据获取方法""" try: response = self.session.get(url, timeout=15) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: logger.error(f"请求失败: {url}, 错误: {e}") return None def _save_record(self, db, record: LotteryDLT) -> bool: """保存单条记录到数据库""" try: existing = db.query(LotteryDLT).filter( LotteryDLT.issue == record.issue ).first() if existing: logger.debug(f"期号 {record.issue} 已存在,跳过") return False db.add(record) return True except Exception as e: logger.error(f"保存期号 {record.issue} 失败: {e}") return False def fetch_latest(self): """获取最新一期大乐透数据""" url = "https://webapi.sporttery.cn/gateway/lottery/getDigitalDrawInfoV1.qry?param=85,0&isVerify=1" data = self._fetch_data(url) if not data: logger.error("获取最新数据失败") return dlt_data = dict(data.get("value", {}).get("dlt", {})) if not dlt_data: logger.error("数据格式错误") return record = result_handle(dlt_data) if not record: logger.error("数据转换失败") return with get_db_session() as db: if self._save_record(db, record): db.commit() logger.info(f"成功保存最新数据: 期号 {record.issue}") else: logger.info(f"期号 {record.issue} 已存在") def fetch_by_year(self, year: int) -> tuple[List[LotteryDLT], ResultCount]: """获取指定年份的所有数据""" logger.info(f"开始爬取 {year} 年的数据") year_prefix = str(year)[-2:] all_records = [] success_count = 0 fail_count = 0 # 分两段获取:001-100 和 101-200 segments = [ (f"{year_prefix}001", f"{year_prefix}100"), (f"{year_prefix}101", f"{year_prefix}200") ] for start_term, end_term in segments: url = ( f"https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry" f"?gameNo=85&provinceId=0&isVerify=1" f"&pageNo=1&pageSize=100" f"&startTerm={start_term}&endTerm={end_term}" ) data = self._fetch_data(url) if not data: logger.warning(f"获取 {start_term}-{end_term} 数据失败") continue dlt_list = data.get("value", {}).get("list", []) if not dlt_list: logger.warning(f"{start_term}-{end_term} 没有数据") continue for item_data in dlt_list: record = result_handle(item_data) if record: all_records.append(record) success_count += 1 else: fail_count += 1 # 避免请求过快 time.sleep(0.5) # 按期号排序 all_records.sort(key=lambda x: int(x.issue)) logger.info(f"{year} 年成功获取 {success_count} 条数据, 失败 {fail_count} 条数据") return all_records, ResultCount(success_count, fail_count) def fetch_range(self, start_year: int = 2007, end_year: Optional[int] = None): """ 获取年限范围内历史所有的大乐透数据 Args: start_year: 开始年份(大乐透从2007年开始) end_year: 结束年份,默认为当前年份 """ if end_year is None: end_year = datetime.now().year + 1 logger.info(f"开始爬取 {start_year} 到 {end_year} 年的数据") total_count = 0 success_count = 0 fail_count = 0 for year in range(start_year, end_year): records, state = self.fetch_by_year(year) success_count += state.success_count fail_count += state.fail_count total_count += (success_count + fail_count) # 批量保存到数据库 if records: with get_db_session() as db: saved_count = 0 for record in records: if self._save_record(db, record): saved_count += 1 if saved_count > 0: logger.info(f"{year} 年成功保存 {saved_count} 条新数据") else: db.rollback() # 避免请求过快 time.sleep(1) logger.info(f"爬取完成!总共 {total_count} 条数据, 成功: {success_count}, 失败: {fail_count}") def fetch_missing(self, target_year: int = 2026): """ 补全指定年份缺失的数据 Args: target_year: 要补全的年份 """ logger.info(f"开始补全 {target_year} 年的数据") # 获取数据库中已存在的期号 with get_db_session() as db: existing_issues = [ r[0] for r in db.query(LotteryDLT.issue) .filter(LotteryDLT.issue.startswith(str(target_year)[-2:])) .all() ] # 获取该年份所有数据 records, state = self.fetch_by_year(target_year) # 过滤出缺失的 missing_records = [r for r in records if r.issue not in existing_issues] success_count = 0 if missing_records: logger.info(f"发现 {len(missing_records)} 条缺失数据") with get_db_session() as db: for record in missing_records: if self._save_record(db, record): success_count += 1 db.commit() logger.info(f"成功补全 {success_count} 条数据") else: logger.info(f"{target_year} 年数据完整,无需补全") def get_latest_issue(self) -> Optional[str]: """获取数据库中最新的期号""" with get_db_session() as db: record = db.query(LotteryDLT).order_by( desc(LotteryDLT.issue) ).first() return record.issue if record else None if __name__ == "__main__": spider = DLTSpider() # 爬取所有历史数据(从2007年开始) # spider.fetch_range(start_year=2007) # 或者只补全2026年的数据 spider.fetch_missing(2017) # 或者只获取最新一期 # spider.fetch_latest()