初始化
This commit is contained in:
0
core/__init__.py
Normal file
0
core/__init__.py
Normal file
0
core/config/__init__.py
Normal file
0
core/config/__init__.py
Normal file
179
core/config/settings.py
Normal file
179
core/config/settings.py
Normal file
@@ -0,0 +1,179 @@
|
||||
from typing import List
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class Environment(str, Enum):
|
||||
"""环境枚举"""
|
||||
LOCAL = "local"
|
||||
DEVELOPMENT = "dev"
|
||||
STAGING = "staging"
|
||||
PRODUCTION = "prod"
|
||||
|
||||
|
||||
class LogLevel(str, Enum):
|
||||
"""日志级别枚举"""
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""
|
||||
应用配置类
|
||||
自动从 .env 文件和环境变量中读取配置
|
||||
"""
|
||||
|
||||
# ---------- 基础配置 ----------
|
||||
APP_NAME: str = Field(
|
||||
default="Lottery Server",
|
||||
description="应用名称"
|
||||
)
|
||||
|
||||
APP_DESCRIPTION: str = Field(
|
||||
default="一个自己看的彩票信息服务",
|
||||
description="应用描述"
|
||||
)
|
||||
|
||||
ENVIRONMENT: Environment = Field(
|
||||
default=Environment.LOCAL,
|
||||
description="运行环境"
|
||||
)
|
||||
|
||||
# ---------- 服务器配置 ----------
|
||||
HOST: str = Field(
|
||||
default="0.0.0.0",
|
||||
description="服务监听地址"
|
||||
)
|
||||
PORT: int = Field(
|
||||
default=8000,
|
||||
ge=1,
|
||||
le=65535,
|
||||
description="服务监听端口"
|
||||
)
|
||||
|
||||
# ---------- 数据库MYSQL配置 ----------
|
||||
MYSQL_USER: str = Field(
|
||||
default='root',
|
||||
description="MySQL用户名"
|
||||
)
|
||||
MYSQL_PASSWORD: str = Field(
|
||||
default='Azure1314',
|
||||
description="MySQL密码"
|
||||
)
|
||||
MYSQL_HOST: str = Field(
|
||||
default='127.0.0.1',
|
||||
description="MySQL主机地址"
|
||||
)
|
||||
MYSQL_PORT: int = Field(
|
||||
default=3306,
|
||||
ge=1,
|
||||
le=65535,
|
||||
description="MySQL端口"
|
||||
)
|
||||
MYSQL_DB: str = Field(
|
||||
default='lottery_server',
|
||||
description="MySQL数据库名"
|
||||
)
|
||||
|
||||
DATABASE_POOL_SIZE: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
description="数据库连接池大小"
|
||||
)
|
||||
DATABASE_MAX_OVERFLOW: int = Field(
|
||||
default=20,
|
||||
description="连接池最大溢出连接数"
|
||||
)
|
||||
DATABASE_ECHO: bool = Field(
|
||||
default=False,
|
||||
description="是否打印SQL语句"
|
||||
)
|
||||
|
||||
# ---------- CORS配置 ----------
|
||||
ALLOWED_ORIGINS: List[str] = Field(
|
||||
default=["*"],
|
||||
description="允许的CORS源"
|
||||
)
|
||||
ALLOWED_METHODS: List[str] = Field(
|
||||
default=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
description="允许的HTTP方法"
|
||||
)
|
||||
ALLOWED_HEADERS: List[str] = Field(
|
||||
default=["*"],
|
||||
description="允许的HTTP头"
|
||||
)
|
||||
|
||||
# ---------- 限流配置 ----------
|
||||
RATE_LIMIT_PER_MINUTE: int = Field(
|
||||
default=60,
|
||||
ge=1,
|
||||
description="每分钟请求限制"
|
||||
)
|
||||
RATE_LIMIT_PER_DAY: int = Field(
|
||||
default=10000,
|
||||
ge=1,
|
||||
description="每天请求限制"
|
||||
)
|
||||
|
||||
# ---------- 文件上传配置 ----------
|
||||
MAX_UPLOAD_SIZE_MB: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
description="最大上传文件大小(MB)"
|
||||
)
|
||||
ALLOWED_UPLOAD_EXTENSIONS: List[str] = Field(
|
||||
default=[".jpg", ".jpeg", ".png", ".pdf", ".docx"],
|
||||
description="允许上传的文件扩展名"
|
||||
)
|
||||
UPLOAD_DIR: str = Field(
|
||||
default="./uploads",
|
||||
description="文件上传目录"
|
||||
)
|
||||
|
||||
# ---------- 缓存配置 ----------
|
||||
CACHE_TTL_SECONDS: int = Field(
|
||||
default=300,
|
||||
ge=1,
|
||||
description="缓存过期时间(秒)"
|
||||
)
|
||||
|
||||
# ---------- 计算属性 ----------
|
||||
@property
|
||||
def mysql_url(self) -> str:
|
||||
"""MySQL连接URL"""
|
||||
# 连接url格式:mysql+pymysql://user:password@host:port/dbname
|
||||
return f"mysql+pymysql://{self.MYSQL_USER}:{self.MYSQL_PASSWORD}@{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DB}?charset=utf8mb4"
|
||||
|
||||
@property
|
||||
def is_production(self) -> bool:
|
||||
"""是否为生产环境"""
|
||||
return self.ENVIRONMENT == Environment.PRODUCTION
|
||||
|
||||
@property
|
||||
def is_development(self) -> bool:
|
||||
"""是否为开发环境"""
|
||||
return self.ENVIRONMENT in [Environment.LOCAL, Environment.DEVELOPMENT]
|
||||
|
||||
# ---------- 配置加载设置 ----------
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env.local", # .env文件路径
|
||||
env_file_encoding="utf-8", # 文件编码
|
||||
case_sensitive=True, # 环境变量名大小写敏感
|
||||
extra="ignore", # 忽略未定义的额外字段
|
||||
validate_default=True, # 验证默认值
|
||||
# secrets_dir="/run/secrets", # Docker secrets支持(可选)
|
||||
)
|
||||
|
||||
|
||||
# 创建全局单例配置实例
|
||||
settings = Settings()
|
||||
|
||||
|
||||
# 导出配置实例
|
||||
__all__ = ["settings", "Environment", "LogLevel"]
|
||||
62
core/db.py
Normal file
62
core/db.py
Normal file
@@ -0,0 +1,62 @@
|
||||
from fastapi import Depends
|
||||
from typing import Annotated
|
||||
from sqlmodel import Session, create_engine
|
||||
from core.config.settings import settings
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
# 1.引擎配置
|
||||
engine = create_engine(
|
||||
settings.mysql_url, # 数据库连接url
|
||||
echo=settings.DATABASE_ECHO, # echo=True 打印生成的SQL语句,开发调试打开,生产关闭
|
||||
pool_recycle = 3600, # 1小时回收连接,解决mysql 8小时断开问题
|
||||
pool_pre_ping=True, # 连接前检查
|
||||
pool_size=settings.DATABASE_POOL_SIZE, # 连接池大小
|
||||
max_overflow=settings.DATABASE_MAX_OVERFLOW, # 最大溢出连接
|
||||
)
|
||||
|
||||
# 2. 创建 sessionmaker(工厂)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
# 3.依赖注入函数
|
||||
def get_db():
|
||||
"""FastAPI 依赖注入用"""
|
||||
db = SessionLocal() # 使用工厂创建 Session
|
||||
try:
|
||||
yield db
|
||||
db.commit() # 无异常则提交
|
||||
except Exception:
|
||||
db.rollback() # 异常则回滚
|
||||
raise
|
||||
finally:
|
||||
db.close() # 确保关闭
|
||||
|
||||
# 4. 定义依赖类型
|
||||
SessionDep = Annotated[Session, Depends(get_db)]
|
||||
|
||||
# 5. 供定时任务使用的上下文管理器
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def get_db_session():
|
||||
"""定时任务、脚本用"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# 使用方式
|
||||
# def crawl_task():
|
||||
# data = fetch_lottery_data()
|
||||
#
|
||||
# with get_db_session() as db:
|
||||
# # 检查是否存在
|
||||
# existing = db.query(LotteryResult).filter_by(draw_no=data['draw_no']).first()
|
||||
# if not existing:
|
||||
# record = LotteryResult(**data)
|
||||
# db.add(record)
|
||||
# # with 块结束时自动 commit
|
||||
0
core/dto/__init__.py
Normal file
0
core/dto/__init__.py
Normal file
12
core/dto/api_response.py
Normal file
12
core/dto/api_response.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from typing import TypeVar, Generic
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# 声明一个泛型类型变量
|
||||
T = TypeVar("T")
|
||||
|
||||
# 泛型响应体
|
||||
class ApiResponse(BaseModel, Generic[T]):
|
||||
code: int
|
||||
message: str
|
||||
data: T | None = None # data可以为空
|
||||
0
core/task/__init__.py
Normal file
0
core/task/__init__.py
Normal file
183
core/task/dlt_scheduler.py
Normal file
183
core/task/dlt_scheduler.py
Normal file
@@ -0,0 +1,183 @@
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from apscheduler.job import Job
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.schedulers.base import STATE_PAUSED, STATE_STOPPED, STATE_RUNNING
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR
|
||||
|
||||
# 添加项目根目录到路径
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from core.task.dlt_spider import DLTSpider
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('lottery_scheduler.log', encoding='utf-8'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def test_fun():
|
||||
logger.info(f"当前时间---{datetime.now()}")
|
||||
|
||||
|
||||
class LotteryScheduler:
|
||||
"""彩票定时任务调度器"""
|
||||
|
||||
def __init__(self):
|
||||
self.scheduler = BackgroundScheduler()
|
||||
self.spider = DLTSpider()
|
||||
self._setup_job_listeners()
|
||||
|
||||
def _setup_job_listeners(self):
|
||||
"""设置任务监听器"""
|
||||
|
||||
def job_listener(event):
|
||||
if event.exception:
|
||||
logger.error(f"任务执行失败: {event.job_id}, 错误: {event.exception}")
|
||||
else:
|
||||
logger.info(f"任务执行成功: {event.job_id}")
|
||||
|
||||
self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
|
||||
|
||||
def fetch_latest_job(self):
|
||||
"""获取最新数据的定时任务"""
|
||||
logger.info(f"开始执行定时任务: 获取最新大乐透数据")
|
||||
try:
|
||||
self.spider.fetch_latest()
|
||||
logger.info("定时任务执行完成")
|
||||
except Exception as e:
|
||||
logger.error(f"定时任务执行异常: {e}", exc_info=True)
|
||||
|
||||
def add_job(self, *args, **kwargs):
|
||||
"""添加任务"""
|
||||
job = self.scheduler.add_job(*args, **kwargs)
|
||||
logger.info(f"任务已添加: {job.id}, 下次执行: {job.next_run_time}")
|
||||
return job
|
||||
|
||||
def remove_job(self, job_id: str):
|
||||
"""移除任务"""
|
||||
try:
|
||||
self.scheduler.remove_job(job_id)
|
||||
logger.info(f"任务已移除: {job_id}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"移除任务失败: {e}")
|
||||
return False
|
||||
|
||||
def start(self):
|
||||
"""启动或恢复调度器"""
|
||||
if self.scheduler.state == STATE_RUNNING:
|
||||
logger.info("调度器已在运行")
|
||||
return
|
||||
|
||||
# 暂停状态:直接恢复
|
||||
if self.scheduler.state == STATE_PAUSED:
|
||||
self.scheduler.resume()
|
||||
logger.info("调度器已恢复运行")
|
||||
return
|
||||
|
||||
# 首次启动:注册任务
|
||||
self.scheduler.add_job(
|
||||
self.fetch_latest_job,
|
||||
trigger=CronTrigger(hour=21, minute=30),
|
||||
id='fetch_latest_dlt',
|
||||
name='获取最新大乐透数据',
|
||||
replace_existing=True
|
||||
)
|
||||
|
||||
# 启动时立即执行一次(可选)
|
||||
# self.scheduler.add_job(
|
||||
# self.fetch_latest_job,
|
||||
# trigger='date',
|
||||
# run_date=datetime.now(),
|
||||
# id='startup_fetch',
|
||||
# name='启动时获取最新数据'
|
||||
# )
|
||||
|
||||
self.scheduler.start()
|
||||
logger.info("定时调度器已启动,将在每晚 21:30 执行")
|
||||
|
||||
# 打印所有任务
|
||||
self.print_jobs()
|
||||
|
||||
def pause(self) -> bool:
|
||||
"""暂停调度器"""
|
||||
if self.scheduler.state == STATE_RUNNING:
|
||||
self.scheduler.pause()
|
||||
logger.info("定时调度器已暂停")
|
||||
return True
|
||||
else:
|
||||
logger.info("调度器未在运行")
|
||||
return False
|
||||
|
||||
def resume(self) -> bool:
|
||||
"""恢复调度器"""
|
||||
if self.scheduler.state == STATE_PAUSED:
|
||||
self.scheduler.resume()
|
||||
logger.info("定时调度器已恢复运行")
|
||||
return True
|
||||
else:
|
||||
logger.info("定时调度器已未在运行状态")
|
||||
return False
|
||||
|
||||
def shutdown(self):
|
||||
"""彻底关闭调度器(应用退出时调用)"""
|
||||
if self.scheduler.state != STATE_STOPPED:
|
||||
self.scheduler.shutdown(wait=False)
|
||||
logger.info("定时调度器已关闭")
|
||||
|
||||
def print_jobs(self):
|
||||
"""打印所有定时任务"""
|
||||
jobs = self.scheduler.get_jobs()
|
||||
if jobs:
|
||||
logger.info("当前定时任务:")
|
||||
for job in jobs:
|
||||
logger.info(f" - {job.id}: {job.name}, 下次执行: {job.next_run_time}")
|
||||
else:
|
||||
logger.info("当前没有定时任务")
|
||||
|
||||
def get_jobs(self) -> List[Job]:
|
||||
"""获取所有定时任务"""
|
||||
jobs = self.scheduler.get_jobs()
|
||||
if jobs:
|
||||
return jobs
|
||||
else:
|
||||
return []
|
||||
|
||||
|
||||
# 创建全局调度器实例
|
||||
scheduler = LotteryScheduler()
|
||||
|
||||
|
||||
def start_scheduler():
|
||||
"""启动调度器的入口函数"""
|
||||
scheduler.start()
|
||||
return scheduler
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试运行
|
||||
scheduler = LotteryScheduler()
|
||||
|
||||
# 启动调度器
|
||||
scheduler.start()
|
||||
|
||||
# 保持运行
|
||||
try:
|
||||
import time
|
||||
|
||||
while True:
|
||||
time.sleep(60)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("收到停止信号,正在关闭调度器...")
|
||||
scheduler.shutdown()
|
||||
366
core/task/dlt_spider.py
Normal file
366
core/task/dlt_spider.py
Normal file
@@ -0,0 +1,366 @@
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Dict, Union, Optional, NamedTuple
|
||||
import time
|
||||
|
||||
import requests
|
||||
from pydantic import BaseModel, field_validator
|
||||
from sqlalchemy import desc
|
||||
|
||||
from core.db import get_db_session
|
||||
from models import LotteryDLT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_draw_numbers(result_str: str) -> List[str]:
|
||||
"""解析开奖号码,支持空格或逗号分隔"""
|
||||
if not result_str or not isinstance(result_str, str):
|
||||
return []
|
||||
|
||||
# 先尝试用空格分割
|
||||
if ' ' in result_str:
|
||||
numbers = result_str.strip().split()
|
||||
elif ',' in result_str:
|
||||
numbers = [num.strip() for num in result_str.split(',')]
|
||||
else:
|
||||
# 如果没有分隔符,尝试按每2个字符分割(如 "0208172933")
|
||||
numbers = [result_str[i:i + 2] for i in range(0, len(result_str), 2)]
|
||||
|
||||
# 过滤掉空字符串并确保是2位数字
|
||||
numbers = [num.zfill(2) for num in numbers if num]
|
||||
return numbers
|
||||
|
||||
|
||||
def get_safe_idx(arr: List[str], idx: int, default: str = "0") -> str:
|
||||
"""安全获取列表元素"""
|
||||
try:
|
||||
if arr and idx < len(arr):
|
||||
return arr[idx].zfill(2)
|
||||
return default.zfill(2)
|
||||
except (IndexError, TypeError):
|
||||
return default.zfill(2)
|
||||
|
||||
|
||||
def normalize_unsort_result(un_result: Any, sort_result_str: str, issue: str) -> str:
|
||||
"""
|
||||
标准化 lotteryUnsortDrawresult 字段
|
||||
|
||||
Args:
|
||||
un_result: 原始 unsort 值
|
||||
sort_result_str: 排序后的结果
|
||||
issue: 期号(用于日志)
|
||||
|
||||
Returns:
|
||||
标准化后的字符串
|
||||
"""
|
||||
# 处理各种情况
|
||||
if isinstance(un_result, str):
|
||||
if un_result == "current" or un_result == "":
|
||||
logger.debug(f"期号 {issue}: unsort 为 '{un_result}',使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
return un_result
|
||||
|
||||
if isinstance(un_result, dict):
|
||||
# 字典类型(包括空字典)
|
||||
logger.debug(f"期号 {issue}: lotteryUnsortDrawresult 是字典,使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
|
||||
if un_result is None:
|
||||
return sort_result_str
|
||||
|
||||
# 其他类型,尝试转换为字符串
|
||||
try:
|
||||
return str(un_result)
|
||||
except:
|
||||
logger.warning(f"期号 {issue}: 无法转换 unsort 结果,使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
|
||||
|
||||
def result_handle(data: dict) -> Optional[LotteryDLT]:
|
||||
"""将爬取的数据格式化为当前服务的数据格式"""
|
||||
try:
|
||||
# 获取基本字段
|
||||
issue = data.get('lotteryDrawNum')
|
||||
if not issue:
|
||||
logger.warning("跳过无效数据: 缺少 lotteryDrawNum")
|
||||
return None
|
||||
|
||||
# 获取排序结果(必须存在)
|
||||
sort_result_str = data.get('lotteryDrawResult', '')
|
||||
if not sort_result_str or not isinstance(sort_result_str, str):
|
||||
logger.warning(f"跳过期号 {issue}: lotteryDrawResult 无效")
|
||||
return None
|
||||
|
||||
# 标准化 unsort 结果
|
||||
un_result = normalize_unsort_result(
|
||||
data.get('lotteryUnsortDrawresult'),
|
||||
sort_result_str,
|
||||
str(issue)
|
||||
)
|
||||
|
||||
# 解析号码
|
||||
un_sort_result = parse_draw_numbers(un_result)
|
||||
sort_result = parse_draw_numbers(sort_result_str)
|
||||
|
||||
# 验证解析结果
|
||||
if len(sort_result) < 7:
|
||||
logger.warning(f"期号 {issue}: 排序号码解析失败,原始数据: {sort_result_str}")
|
||||
return None
|
||||
|
||||
if len(un_sort_result) < 7:
|
||||
logger.debug(f"期号 {issue}: 未排序号码解析失败,使用排序结果")
|
||||
un_sort_result = sort_result
|
||||
|
||||
# 解析时间
|
||||
dt_str = data.get('lotteryDrawTime')
|
||||
if not dt_str or not isinstance(dt_str, str):
|
||||
logger.warning(f"期号 {issue}: 无效的时间格式")
|
||||
return None
|
||||
|
||||
try:
|
||||
dt = datetime.fromisoformat(dt_str.replace(' ', 'T'))
|
||||
except ValueError:
|
||||
# 尝试其他格式
|
||||
dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
record = LotteryDLT(
|
||||
issue=str(issue),
|
||||
game_name=str(data.get('lotteryGameName', '超级大乐透')),
|
||||
draw_date=dt.date(),
|
||||
draw_time=dt,
|
||||
front_sort_1=get_safe_idx(sort_result, 0),
|
||||
front_sort_2=get_safe_idx(sort_result, 1),
|
||||
front_sort_3=get_safe_idx(sort_result, 2),
|
||||
front_sort_4=get_safe_idx(sort_result, 3),
|
||||
front_sort_5=get_safe_idx(sort_result, 4),
|
||||
back_sort_1=get_safe_idx(sort_result, 5),
|
||||
back_sort_2=get_safe_idx(sort_result, 6),
|
||||
front_1=get_safe_idx(un_sort_result, 0),
|
||||
front_2=get_safe_idx(un_sort_result, 1),
|
||||
front_3=get_safe_idx(un_sort_result, 2),
|
||||
front_4=get_safe_idx(un_sort_result, 3),
|
||||
front_5=get_safe_idx(un_sort_result, 4),
|
||||
back_1=get_safe_idx(un_sort_result, 5),
|
||||
back_2=get_safe_idx(un_sort_result, 6)
|
||||
)
|
||||
return record
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理期号 {data.get('lotteryDrawNum', '未知')} 失败: {e}")
|
||||
return None
|
||||
|
||||
@dataclass
|
||||
class ResultCount:
|
||||
"""成功失败数据类"""
|
||||
success_count: int
|
||||
fail_count: int
|
||||
|
||||
class DLTSpider:
|
||||
"""大乐透爬虫"""
|
||||
|
||||
def __init__(self):
|
||||
self.headers = {
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.sporttery.cn/"
|
||||
}
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(self.headers)
|
||||
|
||||
def _fetch_data(self, url: str) -> Optional[dict]:
|
||||
"""通用的数据获取方法"""
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"请求失败: {url}, 错误: {e}")
|
||||
return None
|
||||
|
||||
def _save_record(self, db, record: LotteryDLT) -> bool:
|
||||
"""保存单条记录到数据库"""
|
||||
try:
|
||||
existing = db.query(LotteryDLT).filter(
|
||||
LotteryDLT.issue == record.issue
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"期号 {record.issue} 已存在,跳过")
|
||||
return False
|
||||
|
||||
db.add(record)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存期号 {record.issue} 失败: {e}")
|
||||
return False
|
||||
|
||||
def fetch_latest(self):
|
||||
"""获取最新一期大乐透数据"""
|
||||
url = "https://webapi.sporttery.cn/gateway/lottery/getDigitalDrawInfoV1.qry?param=85,0&isVerify=1"
|
||||
|
||||
data = self._fetch_data(url)
|
||||
if not data:
|
||||
logger.error("获取最新数据失败")
|
||||
return
|
||||
|
||||
dlt_data = dict(data.get("value", {}).get("dlt", {}))
|
||||
if not dlt_data:
|
||||
logger.error("数据格式错误")
|
||||
return
|
||||
|
||||
record = result_handle(dlt_data)
|
||||
if not record:
|
||||
logger.error("数据转换失败")
|
||||
return
|
||||
|
||||
with get_db_session() as db:
|
||||
if self._save_record(db, record):
|
||||
db.commit()
|
||||
logger.info(f"成功保存最新数据: 期号 {record.issue}")
|
||||
else:
|
||||
logger.info(f"期号 {record.issue} 已存在")
|
||||
|
||||
def fetch_by_year(self, year: int) -> tuple[List[LotteryDLT], ResultCount]:
|
||||
"""获取指定年份的所有数据"""
|
||||
logger.info(f"开始爬取 {year} 年的数据")
|
||||
|
||||
year_prefix = str(year)[-2:]
|
||||
all_records = []
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
# 分两段获取:001-100 和 101-200
|
||||
segments = [
|
||||
(f"{year_prefix}001", f"{year_prefix}100"),
|
||||
(f"{year_prefix}101", f"{year_prefix}200")
|
||||
]
|
||||
|
||||
for start_term, end_term in segments:
|
||||
url = (
|
||||
f"https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry"
|
||||
f"?gameNo=85&provinceId=0&isVerify=1"
|
||||
f"&pageNo=1&pageSize=100"
|
||||
f"&startTerm={start_term}&endTerm={end_term}"
|
||||
)
|
||||
|
||||
data = self._fetch_data(url)
|
||||
if not data:
|
||||
logger.warning(f"获取 {start_term}-{end_term} 数据失败")
|
||||
continue
|
||||
|
||||
dlt_list = data.get("value", {}).get("list", [])
|
||||
if not dlt_list:
|
||||
logger.warning(f"{start_term}-{end_term} 没有数据")
|
||||
continue
|
||||
|
||||
for item_data in dlt_list:
|
||||
record = result_handle(item_data)
|
||||
if record:
|
||||
all_records.append(record)
|
||||
success_count += 1
|
||||
else:
|
||||
fail_count += 1
|
||||
|
||||
# 避免请求过快
|
||||
time.sleep(0.5)
|
||||
|
||||
# 按期号排序
|
||||
all_records.sort(key=lambda x: int(x.issue))
|
||||
logger.info(f"{year} 年成功获取 {success_count} 条数据, 失败 {fail_count} 条数据")
|
||||
return all_records, ResultCount(success_count, fail_count)
|
||||
|
||||
def fetch_range(self, start_year: int = 2007, end_year: Optional[int] = None):
|
||||
"""
|
||||
获取年限范围内历史所有的大乐透数据
|
||||
|
||||
Args:
|
||||
start_year: 开始年份(大乐透从2007年开始)
|
||||
end_year: 结束年份,默认为当前年份
|
||||
"""
|
||||
if end_year is None:
|
||||
end_year = datetime.now().year + 1
|
||||
|
||||
logger.info(f"开始爬取 {start_year} 到 {end_year} 年的数据")
|
||||
|
||||
total_count = 0
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
for year in range(start_year, end_year):
|
||||
records, state = self.fetch_by_year(year)
|
||||
success_count += state.success_count
|
||||
fail_count += state.fail_count
|
||||
total_count += (success_count + fail_count)
|
||||
|
||||
# 批量保存到数据库
|
||||
if records:
|
||||
with get_db_session() as db:
|
||||
saved_count = 0
|
||||
for record in records:
|
||||
if self._save_record(db, record):
|
||||
saved_count += 1
|
||||
|
||||
if saved_count > 0:
|
||||
logger.info(f"{year} 年成功保存 {saved_count} 条新数据")
|
||||
else:
|
||||
db.rollback()
|
||||
|
||||
# 避免请求过快
|
||||
time.sleep(1)
|
||||
|
||||
logger.info(f"爬取完成!总共 {total_count} 条数据, 成功: {success_count}, 失败: {fail_count}")
|
||||
|
||||
def fetch_missing(self, target_year: int = 2026):
|
||||
"""
|
||||
补全指定年份缺失的数据
|
||||
|
||||
Args:
|
||||
target_year: 要补全的年份
|
||||
"""
|
||||
logger.info(f"开始补全 {target_year} 年的数据")
|
||||
|
||||
# 获取数据库中已存在的期号
|
||||
with get_db_session() as db:
|
||||
existing_issues = [
|
||||
r[0] for r in db.query(LotteryDLT.issue)
|
||||
.filter(LotteryDLT.issue.startswith(str(target_year)[-2:]))
|
||||
.all()
|
||||
]
|
||||
|
||||
# 获取该年份所有数据
|
||||
records, state = self.fetch_by_year(target_year)
|
||||
|
||||
# 过滤出缺失的
|
||||
missing_records = [r for r in records if r.issue not in existing_issues]
|
||||
success_count = 0
|
||||
if missing_records:
|
||||
logger.info(f"发现 {len(missing_records)} 条缺失数据")
|
||||
with get_db_session() as db:
|
||||
for record in missing_records:
|
||||
if self._save_record(db, record):
|
||||
success_count += 1
|
||||
db.commit()
|
||||
logger.info(f"成功补全 {success_count} 条数据")
|
||||
else:
|
||||
logger.info(f"{target_year} 年数据完整,无需补全")
|
||||
|
||||
def get_latest_issue(self) -> Optional[str]:
|
||||
"""获取数据库中最新的期号"""
|
||||
with get_db_session() as db:
|
||||
record = db.query(LotteryDLT).order_by(
|
||||
desc(LotteryDLT.issue)
|
||||
).first()
|
||||
return record.issue if record else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
spider = DLTSpider()
|
||||
|
||||
# 爬取所有历史数据(从2007年开始)
|
||||
# spider.fetch_range(start_year=2007)
|
||||
|
||||
# 或者只补全2026年的数据
|
||||
spider.fetch_missing(2017)
|
||||
|
||||
# 或者只获取最新一期
|
||||
# spider.fetch_latest()
|
||||
Reference in New Issue
Block a user