初始化

This commit is contained in:
2026-09-03 14:31:25 +08:00
parent 30dd0bea14
commit 64a51f2567
23 changed files with 2091 additions and 144 deletions

0
core/__init__.py Normal file
View File

0
core/config/__init__.py Normal file
View File

179
core/config/settings.py Normal file
View File

@@ -0,0 +1,179 @@
from typing import List
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from enum import Enum
class Environment(str, Enum):
"""环境枚举"""
LOCAL = "local"
DEVELOPMENT = "dev"
STAGING = "staging"
PRODUCTION = "prod"
class LogLevel(str, Enum):
"""日志级别枚举"""
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
CRITICAL = "CRITICAL"
class Settings(BaseSettings):
"""
应用配置类
自动从 .env 文件和环境变量中读取配置
"""
# ---------- 基础配置 ----------
APP_NAME: str = Field(
default="Lottery Server",
description="应用名称"
)
APP_DESCRIPTION: str = Field(
default="一个自己看的彩票信息服务",
description="应用描述"
)
ENVIRONMENT: Environment = Field(
default=Environment.LOCAL,
description="运行环境"
)
# ---------- 服务器配置 ----------
HOST: str = Field(
default="0.0.0.0",
description="服务监听地址"
)
PORT: int = Field(
default=8000,
ge=1,
le=65535,
description="服务监听端口"
)
# ---------- 数据库MYSQL配置 ----------
MYSQL_USER: str = Field(
default='root',
description="MySQL用户名"
)
MYSQL_PASSWORD: str = Field(
default='Azure1314',
description="MySQL密码"
)
MYSQL_HOST: str = Field(
default='127.0.0.1',
description="MySQL主机地址"
)
MYSQL_PORT: int = Field(
default=3306,
ge=1,
le=65535,
description="MySQL端口"
)
MYSQL_DB: str = Field(
default='lottery_server',
description="MySQL数据库名"
)
DATABASE_POOL_SIZE: int = Field(
default=10,
ge=1,
le=100,
description="数据库连接池大小"
)
DATABASE_MAX_OVERFLOW: int = Field(
default=20,
description="连接池最大溢出连接数"
)
DATABASE_ECHO: bool = Field(
default=False,
description="是否打印SQL语句"
)
# ---------- CORS配置 ----------
ALLOWED_ORIGINS: List[str] = Field(
default=["*"],
description="允许的CORS源"
)
ALLOWED_METHODS: List[str] = Field(
default=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
description="允许的HTTP方法"
)
ALLOWED_HEADERS: List[str] = Field(
default=["*"],
description="允许的HTTP头"
)
# ---------- 限流配置 ----------
RATE_LIMIT_PER_MINUTE: int = Field(
default=60,
ge=1,
description="每分钟请求限制"
)
RATE_LIMIT_PER_DAY: int = Field(
default=10000,
ge=1,
description="每天请求限制"
)
# ---------- 文件上传配置 ----------
MAX_UPLOAD_SIZE_MB: int = Field(
default=10,
ge=1,
le=100,
description="最大上传文件大小MB"
)
ALLOWED_UPLOAD_EXTENSIONS: List[str] = Field(
default=[".jpg", ".jpeg", ".png", ".pdf", ".docx"],
description="允许上传的文件扩展名"
)
UPLOAD_DIR: str = Field(
default="./uploads",
description="文件上传目录"
)
# ---------- 缓存配置 ----------
CACHE_TTL_SECONDS: int = Field(
default=300,
ge=1,
description="缓存过期时间(秒)"
)
# ---------- 计算属性 ----------
@property
def mysql_url(self) -> str:
"""MySQL连接URL"""
# 连接url格式mysql+pymysql://user:password@host:port/dbname
return f"mysql+pymysql://{self.MYSQL_USER}:{self.MYSQL_PASSWORD}@{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DB}?charset=utf8mb4"
@property
def is_production(self) -> bool:
"""是否为生产环境"""
return self.ENVIRONMENT == Environment.PRODUCTION
@property
def is_development(self) -> bool:
"""是否为开发环境"""
return self.ENVIRONMENT in [Environment.LOCAL, Environment.DEVELOPMENT]
# ---------- 配置加载设置 ----------
model_config = SettingsConfigDict(
env_file=".env.local", # .env文件路径
env_file_encoding="utf-8", # 文件编码
case_sensitive=True, # 环境变量名大小写敏感
extra="ignore", # 忽略未定义的额外字段
validate_default=True, # 验证默认值
# secrets_dir="/run/secrets", # Docker secrets支持可选
)
# 创建全局单例配置实例
settings = Settings()
# 导出配置实例
__all__ = ["settings", "Environment", "LogLevel"]

62
core/db.py Normal file
View File

@@ -0,0 +1,62 @@
from fastapi import Depends
from typing import Annotated
from sqlmodel import Session, create_engine
from core.config.settings import settings
from sqlalchemy.orm import sessionmaker
# 1.引擎配置
engine = create_engine(
settings.mysql_url, # 数据库连接url
echo=settings.DATABASE_ECHO, # echo=True 打印生成的SQL语句开发调试打开生产关闭
pool_recycle = 3600, # 1小时回收连接解决mysql 8小时断开问题
pool_pre_ping=True, # 连接前检查
pool_size=settings.DATABASE_POOL_SIZE, # 连接池大小
max_overflow=settings.DATABASE_MAX_OVERFLOW, # 最大溢出连接
)
# 2. 创建 sessionmaker工厂
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# 3.依赖注入函数
def get_db():
"""FastAPI 依赖注入用"""
db = SessionLocal() # 使用工厂创建 Session
try:
yield db
db.commit() # 无异常则提交
except Exception:
db.rollback() # 异常则回滚
raise
finally:
db.close() # 确保关闭
# 4. 定义依赖类型
SessionDep = Annotated[Session, Depends(get_db)]
# 5. 供定时任务使用的上下文管理器
from contextlib import contextmanager
@contextmanager
def get_db_session():
"""定时任务、脚本用"""
db = SessionLocal()
try:
yield db
db.commit()
except Exception:
db.rollback()
raise
finally:
db.close()
# 使用方式
# def crawl_task():
# data = fetch_lottery_data()
#
# with get_db_session() as db:
# # 检查是否存在
# existing = db.query(LotteryResult).filter_by(draw_no=data['draw_no']).first()
# if not existing:
# record = LotteryResult(**data)
# db.add(record)
# # with 块结束时自动 commit

0
core/dto/__init__.py Normal file
View File

12
core/dto/api_response.py Normal file
View File

@@ -0,0 +1,12 @@
from typing import TypeVar, Generic
from pydantic import BaseModel
# 声明一个泛型类型变量
T = TypeVar("T")
# 泛型响应体
class ApiResponse(BaseModel, Generic[T]):
code: int
message: str
data: T | None = None # data可以为空

0
core/task/__init__.py Normal file
View File

183
core/task/dlt_scheduler.py Normal file
View File

@@ -0,0 +1,183 @@
import logging
import sys
from pathlib import Path
from datetime import datetime
from typing import List
from apscheduler.job import Job
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.schedulers.base import STATE_PAUSED, STATE_STOPPED, STATE_RUNNING
from apscheduler.triggers.cron import CronTrigger
from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR
# 添加项目根目录到路径
sys.path.append(str(Path(__file__).parent.parent.parent))
from core.task.dlt_spider import DLTSpider
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('lottery_scheduler.log', encoding='utf-8'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
def test_fun():
logger.info(f"当前时间---{datetime.now()}")
class LotteryScheduler:
"""彩票定时任务调度器"""
def __init__(self):
self.scheduler = BackgroundScheduler()
self.spider = DLTSpider()
self._setup_job_listeners()
def _setup_job_listeners(self):
"""设置任务监听器"""
def job_listener(event):
if event.exception:
logger.error(f"任务执行失败: {event.job_id}, 错误: {event.exception}")
else:
logger.info(f"任务执行成功: {event.job_id}")
self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
def fetch_latest_job(self):
"""获取最新数据的定时任务"""
logger.info(f"开始执行定时任务: 获取最新大乐透数据")
try:
self.spider.fetch_latest()
logger.info("定时任务执行完成")
except Exception as e:
logger.error(f"定时任务执行异常: {e}", exc_info=True)
def add_job(self, *args, **kwargs):
"""添加任务"""
job = self.scheduler.add_job(*args, **kwargs)
logger.info(f"任务已添加: {job.id}, 下次执行: {job.next_run_time}")
return job
def remove_job(self, job_id: str):
"""移除任务"""
try:
self.scheduler.remove_job(job_id)
logger.info(f"任务已移除: {job_id}")
return True
except Exception as e:
logger.error(f"移除任务失败: {e}")
return False
def start(self):
"""启动或恢复调度器"""
if self.scheduler.state == STATE_RUNNING:
logger.info("调度器已在运行")
return
# 暂停状态:直接恢复
if self.scheduler.state == STATE_PAUSED:
self.scheduler.resume()
logger.info("调度器已恢复运行")
return
# 首次启动:注册任务
self.scheduler.add_job(
self.fetch_latest_job,
trigger=CronTrigger(hour=21, minute=30),
id='fetch_latest_dlt',
name='获取最新大乐透数据',
replace_existing=True
)
# 启动时立即执行一次(可选)
# self.scheduler.add_job(
# self.fetch_latest_job,
# trigger='date',
# run_date=datetime.now(),
# id='startup_fetch',
# name='启动时获取最新数据'
# )
self.scheduler.start()
logger.info("定时调度器已启动,将在每晚 21:30 执行")
# 打印所有任务
self.print_jobs()
def pause(self) -> bool:
"""暂停调度器"""
if self.scheduler.state == STATE_RUNNING:
self.scheduler.pause()
logger.info("定时调度器已暂停")
return True
else:
logger.info("调度器未在运行")
return False
def resume(self) -> bool:
"""恢复调度器"""
if self.scheduler.state == STATE_PAUSED:
self.scheduler.resume()
logger.info("定时调度器已恢复运行")
return True
else:
logger.info("定时调度器已未在运行状态")
return False
def shutdown(self):
"""彻底关闭调度器(应用退出时调用)"""
if self.scheduler.state != STATE_STOPPED:
self.scheduler.shutdown(wait=False)
logger.info("定时调度器已关闭")
def print_jobs(self):
"""打印所有定时任务"""
jobs = self.scheduler.get_jobs()
if jobs:
logger.info("当前定时任务:")
for job in jobs:
logger.info(f" - {job.id}: {job.name}, 下次执行: {job.next_run_time}")
else:
logger.info("当前没有定时任务")
def get_jobs(self) -> List[Job]:
"""获取所有定时任务"""
jobs = self.scheduler.get_jobs()
if jobs:
return jobs
else:
return []
# 创建全局调度器实例
scheduler = LotteryScheduler()
def start_scheduler():
"""启动调度器的入口函数"""
scheduler.start()
return scheduler
if __name__ == "__main__":
# 测试运行
scheduler = LotteryScheduler()
# 启动调度器
scheduler.start()
# 保持运行
try:
import time
while True:
time.sleep(60)
except KeyboardInterrupt:
logger.info("收到停止信号,正在关闭调度器...")
scheduler.shutdown()

366
core/task/dlt_spider.py Normal file
View File

@@ -0,0 +1,366 @@
import logging
from dataclasses import dataclass
from datetime import datetime
from typing import Any, List, Dict, Union, Optional, NamedTuple
import time
import requests
from pydantic import BaseModel, field_validator
from sqlalchemy import desc
from core.db import get_db_session
from models import LotteryDLT
logger = logging.getLogger(__name__)
def parse_draw_numbers(result_str: str) -> List[str]:
"""解析开奖号码,支持空格或逗号分隔"""
if not result_str or not isinstance(result_str, str):
return []
# 先尝试用空格分割
if ' ' in result_str:
numbers = result_str.strip().split()
elif ',' in result_str:
numbers = [num.strip() for num in result_str.split(',')]
else:
# 如果没有分隔符尝试按每2个字符分割如 "0208172933"
numbers = [result_str[i:i + 2] for i in range(0, len(result_str), 2)]
# 过滤掉空字符串并确保是2位数字
numbers = [num.zfill(2) for num in numbers if num]
return numbers
def get_safe_idx(arr: List[str], idx: int, default: str = "0") -> str:
"""安全获取列表元素"""
try:
if arr and idx < len(arr):
return arr[idx].zfill(2)
return default.zfill(2)
except (IndexError, TypeError):
return default.zfill(2)
def normalize_unsort_result(un_result: Any, sort_result_str: str, issue: str) -> str:
"""
标准化 lotteryUnsortDrawresult 字段
Args:
un_result: 原始 unsort 值
sort_result_str: 排序后的结果
issue: 期号(用于日志)
Returns:
标准化后的字符串
"""
# 处理各种情况
if isinstance(un_result, str):
if un_result == "current" or un_result == "":
logger.debug(f"期号 {issue}: unsort 为 '{un_result}',使用 lotteryDrawResult")
return sort_result_str
return un_result
if isinstance(un_result, dict):
# 字典类型(包括空字典)
logger.debug(f"期号 {issue}: lotteryUnsortDrawresult 是字典,使用 lotteryDrawResult")
return sort_result_str
if un_result is None:
return sort_result_str
# 其他类型,尝试转换为字符串
try:
return str(un_result)
except:
logger.warning(f"期号 {issue}: 无法转换 unsort 结果,使用 lotteryDrawResult")
return sort_result_str
def result_handle(data: dict) -> Optional[LotteryDLT]:
"""将爬取的数据格式化为当前服务的数据格式"""
try:
# 获取基本字段
issue = data.get('lotteryDrawNum')
if not issue:
logger.warning("跳过无效数据: 缺少 lotteryDrawNum")
return None
# 获取排序结果(必须存在)
sort_result_str = data.get('lotteryDrawResult', '')
if not sort_result_str or not isinstance(sort_result_str, str):
logger.warning(f"跳过期号 {issue}: lotteryDrawResult 无效")
return None
# 标准化 unsort 结果
un_result = normalize_unsort_result(
data.get('lotteryUnsortDrawresult'),
sort_result_str,
str(issue)
)
# 解析号码
un_sort_result = parse_draw_numbers(un_result)
sort_result = parse_draw_numbers(sort_result_str)
# 验证解析结果
if len(sort_result) < 7:
logger.warning(f"期号 {issue}: 排序号码解析失败,原始数据: {sort_result_str}")
return None
if len(un_sort_result) < 7:
logger.debug(f"期号 {issue}: 未排序号码解析失败,使用排序结果")
un_sort_result = sort_result
# 解析时间
dt_str = data.get('lotteryDrawTime')
if not dt_str or not isinstance(dt_str, str):
logger.warning(f"期号 {issue}: 无效的时间格式")
return None
try:
dt = datetime.fromisoformat(dt_str.replace(' ', 'T'))
except ValueError:
# 尝试其他格式
dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
record = LotteryDLT(
issue=str(issue),
game_name=str(data.get('lotteryGameName', '超级大乐透')),
draw_date=dt.date(),
draw_time=dt,
front_sort_1=get_safe_idx(sort_result, 0),
front_sort_2=get_safe_idx(sort_result, 1),
front_sort_3=get_safe_idx(sort_result, 2),
front_sort_4=get_safe_idx(sort_result, 3),
front_sort_5=get_safe_idx(sort_result, 4),
back_sort_1=get_safe_idx(sort_result, 5),
back_sort_2=get_safe_idx(sort_result, 6),
front_1=get_safe_idx(un_sort_result, 0),
front_2=get_safe_idx(un_sort_result, 1),
front_3=get_safe_idx(un_sort_result, 2),
front_4=get_safe_idx(un_sort_result, 3),
front_5=get_safe_idx(un_sort_result, 4),
back_1=get_safe_idx(un_sort_result, 5),
back_2=get_safe_idx(un_sort_result, 6)
)
return record
except Exception as e:
logger.error(f"处理期号 {data.get('lotteryDrawNum', '未知')} 失败: {e}")
return None
@dataclass
class ResultCount:
"""成功失败数据类"""
success_count: int
fail_count: int
class DLTSpider:
"""大乐透爬虫"""
def __init__(self):
self.headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
"Referer": "https://www.sporttery.cn/"
}
self.session = requests.Session()
self.session.headers.update(self.headers)
def _fetch_data(self, url: str) -> Optional[dict]:
"""通用的数据获取方法"""
try:
response = self.session.get(url, timeout=15)
response.raise_for_status()
return response.json()
except requests.exceptions.RequestException as e:
logger.error(f"请求失败: {url}, 错误: {e}")
return None
def _save_record(self, db, record: LotteryDLT) -> bool:
"""保存单条记录到数据库"""
try:
existing = db.query(LotteryDLT).filter(
LotteryDLT.issue == record.issue
).first()
if existing:
logger.debug(f"期号 {record.issue} 已存在,跳过")
return False
db.add(record)
return True
except Exception as e:
logger.error(f"保存期号 {record.issue} 失败: {e}")
return False
def fetch_latest(self):
"""获取最新一期大乐透数据"""
url = "https://webapi.sporttery.cn/gateway/lottery/getDigitalDrawInfoV1.qry?param=85,0&isVerify=1"
data = self._fetch_data(url)
if not data:
logger.error("获取最新数据失败")
return
dlt_data = dict(data.get("value", {}).get("dlt", {}))
if not dlt_data:
logger.error("数据格式错误")
return
record = result_handle(dlt_data)
if not record:
logger.error("数据转换失败")
return
with get_db_session() as db:
if self._save_record(db, record):
db.commit()
logger.info(f"成功保存最新数据: 期号 {record.issue}")
else:
logger.info(f"期号 {record.issue} 已存在")
def fetch_by_year(self, year: int) -> tuple[List[LotteryDLT], ResultCount]:
"""获取指定年份的所有数据"""
logger.info(f"开始爬取 {year} 年的数据")
year_prefix = str(year)[-2:]
all_records = []
success_count = 0
fail_count = 0
# 分两段获取001-100 和 101-200
segments = [
(f"{year_prefix}001", f"{year_prefix}100"),
(f"{year_prefix}101", f"{year_prefix}200")
]
for start_term, end_term in segments:
url = (
f"https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry"
f"?gameNo=85&provinceId=0&isVerify=1"
f"&pageNo=1&pageSize=100"
f"&startTerm={start_term}&endTerm={end_term}"
)
data = self._fetch_data(url)
if not data:
logger.warning(f"获取 {start_term}-{end_term} 数据失败")
continue
dlt_list = data.get("value", {}).get("list", [])
if not dlt_list:
logger.warning(f"{start_term}-{end_term} 没有数据")
continue
for item_data in dlt_list:
record = result_handle(item_data)
if record:
all_records.append(record)
success_count += 1
else:
fail_count += 1
# 避免请求过快
time.sleep(0.5)
# 按期号排序
all_records.sort(key=lambda x: int(x.issue))
logger.info(f"{year} 年成功获取 {success_count} 条数据, 失败 {fail_count} 条数据")
return all_records, ResultCount(success_count, fail_count)
def fetch_range(self, start_year: int = 2007, end_year: Optional[int] = None):
"""
获取年限范围内历史所有的大乐透数据
Args:
start_year: 开始年份大乐透从2007年开始
end_year: 结束年份,默认为当前年份
"""
if end_year is None:
end_year = datetime.now().year + 1
logger.info(f"开始爬取 {start_year}{end_year} 年的数据")
total_count = 0
success_count = 0
fail_count = 0
for year in range(start_year, end_year):
records, state = self.fetch_by_year(year)
success_count += state.success_count
fail_count += state.fail_count
total_count += (success_count + fail_count)
# 批量保存到数据库
if records:
with get_db_session() as db:
saved_count = 0
for record in records:
if self._save_record(db, record):
saved_count += 1
if saved_count > 0:
logger.info(f"{year} 年成功保存 {saved_count} 条新数据")
else:
db.rollback()
# 避免请求过快
time.sleep(1)
logger.info(f"爬取完成!总共 {total_count} 条数据, 成功: {success_count}, 失败: {fail_count}")
def fetch_missing(self, target_year: int = 2026):
"""
补全指定年份缺失的数据
Args:
target_year: 要补全的年份
"""
logger.info(f"开始补全 {target_year} 年的数据")
# 获取数据库中已存在的期号
with get_db_session() as db:
existing_issues = [
r[0] for r in db.query(LotteryDLT.issue)
.filter(LotteryDLT.issue.startswith(str(target_year)[-2:]))
.all()
]
# 获取该年份所有数据
records, state = self.fetch_by_year(target_year)
# 过滤出缺失的
missing_records = [r for r in records if r.issue not in existing_issues]
success_count = 0
if missing_records:
logger.info(f"发现 {len(missing_records)} 条缺失数据")
with get_db_session() as db:
for record in missing_records:
if self._save_record(db, record):
success_count += 1
db.commit()
logger.info(f"成功补全 {success_count} 条数据")
else:
logger.info(f"{target_year} 年数据完整,无需补全")
def get_latest_issue(self) -> Optional[str]:
"""获取数据库中最新的期号"""
with get_db_session() as db:
record = db.query(LotteryDLT).order_by(
desc(LotteryDLT.issue)
).first()
return record.issue if record else None
if __name__ == "__main__":
spider = DLTSpider()
# 爬取所有历史数据从2007年开始
# spider.fetch_range(start_year=2007)
# 或者只补全2026年的数据
spider.fetch_missing(2017)
# 或者只获取最新一期
# spider.fetch_latest()