初始化
This commit is contained in:
0
core/task/__init__.py
Normal file
0
core/task/__init__.py
Normal file
183
core/task/dlt_scheduler.py
Normal file
183
core/task/dlt_scheduler.py
Normal file
@@ -0,0 +1,183 @@
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from apscheduler.job import Job
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.schedulers.base import STATE_PAUSED, STATE_STOPPED, STATE_RUNNING
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR
|
||||
|
||||
# 添加项目根目录到路径
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from core.task.dlt_spider import DLTSpider
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler('lottery_scheduler.log', encoding='utf-8'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def test_fun():
|
||||
logger.info(f"当前时间---{datetime.now()}")
|
||||
|
||||
|
||||
class LotteryScheduler:
|
||||
"""彩票定时任务调度器"""
|
||||
|
||||
def __init__(self):
|
||||
self.scheduler = BackgroundScheduler()
|
||||
self.spider = DLTSpider()
|
||||
self._setup_job_listeners()
|
||||
|
||||
def _setup_job_listeners(self):
|
||||
"""设置任务监听器"""
|
||||
|
||||
def job_listener(event):
|
||||
if event.exception:
|
||||
logger.error(f"任务执行失败: {event.job_id}, 错误: {event.exception}")
|
||||
else:
|
||||
logger.info(f"任务执行成功: {event.job_id}")
|
||||
|
||||
self.scheduler.add_listener(job_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
|
||||
|
||||
def fetch_latest_job(self):
|
||||
"""获取最新数据的定时任务"""
|
||||
logger.info(f"开始执行定时任务: 获取最新大乐透数据")
|
||||
try:
|
||||
self.spider.fetch_latest()
|
||||
logger.info("定时任务执行完成")
|
||||
except Exception as e:
|
||||
logger.error(f"定时任务执行异常: {e}", exc_info=True)
|
||||
|
||||
def add_job(self, *args, **kwargs):
|
||||
"""添加任务"""
|
||||
job = self.scheduler.add_job(*args, **kwargs)
|
||||
logger.info(f"任务已添加: {job.id}, 下次执行: {job.next_run_time}")
|
||||
return job
|
||||
|
||||
def remove_job(self, job_id: str):
|
||||
"""移除任务"""
|
||||
try:
|
||||
self.scheduler.remove_job(job_id)
|
||||
logger.info(f"任务已移除: {job_id}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"移除任务失败: {e}")
|
||||
return False
|
||||
|
||||
def start(self):
|
||||
"""启动或恢复调度器"""
|
||||
if self.scheduler.state == STATE_RUNNING:
|
||||
logger.info("调度器已在运行")
|
||||
return
|
||||
|
||||
# 暂停状态：直接恢复
|
||||
if self.scheduler.state == STATE_PAUSED:
|
||||
self.scheduler.resume()
|
||||
logger.info("调度器已恢复运行")
|
||||
return
|
||||
|
||||
# 首次启动：注册任务
|
||||
self.scheduler.add_job(
|
||||
self.fetch_latest_job,
|
||||
trigger=CronTrigger(hour=21, minute=30),
|
||||
id='fetch_latest_dlt',
|
||||
name='获取最新大乐透数据',
|
||||
replace_existing=True
|
||||
)
|
||||
|
||||
# 启动时立即执行一次（可选）
|
||||
# self.scheduler.add_job(
|
||||
# self.fetch_latest_job,
|
||||
# trigger='date',
|
||||
# run_date=datetime.now(),
|
||||
# id='startup_fetch',
|
||||
# name='启动时获取最新数据'
|
||||
# )
|
||||
|
||||
self.scheduler.start()
|
||||
logger.info("定时调度器已启动，将在每晚 21:30 执行")
|
||||
|
||||
# 打印所有任务
|
||||
self.print_jobs()
|
||||
|
||||
def pause(self) -> bool:
|
||||
"""暂停调度器"""
|
||||
if self.scheduler.state == STATE_RUNNING:
|
||||
self.scheduler.pause()
|
||||
logger.info("定时调度器已暂停")
|
||||
return True
|
||||
else:
|
||||
logger.info("调度器未在运行")
|
||||
return False
|
||||
|
||||
def resume(self) -> bool:
|
||||
"""恢复调度器"""
|
||||
if self.scheduler.state == STATE_PAUSED:
|
||||
self.scheduler.resume()
|
||||
logger.info("定时调度器已恢复运行")
|
||||
return True
|
||||
else:
|
||||
logger.info("定时调度器已未在运行状态")
|
||||
return False
|
||||
|
||||
def shutdown(self):
|
||||
"""彻底关闭调度器（应用退出时调用）"""
|
||||
if self.scheduler.state != STATE_STOPPED:
|
||||
self.scheduler.shutdown(wait=False)
|
||||
logger.info("定时调度器已关闭")
|
||||
|
||||
def print_jobs(self):
|
||||
"""打印所有定时任务"""
|
||||
jobs = self.scheduler.get_jobs()
|
||||
if jobs:
|
||||
logger.info("当前定时任务:")
|
||||
for job in jobs:
|
||||
logger.info(f" - {job.id}: {job.name}, 下次执行: {job.next_run_time}")
|
||||
else:
|
||||
logger.info("当前没有定时任务")
|
||||
|
||||
def get_jobs(self) -> List[Job]:
|
||||
"""获取所有定时任务"""
|
||||
jobs = self.scheduler.get_jobs()
|
||||
if jobs:
|
||||
return jobs
|
||||
else:
|
||||
return []
|
||||
|
||||
|
||||
# 创建全局调度器实例
|
||||
scheduler = LotteryScheduler()
|
||||
|
||||
|
||||
def start_scheduler():
|
||||
"""启动调度器的入口函数"""
|
||||
scheduler.start()
|
||||
return scheduler
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试运行
|
||||
scheduler = LotteryScheduler()
|
||||
|
||||
# 启动调度器
|
||||
scheduler.start()
|
||||
|
||||
# 保持运行
|
||||
try:
|
||||
import time
|
||||
|
||||
while True:
|
||||
time.sleep(60)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("收到停止信号，正在关闭调度器...")
|
||||
scheduler.shutdown()
|
||||
366
core/task/dlt_spider.py
Normal file
366
core/task/dlt_spider.py
Normal file
@@ -0,0 +1,366 @@
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Dict, Union, Optional, NamedTuple
|
||||
import time
|
||||
|
||||
import requests
|
||||
from pydantic import BaseModel, field_validator
|
||||
from sqlalchemy import desc
|
||||
|
||||
from core.db import get_db_session
|
||||
from models import LotteryDLT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_draw_numbers(result_str: str) -> List[str]:
|
||||
"""解析开奖号码，支持空格或逗号分隔"""
|
||||
if not result_str or not isinstance(result_str, str):
|
||||
return []
|
||||
|
||||
# 先尝试用空格分割
|
||||
if ' ' in result_str:
|
||||
numbers = result_str.strip().split()
|
||||
elif ',' in result_str:
|
||||
numbers = [num.strip() for num in result_str.split(',')]
|
||||
else:
|
||||
# 如果没有分隔符，尝试按每2个字符分割（如 "0208172933"）
|
||||
numbers = [result_str[i:i + 2] for i in range(0, len(result_str), 2)]
|
||||
|
||||
# 过滤掉空字符串并确保是2位数字
|
||||
numbers = [num.zfill(2) for num in numbers if num]
|
||||
return numbers
|
||||
|
||||
|
||||
def get_safe_idx(arr: List[str], idx: int, default: str = "0") -> str:
|
||||
"""安全获取列表元素"""
|
||||
try:
|
||||
if arr and idx < len(arr):
|
||||
return arr[idx].zfill(2)
|
||||
return default.zfill(2)
|
||||
except (IndexError, TypeError):
|
||||
return default.zfill(2)
|
||||
|
||||
|
||||
def normalize_unsort_result(un_result: Any, sort_result_str: str, issue: str) -> str:
|
||||
"""
|
||||
标准化 lotteryUnsortDrawresult 字段
|
||||
|
||||
Args:
|
||||
un_result: 原始 unsort 值
|
||||
sort_result_str: 排序后的结果
|
||||
issue: 期号（用于日志）
|
||||
|
||||
Returns:
|
||||
标准化后的字符串
|
||||
"""
|
||||
# 处理各种情况
|
||||
if isinstance(un_result, str):
|
||||
if un_result == "current" or un_result == "":
|
||||
logger.debug(f"期号 {issue}: unsort 为 '{un_result}'，使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
return un_result
|
||||
|
||||
if isinstance(un_result, dict):
|
||||
# 字典类型（包括空字典）
|
||||
logger.debug(f"期号 {issue}: lotteryUnsortDrawresult 是字典，使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
|
||||
if un_result is None:
|
||||
return sort_result_str
|
||||
|
||||
# 其他类型，尝试转换为字符串
|
||||
try:
|
||||
return str(un_result)
|
||||
except:
|
||||
logger.warning(f"期号 {issue}: 无法转换 unsort 结果，使用 lotteryDrawResult")
|
||||
return sort_result_str
|
||||
|
||||
|
||||
def result_handle(data: dict) -> Optional[LotteryDLT]:
|
||||
"""将爬取的数据格式化为当前服务的数据格式"""
|
||||
try:
|
||||
# 获取基本字段
|
||||
issue = data.get('lotteryDrawNum')
|
||||
if not issue:
|
||||
logger.warning("跳过无效数据: 缺少 lotteryDrawNum")
|
||||
return None
|
||||
|
||||
# 获取排序结果（必须存在）
|
||||
sort_result_str = data.get('lotteryDrawResult', '')
|
||||
if not sort_result_str or not isinstance(sort_result_str, str):
|
||||
logger.warning(f"跳过期号 {issue}: lotteryDrawResult 无效")
|
||||
return None
|
||||
|
||||
# 标准化 unsort 结果
|
||||
un_result = normalize_unsort_result(
|
||||
data.get('lotteryUnsortDrawresult'),
|
||||
sort_result_str,
|
||||
str(issue)
|
||||
)
|
||||
|
||||
# 解析号码
|
||||
un_sort_result = parse_draw_numbers(un_result)
|
||||
sort_result = parse_draw_numbers(sort_result_str)
|
||||
|
||||
# 验证解析结果
|
||||
if len(sort_result) < 7:
|
||||
logger.warning(f"期号 {issue}: 排序号码解析失败，原始数据: {sort_result_str}")
|
||||
return None
|
||||
|
||||
if len(un_sort_result) < 7:
|
||||
logger.debug(f"期号 {issue}: 未排序号码解析失败，使用排序结果")
|
||||
un_sort_result = sort_result
|
||||
|
||||
# 解析时间
|
||||
dt_str = data.get('lotteryDrawTime')
|
||||
if not dt_str or not isinstance(dt_str, str):
|
||||
logger.warning(f"期号 {issue}: 无效的时间格式")
|
||||
return None
|
||||
|
||||
try:
|
||||
dt = datetime.fromisoformat(dt_str.replace(' ', 'T'))
|
||||
except ValueError:
|
||||
# 尝试其他格式
|
||||
dt = datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
record = LotteryDLT(
|
||||
issue=str(issue),
|
||||
game_name=str(data.get('lotteryGameName', '超级大乐透')),
|
||||
draw_date=dt.date(),
|
||||
draw_time=dt,
|
||||
front_sort_1=get_safe_idx(sort_result, 0),
|
||||
front_sort_2=get_safe_idx(sort_result, 1),
|
||||
front_sort_3=get_safe_idx(sort_result, 2),
|
||||
front_sort_4=get_safe_idx(sort_result, 3),
|
||||
front_sort_5=get_safe_idx(sort_result, 4),
|
||||
back_sort_1=get_safe_idx(sort_result, 5),
|
||||
back_sort_2=get_safe_idx(sort_result, 6),
|
||||
front_1=get_safe_idx(un_sort_result, 0),
|
||||
front_2=get_safe_idx(un_sort_result, 1),
|
||||
front_3=get_safe_idx(un_sort_result, 2),
|
||||
front_4=get_safe_idx(un_sort_result, 3),
|
||||
front_5=get_safe_idx(un_sort_result, 4),
|
||||
back_1=get_safe_idx(un_sort_result, 5),
|
||||
back_2=get_safe_idx(un_sort_result, 6)
|
||||
)
|
||||
return record
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理期号 {data.get('lotteryDrawNum', '未知')} 失败: {e}")
|
||||
return None
|
||||
|
||||
@dataclass
|
||||
class ResultCount:
|
||||
"""成功失败数据类"""
|
||||
success_count: int
|
||||
fail_count: int
|
||||
|
||||
class DLTSpider:
|
||||
"""大乐透爬虫"""
|
||||
|
||||
def __init__(self):
|
||||
self.headers = {
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.sporttery.cn/"
|
||||
}
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(self.headers)
|
||||
|
||||
def _fetch_data(self, url: str) -> Optional[dict]:
|
||||
"""通用的数据获取方法"""
|
||||
try:
|
||||
response = self.session.get(url, timeout=15)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"请求失败: {url}, 错误: {e}")
|
||||
return None
|
||||
|
||||
def _save_record(self, db, record: LotteryDLT) -> bool:
|
||||
"""保存单条记录到数据库"""
|
||||
try:
|
||||
existing = db.query(LotteryDLT).filter(
|
||||
LotteryDLT.issue == record.issue
|
||||
).first()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"期号 {record.issue} 已存在，跳过")
|
||||
return False
|
||||
|
||||
db.add(record)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存期号 {record.issue} 失败: {e}")
|
||||
return False
|
||||
|
||||
def fetch_latest(self):
|
||||
"""获取最新一期大乐透数据"""
|
||||
url = "https://webapi.sporttery.cn/gateway/lottery/getDigitalDrawInfoV1.qry?param=85,0&isVerify=1"
|
||||
|
||||
data = self._fetch_data(url)
|
||||
if not data:
|
||||
logger.error("获取最新数据失败")
|
||||
return
|
||||
|
||||
dlt_data = dict(data.get("value", {}).get("dlt", {}))
|
||||
if not dlt_data:
|
||||
logger.error("数据格式错误")
|
||||
return
|
||||
|
||||
record = result_handle(dlt_data)
|
||||
if not record:
|
||||
logger.error("数据转换失败")
|
||||
return
|
||||
|
||||
with get_db_session() as db:
|
||||
if self._save_record(db, record):
|
||||
db.commit()
|
||||
logger.info(f"成功保存最新数据: 期号 {record.issue}")
|
||||
else:
|
||||
logger.info(f"期号 {record.issue} 已存在")
|
||||
|
||||
def fetch_by_year(self, year: int) -> tuple[List[LotteryDLT], ResultCount]:
|
||||
"""获取指定年份的所有数据"""
|
||||
logger.info(f"开始爬取 {year} 年的数据")
|
||||
|
||||
year_prefix = str(year)[-2:]
|
||||
all_records = []
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
# 分两段获取：001-100 和 101-200
|
||||
segments = [
|
||||
(f"{year_prefix}001", f"{year_prefix}100"),
|
||||
(f"{year_prefix}101", f"{year_prefix}200")
|
||||
]
|
||||
|
||||
for start_term, end_term in segments:
|
||||
url = (
|
||||
f"https://webapi.sporttery.cn/gateway/lottery/getHistoryPageListV1.qry"
|
||||
f"?gameNo=85&provinceId=0&isVerify=1"
|
||||
f"&pageNo=1&pageSize=100"
|
||||
f"&startTerm={start_term}&endTerm={end_term}"
|
||||
)
|
||||
|
||||
data = self._fetch_data(url)
|
||||
if not data:
|
||||
logger.warning(f"获取 {start_term}-{end_term} 数据失败")
|
||||
continue
|
||||
|
||||
dlt_list = data.get("value", {}).get("list", [])
|
||||
if not dlt_list:
|
||||
logger.warning(f"{start_term}-{end_term} 没有数据")
|
||||
continue
|
||||
|
||||
for item_data in dlt_list:
|
||||
record = result_handle(item_data)
|
||||
if record:
|
||||
all_records.append(record)
|
||||
success_count += 1
|
||||
else:
|
||||
fail_count += 1
|
||||
|
||||
# 避免请求过快
|
||||
time.sleep(0.5)
|
||||
|
||||
# 按期号排序
|
||||
all_records.sort(key=lambda x: int(x.issue))
|
||||
logger.info(f"{year} 年成功获取 {success_count} 条数据, 失败 {fail_count} 条数据")
|
||||
return all_records, ResultCount(success_count, fail_count)
|
||||
|
||||
def fetch_range(self, start_year: int = 2007, end_year: Optional[int] = None):
|
||||
"""
|
||||
获取年限范围内历史所有的大乐透数据
|
||||
|
||||
Args:
|
||||
start_year: 开始年份（大乐透从2007年开始）
|
||||
end_year: 结束年份，默认为当前年份
|
||||
"""
|
||||
if end_year is None:
|
||||
end_year = datetime.now().year + 1
|
||||
|
||||
logger.info(f"开始爬取 {start_year} 到 {end_year} 年的数据")
|
||||
|
||||
total_count = 0
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
for year in range(start_year, end_year):
|
||||
records, state = self.fetch_by_year(year)
|
||||
success_count += state.success_count
|
||||
fail_count += state.fail_count
|
||||
total_count += (success_count + fail_count)
|
||||
|
||||
# 批量保存到数据库
|
||||
if records:
|
||||
with get_db_session() as db:
|
||||
saved_count = 0
|
||||
for record in records:
|
||||
if self._save_record(db, record):
|
||||
saved_count += 1
|
||||
|
||||
if saved_count > 0:
|
||||
logger.info(f"{year} 年成功保存 {saved_count} 条新数据")
|
||||
else:
|
||||
db.rollback()
|
||||
|
||||
# 避免请求过快
|
||||
time.sleep(1)
|
||||
|
||||
logger.info(f"爬取完成！总共 {total_count} 条数据, 成功: {success_count}, 失败: {fail_count}")
|
||||
|
||||
def fetch_missing(self, target_year: int = 2026):
|
||||
"""
|
||||
补全指定年份缺失的数据
|
||||
|
||||
Args:
|
||||
target_year: 要补全的年份
|
||||
"""
|
||||
logger.info(f"开始补全 {target_year} 年的数据")
|
||||
|
||||
# 获取数据库中已存在的期号
|
||||
with get_db_session() as db:
|
||||
existing_issues = [
|
||||
r[0] for r in db.query(LotteryDLT.issue)
|
||||
.filter(LotteryDLT.issue.startswith(str(target_year)[-2:]))
|
||||
.all()
|
||||
]
|
||||
|
||||
# 获取该年份所有数据
|
||||
records, state = self.fetch_by_year(target_year)
|
||||
|
||||
# 过滤出缺失的
|
||||
missing_records = [r for r in records if r.issue not in existing_issues]
|
||||
success_count = 0
|
||||
if missing_records:
|
||||
logger.info(f"发现 {len(missing_records)} 条缺失数据")
|
||||
with get_db_session() as db:
|
||||
for record in missing_records:
|
||||
if self._save_record(db, record):
|
||||
success_count += 1
|
||||
db.commit()
|
||||
logger.info(f"成功补全 {success_count} 条数据")
|
||||
else:
|
||||
logger.info(f"{target_year} 年数据完整，无需补全")
|
||||
|
||||
def get_latest_issue(self) -> Optional[str]:
|
||||
"""获取数据库中最新的期号"""
|
||||
with get_db_session() as db:
|
||||
record = db.query(LotteryDLT).order_by(
|
||||
desc(LotteryDLT.issue)
|
||||
).first()
|
||||
return record.issue if record else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
spider = DLTSpider()
|
||||
|
||||
# 爬取所有历史数据（从2007年开始）
|
||||
# spider.fetch_range(start_year=2007)
|
||||
|
||||
# 或者只补全2026年的数据
|
||||
spider.fetch_missing(2017)
|
||||
|
||||
# 或者只获取最新一期
|
||||
# spider.fetch_latest()
|
||||
Reference in New Issue
Block a user