Skip to content

Commit 3c438bd

Browse files
committed
更新 LPL.py 和 team_list.json,优化赛季ID获取逻辑并更新团队统计数据
1 parent 6abb93c commit 3c438bd

File tree

3 files changed

+5230
-5221
lines changed

3 files changed

+5230
-5221
lines changed

Data_CrawlProcess/LPL.py

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -181,24 +181,33 @@ async def consumer():
181181
rich_logger.info(f"[LPL] auto_seasonIds: 赛季数据已全部入库,共{len(merged_seasons)}条")
182182

183183
@staticmethod
184-
async def get_seasonIds(col_name: str, seasons: Dict[str, str], rich_progress) -> None:
184+
async def get_seasonIds(col_name: str, seasons: list, rich_progress) -> None:
185185
"""
186186
异步生产者-消费者:获取赛季ID并写入数据库。
187187
:param col_name: MongoDB集合名称
188-
:param seasons: 赛季ID映射字典{str: str}
188+
:param seasons: 赛季信息列表,每项为dict,需提取id和name,id去重
189189
:param rich_progress: RichProgressUtils实例
190190
:return: None
191191
"""
192192
rich_progress = rich_progress or RichProgressUtils()
193193
collection = mongo_utils.use_collection(col_name)
194194
collection.create_index([("season_id", ASCENDING)], unique=True)
195+
# 只保留id和name,并对id去重
196+
seen_ids = set()
197+
season_pairs = []
198+
for s in seasons:
199+
sid = s.get('id')
200+
sname = s.get('name')
201+
if sid and sname and sid not in seen_ids:
202+
season_pairs.append({"season_name": sname, "season_id": sid})
203+
seen_ids.add(sid)
195204
queue = asyncio.Queue(maxsize=20)
196-
fetch_task_id = rich_progress.add_task("[LPL] seasonIDs生产", total=len(seasons))
197-
store_task_id = rich_progress.add_task("[LPL] seasonIDs入库", total=len(seasons))
205+
fetch_task_id = rich_progress.add_task("[LPL] seasonIDs生产", total=len(season_pairs))
206+
store_task_id = rich_progress.add_task("[LPL] seasonIDs入库", total=len(season_pairs))
198207

199208
async def producer():
200-
for season_name, season_id in seasons.items():
201-
await queue.put({"season_name": season_name, "season_id": season_id})
209+
for pair in season_pairs:
210+
await queue.put(pair)
202211
rich_progress.advance(fetch_task_id)
203212
await queue.put(None)
204213

@@ -214,10 +223,10 @@ async def consumer():
214223
pass
215224
count += 1
216225
rich_progress.advance(store_task_id)
217-
rich_progress.update(store_task_id, completed=len(seasons))
226+
rich_progress.update(store_task_id, completed=len(season_pairs))
218227

219228
await asyncio.gather(producer(), consumer())
220-
rich_logger.info(f"爬取完成丨共计[{len(seasons)}]LPL_season")
229+
rich_logger.info(f"爬取完成丨共计[{len(season_pairs)}]LPL_season")
221230

222231
async def get_bMatchIds(self, col_name: str, seasons: Dict[str, str], rich_progress) -> None:
223232
"""

0 commit comments

Comments
 (0)