@@ -181,24 +181,33 @@ async def consumer():
181
181
rich_logger .info (f"[LPL] auto_seasonIds: 赛季数据已全部入库,共{ len (merged_seasons )} 条" )
182
182
183
183
@staticmethod
184
- async def get_seasonIds (col_name : str , seasons : Dict [ str , str ] , rich_progress ) -> None :
184
+ async def get_seasonIds (col_name : str , seasons : list , rich_progress ) -> None :
185
185
"""
186
186
异步生产者-消费者:获取赛季ID并写入数据库。
187
187
:param col_name: MongoDB集合名称
188
- :param seasons: 赛季ID映射字典{str: str}
188
+ :param seasons: 赛季信息列表,每项为dict,需提取id和name,id去重
189
189
:param rich_progress: RichProgressUtils实例
190
190
:return: None
191
191
"""
192
192
rich_progress = rich_progress or RichProgressUtils ()
193
193
collection = mongo_utils .use_collection (col_name )
194
194
collection .create_index ([("season_id" , ASCENDING )], unique = True )
195
+ # 只保留id和name,并对id去重
196
+ seen_ids = set ()
197
+ season_pairs = []
198
+ for s in seasons :
199
+ sid = s .get ('id' )
200
+ sname = s .get ('name' )
201
+ if sid and sname and sid not in seen_ids :
202
+ season_pairs .append ({"season_name" : sname , "season_id" : sid })
203
+ seen_ids .add (sid )
195
204
queue = asyncio .Queue (maxsize = 20 )
196
- fetch_task_id = rich_progress .add_task ("[LPL] seasonIDs生产" , total = len (seasons ))
197
- store_task_id = rich_progress .add_task ("[LPL] seasonIDs入库" , total = len (seasons ))
205
+ fetch_task_id = rich_progress .add_task ("[LPL] seasonIDs生产" , total = len (season_pairs ))
206
+ store_task_id = rich_progress .add_task ("[LPL] seasonIDs入库" , total = len (season_pairs ))
198
207
199
208
async def producer ():
200
- for season_name , season_id in seasons . items () :
201
- await queue .put ({ "season_name" : season_name , "season_id" : season_id } )
209
+ for pair in season_pairs :
210
+ await queue .put (pair )
202
211
rich_progress .advance (fetch_task_id )
203
212
await queue .put (None )
204
213
@@ -214,10 +223,10 @@ async def consumer():
214
223
pass
215
224
count += 1
216
225
rich_progress .advance (store_task_id )
217
- rich_progress .update (store_task_id , completed = len (seasons ))
226
+ rich_progress .update (store_task_id , completed = len (season_pairs ))
218
227
219
228
await asyncio .gather (producer (), consumer ())
220
- rich_logger .info (f"爬取完成丨共计[{ len (seasons )} ]LPL_season" )
229
+ rich_logger .info (f"爬取完成丨共计[{ len (season_pairs )} ]LPL_season" )
221
230
222
231
async def get_bMatchIds (self , col_name : str , seasons : Dict [str , str ], rich_progress ) -> None :
223
232
"""
0 commit comments