File tree Expand file tree Collapse file tree 2 files changed +4
-0
lines changed Expand file tree Collapse file tree 2 files changed +4
-0
lines changed Original file line number Diff line number Diff line change @@ -110,6 +110,7 @@ async def semantic_text_chunker(req: func.HttpRequest) -> func.HttpResponse:
110
110
"similarity_threshold" , 0.8
111
111
)
112
112
max_chunk_tokens = semantic_text_chunker_config .get ("max_chunk_tokens" , 500 )
113
+ min_chunk_tokens = semantic_text_chunker_config .get ("min_chunk_tokens" , 50 )
113
114
114
115
except ValueError :
115
116
return func .HttpResponse (
@@ -124,6 +125,7 @@ async def semantic_text_chunker(req: func.HttpRequest) -> func.HttpResponse:
124
125
num_surrounding_sentences = num_surrounding_sentences ,
125
126
similarity_threshold = similarity_threshold ,
126
127
max_chunk_tokens = max_chunk_tokens ,
128
+ min_chunk_tokens = min_chunk_tokens ,
127
129
)
128
130
129
131
for value in values :
Original file line number Diff line number Diff line change @@ -260,6 +260,7 @@ def get_text_split_skill(
260
260
num_surrounding_sentences : int = 1 ,
261
261
similarity_threshold : float = 0.8 ,
262
262
max_chunk_tokens : int = 200 ,
263
+ min_chunk_tokens : int = 50 ,
263
264
) -> SplitSkill :
264
265
"""Get the skill for text split.
265
266
@@ -303,6 +304,7 @@ def get_text_split_skill(
303
304
"num_surrounding_sentences" : num_surrounding_sentences ,
304
305
"similarity_threshold" : similarity_threshold ,
305
306
"max_chunk_tokens" : max_chunk_tokens ,
307
+ "min_chunk_tokens" : min_chunk_tokens ,
306
308
},
307
309
inputs = semantic_text_chunker_skill_inputs ,
308
310
outputs = semantic_text_chunker_skill_outputs ,
You can’t perform that action at this time.
0 commit comments