Skip to content

Commit 6994c06

Browse files
aleksandr-mokrovaleksandr-mokrov
authored
Remove templates from system promt for genai. genai applies templates… (#3014)
… inside CVS-169418 genai applies chat template to the system message inside pipeline. It shouldn't be applied before start_chat anymore.
1 parent d2d07dd commit 6994c06

File tree

3 files changed

+22
-26
lines changed

3 files changed

+22
-26
lines changed

‎notebooks/deepseek-r1/gradio_helper.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def get_system_prompt(model_language, system_prompt=None):
5656
def make_demo(pipe, model_configuration, model_id, model_language, disable_advanced=False):
5757
import gradio as gr
5858

59-
start_message = get_system_prompt(model_language, model_configuration.get("system_prompt"))
59+
start_message = get_system_prompt(model_language, model_configuration.get("start_message"))
6060
if "genai_chat_template" in model_configuration:
6161
pipe.get_tokenizer().set_chat_template(model_configuration["genai_chat_template"])
6262

‎notebooks/llm-chatbot/gradio_helper_genai.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def make_demo(pipe, model_configuration, model_id, model_language, disable_advan
7070

7171
max_new_tokens = 2048
7272

73-
start_message = get_system_prompt(model_language, model_configuration.get("system_prompt"))
73+
start_message = get_system_prompt(model_language, model_configuration.get("start_message"))
7474
if "genai_chat_template" in model_configuration:
7575
pipe.get_tokenizer().set_chat_template(model_configuration["genai_chat_template"])
7676

‎utils/llm_config.py

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ def qwen_completion_to_prompt(completion):
136136
"tiny-llama-1b-chat": {
137137
"model_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
138138
"remote_code": False,
139-
"start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
139+
"start_message": DEFAULT_SYSTEM_PROMPT,
140140
"history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
141141
"current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
142142
"rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
@@ -150,24 +150,23 @@ def qwen_completion_to_prompt(completion):
150150
"DeepSeek-R1-Distill-Qwen-1.5B": {
151151
"model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
152152
"genai_chat_template": "{% for message in messages %}{% if loop.first %}{{ '<｜begin▁of▁sentence｜>' }}{% endif %}{% if message['role'] == 'system' and message['content'] %}{{ message['content'] }}{% elif message['role'] == 'user' %}{{ '<｜User｜>' + message['content'] }}{% elif message['role'] == 'assistant' %}{{ '<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>' }}{% endif %}{% if loop.last and add_generation_prompt and message['role'] != 'assistant' %}{{ '<｜Assistant｜>' }}{% endif %}{% endfor %}",
153-
"system_prompt": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoidi mixing languages.",
153+
"start_message": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoidi mixing languages.",
154154
},
155155
"DeepSeek-R1-Distill-Qwen-7B": {
156156
"model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
157157
"genai_chat_template": "{% for message in messages %}{% if loop.first %}{{ '<｜begin▁of▁sentence｜>' }}{% endif %}{% if message['role'] == 'system' and message['content'] %}{{ message['content'] }}{% elif message['role'] == 'user' %}{{ '<｜User｜>' + message['content'] }}{% elif message['role'] == 'assistant' %}{{ '<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>' }}{% endif %}{% if loop.last and add_generation_prompt and message['role'] != 'assistant' %}{{ '<｜Assistant｜>' }}{% endif %}{% endfor %}",
158-
"system_prompt": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoid mixing languages.",
158+
"start_message": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoid mixing languages.",
159159
},
160160
"DeepSeek-R1-Distill-Llama-8B": {
161161
"model_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
162162
"genai_chat_template": "{% for message in messages %}{% if loop.first %}{{ '<｜begin▁of▁sentence｜>' }}{% endif %}{% if message['role'] == 'system' and message['content'] %}{{ message['content'] }}{% elif message['role'] == 'user' %}{{ '<｜User｜>' + message['content'] }}{% elif message['role'] == 'assistant' %}{{ '<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>' }}{% endif %}{% if loop.last and add_generation_prompt and message['role'] != 'assistant' %}{{ '<｜Assistant｜>' }}{% endif %}{% endfor %}",
163-
"system_prompt": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoid mixing languages.",
163+
"start_message": DEFAULT_SYSTEM_PROMPT + "Think briefly and provide informative answers, avoid mixing languages.",
164164
},
165165
"llama-3.2-1b-instruct": {
166166
"model_id": "meta-llama/Llama-3.2-1B-Instruct",
167167
"start_message": DEFAULT_SYSTEM_PROMPT,
168168
"stop_tokens": ["<|eot_id|>"],
169169
"has_chat_template": True,
170-
"start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
171170
"history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
172171
"current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
173172
"rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
@@ -187,7 +186,6 @@ def qwen_completion_to_prompt(completion):
187186
"start_message": DEFAULT_SYSTEM_PROMPT,
188187
"stop_tokens": ["<|eot_id|>"],
189188
"has_chat_template": True,
190-
"start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
191189
"history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
192190
"current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
193191
"rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
@@ -212,7 +210,7 @@ def qwen_completion_to_prompt(completion):
212210
"gemma-2b-it": {
213211
"model_id": "google/gemma-2b-it",
214212
"remote_code": False,
215-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
213+
"start_message": DEFAULT_SYSTEM_PROMPT,
216214
"history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
217215
"current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
218216
"rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
@@ -221,7 +219,7 @@ def qwen_completion_to_prompt(completion):
221219
"gemma-2-2b-it": {
222220
"model_id": "google/gemma-2-2b-it",
223221
"remote_code": False,
224-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
222+
"start_message": DEFAULT_SYSTEM_PROMPT,
225223
"history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
226224
"current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
227225
"rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
@@ -244,7 +242,7 @@ def qwen_completion_to_prompt(completion):
244242
"qwen2.5-3b-instruct": {
245243
"model_id": "Qwen/Qwen2.5-3B-Instruct",
246244
"remote_code": False,
247-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
245+
"start_message": DEFAULT_SYSTEM_PROMPT,
248246
"rag_prompt_template": f"""<|im_start|>system
249247
{DEFAULT_RAG_PROMPT }<|im_end|>"""
250248
+ """
@@ -260,7 +258,7 @@ def qwen_completion_to_prompt(completion):
260258
"qwen2.5-7b-instruct": {
261259
"model_id": "Qwen/Qwen2.5-7B-Instruct",
262260
"remote_code": False,
263-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
261+
"start_message": DEFAULT_SYSTEM_PROMPT,
264262
"rag_prompt_template": f"""<|im_start|>system
265263
{DEFAULT_RAG_PROMPT }<|im_end|>"""
266264
+ """
@@ -275,7 +273,7 @@ def qwen_completion_to_prompt(completion):
275273
"gemma-7b-it": {
276274
"model_id": "google/gemma-7b-it",
277275
"remote_code": False,
278-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
276+
"start_message": DEFAULT_SYSTEM_PROMPT,
279277
"history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
280278
"current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
281279
"rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
@@ -284,7 +282,7 @@ def qwen_completion_to_prompt(completion):
284282
"gemma-2-9b-it": {
285283
"model_id": "google/gemma-2-9b-it",
286284
"remote_code": False,
287-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
285+
"start_message": DEFAULT_SYSTEM_PROMPT,
288286
"history_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}<end_of_turn>",
289287
"current_message_template": "<start_of_turn>user{user}<end_of_turn><start_of_turn>model{assistant}",
290288
"rag_prompt_template": f"""{DEFAULT_RAG_PROMPT},"""
@@ -293,7 +291,7 @@ def qwen_completion_to_prompt(completion):
293291
"llama-2-chat-7b": {
294292
"model_id": "meta-llama/Llama-2-7b-chat-hf",
295293
"remote_code": False,
296-
"start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
294+
"start_message": DEFAULT_SYSTEM_PROMPT,
297295
"history_template": "{user}[/INST]{assistant}</s><s>[INST]",
298296
"current_message_template": "{user} [/INST]{assistant}",
299297
"tokenizer_kwargs": {"add_special_tokens": False},
@@ -310,7 +308,6 @@ def qwen_completion_to_prompt(completion):
310308
"start_message": DEFAULT_SYSTEM_PROMPT,
311309
"stop_tokens": ["<|eot_id|>", "<|end_of_text|>"],
312310
"has_chat_template": True,
313-
"start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
314311
"history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
315312
"current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
316313
"rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
@@ -331,7 +328,6 @@ def qwen_completion_to_prompt(completion):
331328
"start_message": DEFAULT_SYSTEM_PROMPT,
332329
"stop_tokens": ["<|eot_id|>", "<|end_of_text|>"],
333330
"has_chat_template": True,
334-
"start_message": " <|start_header_id|>system<|end_header_id|>\n\n" + DEFAULT_SYSTEM_PROMPT + "<|eot_id|>",
335331
"history_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}<|eot_id|>",
336332
"current_message_template": "<|start_header_id|>user<|end_header_id|>\n\n{user}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n{assistant}",
337333
"rag_prompt_template": f"<|start_header_id|>system<|end_header_id|>\n\n{DEFAULT_RAG_PROMPT}<|eot_id|>"
@@ -349,7 +345,7 @@ def qwen_completion_to_prompt(completion):
349345
"mistral-7b-instruct": {
350346
"model_id": "mistralai/Mistral-7B-Instruct-v0.1",
351347
"remote_code": False,
352-
"start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
348+
"start_message": DEFAULT_SYSTEM_PROMPT,
353349
"history_template": "{user}[/INST]{assistant}</s><s>[INST]",
354350
"current_message_template": "{user} [/INST]{assistant}",
355351
"tokenizer_kwargs": {"add_special_tokens": False},
@@ -363,7 +359,7 @@ def qwen_completion_to_prompt(completion):
363359
"mistral-7B-Instruct-v0.3": {
364360
"model_id": "mistralai/Mistral-7B-Instruct-v0.3",
365361
"remote_code": False,
366-
"start_message": f"<s>[INST] {DEFAULT_SYSTEM_PROMPT }\n\n",
362+
"start_message": DEFAULT_SYSTEM_PROMPT,
367363
"history_template": "{user}[/INST]{assistant}</s>[INST]",
368364
"current_message_template": "{user} [/INST]{assistant}</s>",
369365
"tokenizer_kwargs": {"add_special_tokens": False},
@@ -373,7 +369,7 @@ def qwen_completion_to_prompt(completion):
373369
"zephyr-7b-beta": {
374370
"model_id": "HuggingFaceH4/zephyr-7b-beta",
375371
"remote_code": False,
376-
"start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
372+
"start_message": DEFAULT_SYSTEM_PROMPT,
377373
"history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
378374
"current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
379375
"rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
@@ -387,7 +383,7 @@ def qwen_completion_to_prompt(completion):
387383
"notus-7b-v1": {
388384
"model_id": "argilla/notus-7b-v1",
389385
"remote_code": False,
390-
"start_message": f"<|system|>\n{DEFAULT_SYSTEM_PROMPT}</s>\n",
386+
"start_message": DEFAULT_SYSTEM_PROMPT,
391387
"history_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}</s> \n",
392388
"current_message_template": "<|user|>\n{user}</s> \n<|assistant|>\n{assistant}",
393389
"rag_prompt_template": f"""<|system|> {DEFAULT_RAG_PROMPT }</s>"""
@@ -401,7 +397,7 @@ def qwen_completion_to_prompt(completion):
401397
"neural-chat-7b-v3-3": {
402398
"model_id": "Intel/neural-chat-7b-v3-3",
403399
"remote_code": False,
404-
"start_message": f"<s>[INST] <<SYS>>\n{DEFAULT_SYSTEM_PROMPT }\n<</SYS>>\n\n",
400+
"start_message": DEFAULT_SYSTEM_PROMPT,
405401
"history_template": "{user}[/INST]{assistant}</s><s>[INST]",
406402
"current_message_template": "{user} [/INST]{assistant}",
407403
"tokenizer_kwargs": {"add_special_tokens": False},
@@ -415,7 +411,7 @@ def qwen_completion_to_prompt(completion):
415411
"phi-3-mini-instruct": {
416412
"model_id": "microsoft/Phi-3-mini-4k-instruct",
417413
"remote_code": True,
418-
"start_message": "<|system|>\n{DEFAULT_SYSTEM_PROMPT}<|end|>\n",
414+
"start_message": DEFAULT_SYSTEM_PROMPT,
419415
"history_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}<|end|>\n",
420416
"current_message_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}",
421417
"stop_tokens": ["<|end|>"],
@@ -431,7 +427,7 @@ def qwen_completion_to_prompt(completion):
431427
"phi-3.5-mini-instruct": {
432428
"model_id": "microsoft/Phi-3.5-mini-instruct",
433429
"remote_code": True,
434-
"start_message": "<|system|>\n{DEFAULT_SYSTEM_PROMPT}<|end|>\n",
430+
"start_message": DEFAULT_SYSTEM_PROMPT,
435431
"history_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}<|end|>\n",
436432
"current_message_template": "<|user|>\n{user}<|end|> \n<|assistant|>\n{assistant}",
437433
"stop_tokens": ["<|end|>"],
@@ -451,7 +447,7 @@ def qwen_completion_to_prompt(completion):
451447
"qwen2.5-14b-instruct": {
452448
"model_id": "Qwen/Qwen2.5-14B-Instruct",
453449
"remote_code": False,
454-
"start_message": DEFAULT_SYSTEM_PROMPT + ", ",
450+
"start_message": DEFAULT_SYSTEM_PROMPT,
455451
"rag_prompt_template": f"""<|im_start|>system
456452
{DEFAULT_RAG_PROMPT }<|im_end|>"""
457453
+ """
@@ -566,7 +562,7 @@ def qwen_completion_to_prompt(completion):
566562
"qwen-7b-chat": {
567563
"model_id": "Qwen/Qwen-7B-Chat",
568564
"remote_code": True,
569-
"start_message": f"<|im_start|>system\n {DEFAULT_SYSTEM_PROMPT_CHINESE }<|im_end|>",
565+
"start_message": DEFAULT_SYSTEM_PROMPT_CHINESE,
570566
"history_template": "<|im_start|>user\n{user}<im_end><|im_start|>assistant\n{assistant}<|im_end|>",
571567
"current_message_template": '"<|im_start|>user\n{user}<im_end><|im_start|>assistant\n{assistant}',
572568
"stop_tokens": ["<|im_end|>", "<|endoftext|>"],

0 commit comments

Comments
 (0)