Skip to content

Commit f79a4c5

Browse files
authored
align default int4 configuration (#2963)
1 parent 4308704 commit f79a4c5

File tree

1 file changed

+7
-3
lines changed

1 file changed

+7
-3
lines changed

notebooks/llm-chatbot/llm-chatbot.ipynb

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -723,8 +723,6 @@
723723
" \"qwen2.5-0.5b-instruct\": {\"sym\": True, \"group_size\": 128, \"ratio\": 1.0},\n",
724724
" \"default\": {\n",
725725
" \"sym\": False,\n",
726-
" \"group_size\": 128,\n",
727-
" \"ratio\": 0.8,\n",
728726
" },\n",
729727
" }\n",
730728
"\n",
@@ -733,7 +731,13 @@
733731
" return\n",
734732
" remote_code = model_configuration.get(\"remote_code\", False)\n",
735733
" export_command_base = \"optimum-cli export openvino --model {} --task text-generation-with-past --weight-format int4\".format(pt_model_id)\n",
736-
" int4_compression_args = \" --group-size {} --ratio {}\".format(model_compression_params[\"group_size\"], model_compression_params[\"ratio\"])\n",
734+
" group_size = model_compression_params.get(\"group_size\")\n",
735+
" ratio = model_compression_params.get(\"ratio\")\n",
736+
" int4_compression_args = \"\"\n",
737+
" if group_size is not None:\n",
738+
" int4_compression_args += \" --group-size {}\".format(group_size)\n",
739+
" if ratio is not None:\n",
740+
" int4_compression_args += \" --ratio {}\".format(ratio)\n",
737741
" if model_compression_params[\"sym\"]:\n",
738742
" int4_compression_args += \" --sym\"\n",
739743
" if enable_awq.value:\n",

0 commit comments

Comments
 (0)