FIX: switch off native tools on Anthropic Claude Opus (#659)

Native tools do not work well on Opus.

Chain of Thought prompting means it consumes enormous amounts of
tokens and has poor latency.

This commit introduce and XML stripper to remove various chain of
thought XML islands from anthropic prompts when tools are involved.

This mean Opus native tools is now functions (albeit slowly)

From local testing XML just works better now.

Also fixes enum support in Anthropic native tools

Author: SamSaffron

config/locales/server.en.yml

@@ -51,6 +51,7 @@ en:
     ai_openai_embeddings_url: "Custom URL used for the OpenAI embeddings API. (in the case of Azure it can be: https://COMPANY.openai.azure.com/openai/deployments/DEPLOYMENT/embeddings?api-version=2023-05-15)"
     ai_openai_api_key: "API key for OpenAI API"
     ai_anthropic_api_key: "API key for Anthropic API"
+    ai_anthropic_native_tool_call_models: "List of models that will use native tool calls vs legacy XML based tools."
     ai_cohere_api_key: "API key for Cohere API"
     ai_hugging_face_api_url: "Custom URL used for OpenSource LLM inference. Compatible with https://github.com/huggingface/text-generation-inference"
     ai_hugging_face_api_key: API key for Hugging Face API

config/settings.yml

@@ -111,6 +111,15 @@ discourse_ai:
   ai_anthropic_api_key:
     default: ""
     secret: true
+  ai_anthropic_native_tool_call_models:
+    type: list
+    list_type: compact
+    default: "claude-3-sonnet|claude-3-haiku"
+    allow_any: false
+    choices:
+      - claude-3-opus
+      - claude-3-sonnet
+      - claude-3-haiku
   ai_cohere_api_key:
     default: ""
     secret: true

lib/completions/dialects/claude.rb

@@ -22,6 +22,10 @@ def initialize(system_prompt, messages, tools)
             @messages = messages
             @tools = tools
           end
+
+          def has_tools?
+            tools.present?
+          end
         end
 
         def tokenizer
@@ -33,6 +37,10 @@ def translate
 
           system_prompt = messages.shift[:content] if messages.first[:role] == "system"
 
+          if !system_prompt && !native_tool_support?
+            system_prompt = tools_dialect.instructions.presence
+          end
+
           interleving_messages = []
           previous_message = nil
 
@@ -48,11 +56,10 @@ def translate
             previous_message = message
           end
 
-          ClaudePrompt.new(
-            system_prompt.presence,
-            interleving_messages,
-            tools_dialect.translated_tools,
-          )
+          tools = nil
+          tools = tools_dialect.translated_tools if native_tool_support?
+
+          ClaudePrompt.new(system_prompt.presence, interleving_messages, tools)
         end
 
         def max_prompt_tokens
@@ -62,18 +69,28 @@ def max_prompt_tokens
           200_000 # Claude-3 has a 200k context window for now
         end
 
+        def native_tool_support?
+          SiteSetting.ai_anthropic_native_tool_call_models_map.include?(model_name)
+        end
+
         private
 
         def tools_dialect
-          @tools_dialect ||= DiscourseAi::Completions::Dialects::ClaudeTools.new(prompt.tools)
+          if native_tool_support?
+            @tools_dialect ||= DiscourseAi::Completions::Dialects::ClaudeTools.new(prompt.tools)
+          else
+            super
+          end
         end
 
         def tool_call_msg(msg)
-          tools_dialect.from_raw_tool_call(msg)
+          translated = tools_dialect.from_raw_tool_call(msg)
+          { role: "assistant", content: translated }
         end
 
         def tool_msg(msg)
-          tools_dialect.from_raw_tool(msg)
+          translated = tools_dialect.from_raw_tool(msg)
+          { role: "user", content: translated }
         end
 
         def model_msg(msg)

lib/completions/dialects/claude_tools.rb

@@ -15,13 +15,14 @@ def translated_tools
             required = []
 
             if t[:parameters]
-              properties =
-                t[:parameters].each_with_object({}) do |param, h|
-                  h[param[:name]] = {
-                    type: param[:type],
-                    description: param[:description],
-                  }.tap { |hash| hash[:items] = { type: param[:item_type] } if param[:item_type] }
-                end
+              properties = {}
+
+              t[:parameters].each do |param|
+                mapped = { type: param[:type], description: param[:description] }
+                mapped[:items] = { type: param[:item_type] } if param[:item_type]
+                mapped[:enum] = param[:enum] if param[:enum]
+                properties[param[:name]] = mapped
+              end
               required =
                 t[:parameters].select { |param| param[:required] }.map { |param| param[:name] }
             end
@@ -39,37 +40,24 @@ def translated_tools
         end
 
         def instructions
-          "" # Noop. Tools are listed separate.
+          ""
         end
 
         def from_raw_tool_call(raw_message)
           call_details = JSON.parse(raw_message[:content], symbolize_names: true)
           tool_call_id = raw_message[:id]
-
-          {
-            role: "assistant",
-            content: [
-              {
-                type: "tool_use",
-                id: tool_call_id,
-                name: raw_message[:name],
-                input: call_details[:arguments],
-              },
-            ],
-          }
+          [
+            {
+              type: "tool_use",
+              id: tool_call_id,
+              name: raw_message[:name],
+              input: call_details[:arguments],
+            },
+          ]
         end
 
         def from_raw_tool(raw_message)
-          {
-            role: "user",
-            content: [
-              {
-                type: "tool_result",
-                tool_use_id: raw_message[:id],
-                content: raw_message[:content],
-              },
-            ],
-          }
+          [{ type: "tool_result", tool_use_id: raw_message[:id], content: raw_message[:content] }]
         end
 
         private

lib/completions/dialects/xml_tools.rb

@@ -41,13 +41,17 @@ def translated_tools
         def instructions
           return "" if raw_tools.blank?
 
-          has_arrays = raw_tools.any? { |tool| tool[:parameters]&.any? { |p| p[:type] == "array" } }
-
-          (<<~TEXT).strip
-            #{tool_preamble(include_array_tip: has_arrays)}
-            <tools>
-            #{translated_tools}</tools>
-          TEXT
+          @instructions ||=
+            begin
+              has_arrays =
+                raw_tools.any? { |tool| tool[:parameters]&.any? { |p| p[:type] == "array" } }
+
+              (<<~TEXT).strip
+              #{tool_preamble(include_array_tip: has_arrays)}
+              <tools>
+              #{translated_tools}</tools>
+            TEXT
+            end
         end
 
         def from_raw_tool(raw_message)

lib/completions/endpoints/anthropic.rb

@@ -45,7 +45,12 @@ def default_options(dialect)
               raise "Unsupported model: #{model}"
             end
 
-          { model: mapped_model, max_tokens: 3_000 }
+          options = { model: mapped_model, max_tokens: 3_000 }
+
+          options[:stop_sequences] = ["</function_calls>"] if !dialect.native_tool_support? &&
+            dialect.prompt.has_tools?
+
+          options
         end
 
         def provider_id
@@ -54,6 +59,14 @@ def provider_id
 
         private
 
+        def xml_tags_to_strip(dialect)
+          if dialect.prompt.has_tools?
+            %w[thinking search_quality_reflection search_quality_score]
+          else
+            []
+          end
+        end
+
         # this is an approximation, we will update it later if request goes through
         def prompt_size(prompt)
           tokenizer.size(prompt.system_prompt.to_s + " " + prompt.messages.to_s)
@@ -66,11 +79,13 @@ def model_uri
         end
 
         def prepare_payload(prompt, model_params, dialect)
+          @native_tool_support = dialect.native_tool_support?
+
           payload = default_options(dialect).merge(model_params).merge(messages: prompt.messages)
 
           payload[:system] = prompt.system_prompt if prompt.system_prompt.present?
           payload[:stream] = true if @streaming_mode
-          payload[:tools] = prompt.tools if prompt.tools.present?
+          payload[:tools] = prompt.tools if prompt.has_tools?
 
           payload
         end
@@ -108,7 +123,7 @@ def final_log_update(log)
         end
 
         def native_tool_support?
-          true
+          @native_tool_support
         end
 
         def partials_from(decoded_chunk)

lib/completions/endpoints/aws_bedrock.rb

@@ -36,13 +36,24 @@ def normalize_model_params(model_params)
 
         def default_options(dialect)
           options = { max_tokens: 3_000, anthropic_version: "bedrock-2023-05-31" }
+
+          options[:stop_sequences] = ["</function_calls>"] if !dialect.native_tool_support? &&
+            dialect.prompt.has_tools?
           options
         end
 
         def provider_id
           AiApiAuditLog::Provider::Anthropic
         end
 
+        def xml_tags_to_strip(dialect)
+          if dialect.prompt.has_tools?
+            %w[thinking search_quality_reflection search_quality_score]
+          else
+            []
+          end
+        end
+
         private
 
         def prompt_size(prompt)
@@ -79,9 +90,11 @@ def model_uri
         end
 
         def prepare_payload(prompt, model_params, dialect)
+          @native_tool_support = dialect.native_tool_support?
+
           payload = default_options(dialect).merge(model_params).merge(messages: prompt.messages)
           payload[:system] = prompt.system_prompt if prompt.system_prompt.present?
-          payload[:tools] = prompt.tools if prompt.tools.present?
+          payload[:tools] = prompt.tools if prompt.has_tools?
 
           payload
         end
@@ -169,7 +182,7 @@ def partials_from(decoded_chunks)
         end
 
         def native_tool_support?
-          true
+          @native_tool_support
         end
 
         def chunk_to_string(chunk)

lib/completions/endpoints/base.rb

@@ -78,11 +78,27 @@ def use_ssl?
           end
         end
 
+        def xml_tags_to_strip(dialect)
+          []
+        end
+
         def perform_completion!(dialect, user, model_params = {}, feature_name: nil, &blk)
           allow_tools = dialect.prompt.has_tools?
           model_params = normalize_model_params(model_params)
+          orig_blk = blk
 
           @streaming_mode = block_given?
+          to_strip = xml_tags_to_strip(dialect)
+          @xml_stripper =
+            DiscourseAi::Completions::XmlTagStripper.new(to_strip) if to_strip.present?
+
+          if @streaming_mode && @xml_stripper
+            blk =
+              lambda do |partial, cancel|
+                partial = @xml_stripper << partial
+                orig_blk.call(partial, cancel) if partial
+              end
+          end
 
           prompt = dialect.translate
 
@@ -270,6 +286,11 @@ def perform_completion!(dialect, user, model_params = {}, feature_name: nil, &bl
                 blk.call(function_calls, cancel)
               end
 
+              if @xml_stripper
+                leftover = @xml_stripper.finish
+                orig_blk.call(leftover, cancel) if leftover.present?
+              end
+
               return response_data
             ensure
               if log