fix: use gpt-4o tokenizer for apertus model

nikochiko · nikochiko · commit 823a27933f3b · 2025-09-11T20:32:11.000+05:30
diff --git a/daras_ai_v2/text_splitter.py b/daras_ai_v2/text_splitter.py
@@ -51,6 +51,7 @@ def default_length_function(text: str, model: str = "gpt-4") -> int:
             "claude-",
             "google/",
             "aisingapore/",
+            "swiss-ai/",
         ]
     ):
         model = "gpt-4o"

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ def default_length_function(text: str, model: str = "gpt-4") -> int:`
`51`	`51`	`"claude-",`
`52`	`52`	`"google/",`
`53`	`53`	`"aisingapore/",`
	`54`	`+ "swiss-ai/",`
`54`	`55`	`]`
`55`	`56`	`):`
`56`	`57`	`model = "gpt-4o"`