Skip to content

Commit 466d05c

Browse files
filipwfilipw
authored
Merge pull request #35 from filipw/feature/0.8.3
updated to candle 0.8.3 and added support for Phi-4
2 parents 6310c7c + b03c28a commit 466d05c

File tree

10 files changed

+1179
-495
lines changed

10 files changed

+1179
-495
lines changed

‎.github/workflows/ci.yml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
3030
- name: Upload default artifacts
3131
if: github.event_name != 'pull_request'
32-
uses: actions/upload-artifact@v3
32+
uses: actions/upload-artifact@v4
3333
with:
3434
name: native-library-${{ matrix.os }}
3535
path: |
@@ -39,7 +39,7 @@ jobs:
3939
4040
- name: Upload bindings
4141
if: matrix.os == 'ubuntu-latest' && github.event_name != 'pull_request'
42-
uses: actions/upload-artifact@v3
42+
uses: actions/upload-artifact@v4
4343
with:
4444
name: language-bindings
4545
path: |
@@ -63,7 +63,7 @@ jobs:
6363
run: rm -rf packages/swift/Strathweb.Phi.Engine/.build
6464
- name: Upload artifact
6565
if: github.event_name != 'pull_request'
66-
uses: actions/upload-artifact@v3
66+
uses: actions/upload-artifact@v4
6767
with:
6868
name: Strathweb.Phi.Engine-SwiftPackage
6969
path: packages/swift/Strathweb.Phi.Engine
@@ -79,22 +79,22 @@ jobs:
7979
with:
8080
dotnet-version: '9.0.x'
8181
- name: Download Linux artifacts
82-
uses: actions/download-artifact@v3
82+
uses: actions/download-artifact@v4
8383
with:
8484
name: native-library-ubuntu-latest
8585
path: artifacts/linux
8686
- name: Download macOS artifacts
87-
uses: actions/download-artifact@v3
87+
uses: actions/download-artifact@v4
8888
with:
8989
name: native-library-macos-15
9090
path: artifacts/macos
9191
- name: Download Windows artifacts
92-
uses: actions/download-artifact@v3
92+
uses: actions/download-artifact@v4
9393
with:
9494
name: native-library-windows-latest
9595
path: artifacts/windows
9696
- name: Download bindings
97-
uses: actions/download-artifact@v3
97+
uses: actions/download-artifact@v4
9898
with:
9999
name: language-bindings
100100
path: artifacts/bindings
@@ -113,17 +113,17 @@ jobs:
113113
dotnet build -c Release
114114
dotnet pack -c Release
115115
- name: Upload NuGet package (Strathweb.Phi.Engine)
116-
uses: actions/upload-artifact@v3
116+
uses: actions/upload-artifact@v4
117117
with:
118118
name: Strathweb.Phi.Engine
119119
path: packages/csharp/Strathweb.Phi.Engine/bin/Release/*.nupkg
120120
- name: Upload NuGet package (Strathweb.Phi.Engine.AutoGen)
121-
uses: actions/upload-artifact@v3
121+
uses: actions/upload-artifact@v4
122122
with:
123123
name: Strathweb.Phi.Engine.AutoGen
124124
path: packages/csharp/Strathweb.Phi.Engine.AutoGen/bin/Release/*.nupkg
125125
- name: Upload NuGet package (Strathweb.Phi.Engine.Microsoft.Extensions.AI)
126-
uses: actions/upload-artifact@v3
126+
uses: actions/upload-artifact@v4
127127
with:
128128
name: Strathweb.Phi.Engine.Microsoft.Extensions.AI
129129
path: packages/csharp/Strathweb.Phi.Engine.Microsoft.Extensions.AI/bin/Release/*.nupkg

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Strathweb Phi Engine
22

3-
A cross-platform library for running Microsoft's [Phi-3](https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/) locally using [candle](https://github.com/huggingface/candle) in GGUF (quantized) and safe tensors (full models) format.
3+
A cross-platform library for running Microsoft's [Phi-3](https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/) locally using [candle](https://github.com/huggingface/candle) in GGUF and safe tensors format.
44

55
## Supported platforms
66

‎samples/swift/main.swift

Lines changed: 67 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,100 @@
11
import Foundation
22

3-
let isQuantizedMode = CommandLine.arguments.contains("--quantized")
3+
let (isGgufMode, isPhi4, isCpuMode) = (
4+
CommandLine.arguments.contains("--gguf"),
5+
CommandLine.arguments.contains("--phi-4"),
6+
CommandLine.arguments.contains("--cpu")
7+
)
48

5-
if isQuantizedMode {
6-
print(" 🍃 Quantized mode is enabled.")
7-
} else {
8-
print(" 💪 Safe tensors mode is enabled.")
9-
}
9+
let formatMode = isGgufMode ? " 🍃 GGUF" : " 💪 Safe tensors"
10+
let modelMode = isPhi4 ? " 🚀 Phi-4" : " 🚗 Phi-3"
1011

11-
let modelProvider = isQuantizedMode ?
12-
PhiModelProvider.huggingFaceGguf(modelRepo: "microsoft/Phi-3-mini-4k-instruct-gguf", modelFileName: "Phi-3-mini-4k-instruct-q4.gguf", modelRevision: "main") :
13-
PhiModelProvider.huggingFace(modelRepo: "microsoft/Phi-3-mini-4k-instruct", modelRevision: "main")
12+
print("\(formatMode) mode is enabled.\n\(modelMode) mode is enabled.")
13+
14+
let modelProvider = switch (isGgufMode, isPhi4) {
15+
case (true, true):
16+
PhiModelProvider.huggingFaceGguf(
17+
modelRepo: "microsoft/phi-4-gguf",
18+
modelFileName: "phi-4-q4.gguf",
19+
modelRevision: "main"
20+
)
21+
case (true, false):
22+
PhiModelProvider.huggingFaceGguf(
23+
modelRepo: "microsoft/Phi-3-mini-4k-instruct-gguf",
24+
modelFileName: "Phi-3-mini-4k-instruct-q4.gguf",
25+
modelRevision: "main"
26+
)
27+
case (false, true):
28+
PhiModelProvider.huggingFace(
29+
modelRepo: "microsoft/phi-4",
30+
modelRevision: "main"
31+
)
32+
case (false, false):
33+
PhiModelProvider.huggingFace(
34+
modelRepo: "microsoft/Phi-3-mini-4k-instruct",
35+
modelRevision: "main"
36+
)
37+
}
1438

1539
let inferenceOptionsBuilder = InferenceOptionsBuilder()
1640
try! inferenceOptionsBuilder.withTemperature(temperature: 0.9)
1741
try! inferenceOptionsBuilder.withSeed(seed: 146628346)
42+
if isPhi4 {
43+
try! inferenceOptionsBuilder.withChatFormat(chatFormat: ChatFormat.chatMl)
44+
}
1845
let inferenceOptions = try! inferenceOptionsBuilder.build()
1946

2047
let cacheDir = FileManager.default.currentDirectoryPath.appending("/.cache")
2148

2249
class ModelEventsHandler: PhiEventHandler {
23-
func onInferenceStarted() {}
24-
25-
func onInferenceEnded() {}
26-
50+
func onInferenceStarted() {
51+
print(" ℹ️ Inference started...")
52+
}
53+
func onInferenceEnded() {
54+
print("\n ℹ️ Inference ended.")
55+
}
2756
func onInferenceToken(token: String) {
2857
print(token, terminator: "")
2958
}
30-
3159
func onModelLoaded() {
3260
print("""
33-
🧠 Model loaded!
34-
****************************************
35-
""")
61+
🧠 Model loaded!
62+
****************************************
63+
""")
3664
}
3765
}
3866

3967
let modelBuilder = PhiEngineBuilder()
4068
try! modelBuilder.withEventHandler(eventHandler: BoxedPhiEventHandler(handler: ModelEventsHandler()))
41-
let gpuEnabled = try! modelBuilder.tryUseGpu()
4269
try! modelBuilder.withModelProvider(modelProvider: modelProvider)
4370

44-
let model = try! modelBuilder.buildStateful(cacheDir: cacheDir, systemInstruction: "You are a hockey poet. Be brief and polite.")
71+
if isPhi4 {
72+
try! modelBuilder.withTokenizerProvider(tokenizerProvider: .huggingFace(
73+
tokenizerRepo: "microsoft/phi-4",
74+
tokenizerFileName: "tokenizer.json"
75+
))
76+
}
77+
78+
if !isCpuMode {
79+
let gpuEnabled = try! modelBuilder.tryUseGpu()
80+
print(gpuEnabled ? " 🎮 GPU mode enabled." : " 💻 Tried GPU, but falling back to CPU.")
81+
} else {
82+
print(" 💻 CPU mode enabled.")
83+
}
4584

46-
// Run inference
47-
let result = try! model.runInference(promptText: "Write a haiku about ice hockey", inferenceOptions: inferenceOptions)
85+
let model = try! modelBuilder.buildStateful(
86+
cacheDir: cacheDir,
87+
systemInstruction: "You are a hockey poet. Be brief and polite."
88+
)
4889

49-
print("""
90+
let result = try! model.runInference(
91+
promptText: "Write a haiku about ice hockey",
92+
inferenceOptions: inferenceOptions
93+
)
5094

95+
print("""
5196
****************************************
5297
📝 Tokens Generated: \(result.tokenCount)
5398
🖥️ Tokens per second: \(result.tokensPerSecond)
5499
⏱️ Duration: \(result.duration)s
55-
🏎️ GPU enabled: \(gpuEnabled)
56100
""")

‎samples/swift/run.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@ swiftc *.swift \
1515
-framework MetalPerformanceShaders \
1616
-framework SystemConfiguration \
1717
-lc++ \
18-
-O \
18+
-O -whole-module-optimization \
19+
-cross-module-optimization \
20+
-enforce-exclusivity=unchecked \
1921
-o phi-engine-swift-sample
2022

2123
./phi-engine-swift-sample "$@"

0 commit comments

Comments
 (0)