Feature/bump mlx (#75)

# *Update MLX Version*

## ⚙️ Release Notes 
**Breaking Changes**: This pull request introduces additional breaking
changes.
Version Updates:
- mlx-swift has been updated to version 0.18.1.
- mlx-swift-examples has been updated to version 1.18.1.
- `LLMLocalSchema` can now be update with other parameters.
- The `ChatTemplate` used by `swift-transformers` can be overwritten.
- The `LLMContext` can be overwritten with a custom context (`[[String:
String]]`).

## 📝 Code of Conduct & Contributing Guidelines 

By submitting creating this pull request, you agree to follow our [Code
of
Conduct](https://github.com/StanfordSpezi/.github/blob/main/CODE_OF_CONDUCT.md)
and [Contributing
Guidelines](https://github.com/StanfordSpezi/.github/blob/main/CONTRIBUTING.md):
- [x] I agree to follow the [Code of
Conduct](https://github.com/StanfordSpezi/.github/blob/main/CODE_OF_CONDUCT.md)
and [Contributing
Guidelines](https://github.com/StanfordSpezi/.github/blob/main/CONTRIBUTING.md).

---------

Co-authored-by: Leon Nissen <>
Co-authored-by: Paul Schmiedmayer <[email protected]>

Package.swift

@@ -27,12 +27,12 @@ let package = Package(
         .library(name: "SpeziLLMFog", targets: ["SpeziLLMFog"])
     ],
     dependencies: [
-        .package(url: "https://github.com/ml-explore/mlx-swift", branch: "0.21.2"),   // Pin MLX library as it doesn't follow semantic versioning
-        .package(url: "https://github.com/ml-explore/mlx-swift-examples", branch: "1.16.0"),  // Pin MLX library as it doesn't follow semantic versioning
-        .package(url: "https://github.com/huggingface/swift-transformers", .upToNextMinor(from: "0.1.12")),
+        .package(url: "https://github.com/ml-explore/mlx-swift", .upToNextMinor(from: "0.21.2")),
+        .package(url: "https://github.com/ml-explore/mlx-swift-examples", exact: "1.18.1"),  // Pin MLX Swift Examples as it doesn't follow semantic versioning
+        .package(url: "https://github.com/huggingface/swift-transformers", .upToNextMinor(from: "0.1.14")),
         .package(url: "https://github.com/StanfordBDHG/OpenAI", .upToNextMinor(from: "0.2.9")),
         .package(url: "https://github.com/StanfordSpezi/Spezi", from: "1.2.1"),
-        .package(url: "https://github.com/StanfordSpezi/SpeziFoundation", from: "2.0.0-beta.3"),
+        .package(url: "https://github.com/StanfordSpezi/SpeziFoundation", from: "2.0.0"),
         .package(url: "https://github.com/StanfordSpezi/SpeziStorage", from: "1.0.2"),
         .package(url: "https://github.com/StanfordSpezi/SpeziOnboarding", from: "1.1.1"),
         .package(url: "https://github.com/StanfordSpezi/SpeziChat", .upToNextMinor(from: "0.2.1")),

Sources/SpeziLLM/Models/LLMContextEntity.swift

@@ -46,7 +46,7 @@ public struct LLMContextEntity: Codable, Equatable, Hashable, Identifiable {
         case tool(id: String, name: String)
 
 
-        var rawValue: String {
+        package var rawValue: String {
             switch self {
             case .user: "user"
             case .assistant: "assistant"

Sources/SpeziLLMLocal/Configuration/LLMLocalParameters.swift

@@ -23,10 +23,15 @@ public struct LLMLocalParameters: Sendable {
     let systemPrompt: String?
     /// Indicates the maximum output length generated by the LLM.
     let maxOutputLength: Int
-
+    /// Additional End of Sequence tokens at which the generation will be stoped.
     let extraEOSTokens: Set<String>
     /// Interval for displaying output after every N tokens generated.
     let displayEveryNTokens: Int
+    /// RNG seed of the LLM
+    let seed: UInt64?
+    /// The chat template to use for the model in the Jinja format
+    let chatTemplate: String?
+
 
     /// Creates the ``LLMLocalParameters`` which wrap the underlying llama.cpp `llama_model_params` C struct.
     /// Is passed to the underlying llama.cpp model in order to configure the LLM.
@@ -36,15 +41,21 @@ public struct LLMLocalParameters: Sendable {
     ///   - maxOutputLength: The maximum output length generated by the Spezi LLM, defaults to `512`.
     ///   - extraEOSTokens: Additional tokens to use for end of string
     ///   - displayEveryNTokens: Interval for displaying output after every N tokens generated, defaults to `4`.
+    ///   - seed: RNG seed of the LLM, defaults to a random seed.
+    ///   - chatTemplate: Can be set to manually overwrite the chatTemplate within the `swift-transformers` package.
     public init(
         systemPrompt: String? = Defaults.defaultSystemPrompt,
         maxOutputLength: Int = 512,
         extraEOSTokens: Set<String> = [],
-        displayEveryNTokens: Int = 4
+        displayEveryNTokens: Int = 4,
+        seed: UInt64? = nil,
+        chatTemplate: String? = nil
     ) {
         self.systemPrompt = systemPrompt
         self.maxOutputLength = maxOutputLength
         self.extraEOSTokens = extraEOSTokens
         self.displayEveryNTokens = displayEveryNTokens
+        self.seed = seed
+        self.chatTemplate = chatTemplate
     }
 }

Sources/SpeziLLMLocal/Helpers/LLMContext+FormattedChat.swift

@@ -0,0 +1,25 @@
+//
+// This source file is part of the Stanford Spezi open source project
+//
+// SPDX-FileCopyrightText: 2024 Stanford University and the project authors (see CONTRIBUTORS.md)
+//
+// SPDX-License-Identifier: MIT
+//
+
+import SpeziLLM
+
+extension LLMContext {
+    /// Formats the current ``LLMContext`` for compatibility with Transformers-based chat models.
+    ///
+    /// - Returns: An array of dictionaries where each dictionary represents a message in the format:
+    ///   - `role`: The role of the message (e.g., "user", "assistant"), derived from the `rawValue` of the entry's `role`.
+    ///   - `content`: The textual content of the message.
+    package var formattedChat: [[String: String]] {
+        self.map { entry in
+            [
+                "role": entry.role.rawValue,
+                "content": entry.content
+            ]
+        }
+    }
+}

Sources/SpeziLLMLocal/LLMLocalPlatform.swift

@@ -11,7 +11,9 @@ import MLX
 import Spezi
 import SpeziFoundation
 import SpeziLLM
-
+#if targetEnvironment(simulator)
+import OSLog
+#endif
 
 /// LLM execution platform of an ``LLMLocalSchema``.
 ///
@@ -58,19 +60,29 @@ public actor LLMLocalPlatform: LLMPlatform, DefaultInitializable {
 
     public nonisolated func configure() {
 #if targetEnvironment(simulator)
-        assertionFailure("SpeziLLMLocal: Code cannot be run on simulator.")
-#endif
+        Logger(
+            subsystem: "Spezi",
+            category: "LLMLocalPlatform"
+        ).warning("SpeziLLMLocal is only supported on physical devices. Use `LLMMockPlatform` instead.")
+#else
         if let cacheLimit = configuration.cacheLimit {
             MLX.GPU.set(cacheLimit: cacheLimit * 1024 * 1024)
         }
         if let memoryLimit = configuration.memoryLimit {
             MLX.GPU.set(memoryLimit: memoryLimit.limit, relaxed: memoryLimit.relaxed)
         }
+#endif
     }
 
+#if targetEnvironment(simulator)
+    public nonisolated func callAsFunction(with llmSchema: LLMLocalSchema) -> LLMLocalMockSession {
+        LLMLocalMockSession(self, schema: llmSchema)
+    }
+#else
     public nonisolated func callAsFunction(with llmSchema: LLMLocalSchema) -> LLMLocalSession {
         LLMLocalSession(self, schema: llmSchema)
     }
+#endif
 
     deinit {
         MLX.GPU.clearCache()