From e5c191d71458bbf76b72fa4da1af36e426bba650 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 14 Apr 2024 22:57:40 -0400 Subject: [PATCH 001/105] wip --- include/swift/AST/PluginLoader.h | 2 +- lib/AST/PluginLoader.cpp | 17 +- lib/AST/PluginRegistry.cpp | 5 +- lib/Sema/TypeCheckMacros.cpp | 3 +- tools/swift-plugin-server/Package.swift | 10 +- .../swift-plugin-server.swift | 161 +++++++++++++++++- 6 files changed, 183 insertions(+), 15 deletions(-) diff --git a/include/swift/AST/PluginLoader.h b/include/swift/AST/PluginLoader.h index f35d5adea7fb8..232f876c1a3e0 100644 --- a/include/swift/AST/PluginLoader.h +++ b/include/swift/AST/PluginLoader.h @@ -86,7 +86,7 @@ class PluginLoader { /// NOTE: This method is idempotent. If the plugin is already loaded, the same /// instance is simply returned. llvm::Expected - loadExecutablePlugin(llvm::StringRef path); + loadExecutablePlugin(llvm::StringRef path, bool forceDisableSandbox); /// Add the specified plugin associated with the module name to the dependency /// tracker if needed. diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 658b28348e1d4..c6644ef890ab9 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -106,8 +106,17 @@ PluginLoader::getPluginMap() { case PluginSearchOption::Kind::LoadPluginExecutable: { auto &val = entry.get(); assert(!val.ExecutablePath.empty() && "empty plugin path"); - for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, /*libraryPath=*/"", val.ExecutablePath); + if (llvm::sys::path::filename(val.ExecutablePath).ends_with(".wasm")) { + SmallString<128> runner; + // TODO: improve path resolution: we really want tools_dir + llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-plugin-server"); + for (auto &moduleName : val.ModuleNames) { + try_emplace(moduleName, val.ExecutablePath, runner); + } + } else { + for (auto &moduleName : val.ModuleNames) { + try_emplace(moduleName, /*libraryPath=*/"", val.ExecutablePath); + } } continue; } @@ -185,7 +194,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) { } llvm::Expected -PluginLoader::loadExecutablePlugin(StringRef path) { +PluginLoader::loadExecutablePlugin(StringRef path, bool forceDisableSandbox) { auto fs = Ctx.SourceMgr.getFileSystem(); SmallString<128> resolvedPath; if (auto err = fs->getRealPath(path, resolvedPath)) { @@ -194,7 +203,7 @@ PluginLoader::loadExecutablePlugin(StringRef path) { // Load the plugin. auto plugin = - getRegistry()->loadExecutablePlugin(resolvedPath, disableSandbox); + getRegistry()->loadExecutablePlugin(resolvedPath, disableSandbox || forceDisableSandbox); if (!plugin) { resolvedPath.push_back(0); return llvm::handleErrors( diff --git a/lib/AST/PluginRegistry.cpp b/lib/AST/PluginRegistry.cpp index 3bd5d0e870291..5b1d4af90187e 100644 --- a/lib/AST/PluginRegistry.cpp +++ b/lib/AST/PluginRegistry.cpp @@ -93,7 +93,10 @@ PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) { std::lock_guard lock(mtx); // See if the plugin is already loaded. - auto &storage = LoadedPluginExecutables[path]; + // include disableSandbox in the key so that we can have a plugin loaded + // as both sandboxed and unsandboxed at the same time. + const std::string key = (disableSandbox ? "0" : "1") + path.str(); + auto &storage = LoadedPluginExecutables[key]; if (storage) { // See if the loaded one is still usable. if (storage->getLastModificationTime() == stat.getLastModificationTime()) diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 84a0d273f150f..212fa61a58d32 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -345,8 +345,9 @@ CompilerPluginLoadRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, SmallString<0> errorMessage; if (!entry.executablePath.empty()) { + bool forceDisableSandbox = entry.libraryPath.ends_with(".wasm"); llvm::Expected executablePlugin = - loader.loadExecutablePlugin(entry.executablePath); + loader.loadExecutablePlugin(entry.executablePath, forceDisableSandbox); if (executablePlugin) { if (ctx->LangOpts.EnableMacroLoadingRemarks) { unsigned tag = entry.libraryPath.empty() ? 1 : 2; diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 5a4bb6b830ab8..7c2b3cba83d1f 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.6 +// swift-tools-version: 5.9 import PackageDescription @@ -17,11 +17,14 @@ let package = Package( cxxSettings: [ .unsafeFlags([ "-I", "../../include", + "-I", "../../stdlib/public/SwiftShims", + "-I", "../../../build/Default/swift/include", + "-I", "../../../build/Default/llvm/include", "-I", "../../../llvm-project/llvm/include", ]) ] ), - .executableTarget( + .target( name: "swift-plugin-server", dependencies: [ .product(name: "swiftLLVMJSON", package: "ASTGen"), @@ -32,7 +35,8 @@ let package = Package( .product(name: "SwiftParser", package: "swift-syntax"), .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), "CSwiftPluginServer" - ] + ], + swiftSettings: [.interoperabilityMode(.Cxx)] ), ], cxxLanguageStandard: .cxx17 diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 383a1b99491ed..28d5ab5b7a57c 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -14,6 +14,7 @@ import SwiftSyntaxMacros import swiftLLVMJSON import CSwiftPluginServer +import Foundation @main final class SwiftPluginServer { @@ -34,23 +35,37 @@ final class SwiftPluginServer { /// Loaded dylib handles associated with the module name. var loadedLibraryPlugins: [String: LoadedLibraryPlugin] = [:] + var loadedWasmPlugins: [String: WasmPlugin] = [:] + /// Resolved cached macros. var resolvedMacros: [MacroRef: Macro.Type] = [:] /// @main entry point. static func main() throws { - let connection = try PluginHostConnection() + let provider = self.init() + let connection = try PluginHostConnection(provider: provider) let messageHandler = CompilerPluginMessageHandler( connection: connection, - provider: self.init() + provider: provider ) try messageHandler.main() } } +protocol WasmPlugin { + func handleMessage(_ json: String) throws -> String +} + extension SwiftPluginServer: PluginProvider { /// Load a macro implementation from the dynamic link library. func loadPluginLibrary(libraryPath: String, moduleName: String) throws { + if libraryPath.hasSuffix(".wasm") { + guard #available(macOS 12, *) else { throw PluginServerError(message: "Wasm support requires macOS 11+") } + let wasm = try Data(contentsOf: URL(fileURLWithPath: libraryPath)) + let plugin = try WebKitWasmPlugin(wasm: wasm) + loadedWasmPlugins[moduleName] = plugin + return + } var errorMessage: UnsafePointer? guard let dlHandle = PluginServer_load(libraryPath, &errorMessage) else { throw PluginServerError(message: "loader error: " + String(cString: errorMessage!)) @@ -100,13 +115,15 @@ extension SwiftPluginServer: PluginProvider { } final class PluginHostConnection: MessageConnection { + let provider: SwiftPluginServer let handle: UnsafeRawPointer - init() throws { + init(provider: SwiftPluginServer) throws { var errorMessage: UnsafePointer? = nil guard let handle = PluginServer_createConnection(&errorMessage) else { throw PluginServerError(message: String(cString: errorMessage!)) } self.handle = handle + self.provider = provider } deinit { @@ -120,8 +137,27 @@ final class PluginHostConnection: MessageConnection { } func waitForNextMessage(_ type: RX.Type) throws -> RX? { - return try self.withReadingMessageData { jsonData in - try LLVMJSON.decode(RX.self, from: jsonData) + while true { + let result = try self.withReadingMessageData { jsonData -> RX? in + let hostMessage = try LLVMJSON.decode(HostToPluginMessage.self, from: jsonData) + switch hostMessage { + case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), + .expandFreestandingMacro(let macro, _, _, _, _): + if let plugin = provider.loadedWasmPlugins[macro.moduleName] { + var response = try plugin.handleMessage(String(decoding: UnsafeRawBufferPointer(jsonData), as: UTF8.self)) + try response.withUTF8 { + try $0.withMemoryRebound(to: Int8.self) { + try self.sendMessageData($0) + } + } + return nil + } + default: + break + } + return try LLVMJSON.decode(RX.self, from: jsonData) + } ?? nil + if let result { return result } } } @@ -220,3 +256,118 @@ struct PluginServerError: Error, CustomStringConvertible { self.description = message } } + +import WebKit + +@available(macOS 12, *) +final class WebKitWasmPlugin: WasmPlugin { + private let webView: WKWebView + + private init(_ data: Chunks) throws where Chunks.Element == Data { + let configuration = WKWebViewConfiguration() + configuration.setURLSchemeHandler(SchemeHandler { request in + let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: [ + "access-control-allow-origin": "*", + "content-type": "application/wasm" + ])! + return (data, response) + }, forURLScheme: "wasm-runner-data") + webView = WKWebView(frame: .zero, configuration: configuration) + webView.evaluateJavaScript(""" + const initPromise = (async () => { + // somehow compile(await fetch.arrayBuf) is faster than + // compileStreaming(fetch). Beats me. + const data = await (await fetch("wasm-runner-data://")).arrayBuffer(); + const mod = await WebAssembly.compile(data); + // stub WASI imports + const imports = WebAssembly.Module.imports(mod) + .filter(x => x.module === "wasi_snapshot_preview1") + .map(x => [x.name, () => {}]); + const instance = await WebAssembly.instantiate(mod, { + wasi_snapshot_preview1: Object.fromEntries(imports) + }); + api = instance.exports; + enc = new TextEncoder(); + dec = new TextDecoder(); + api._start(); + })() + """, in: nil, in: .defaultClient) + } + + package convenience init(wasm: Data) throws { + try self.init(AsyncStream { + $0.yield(wasm) + $0.finish() + }) + } + + @MainActor func handleMessage(_ json: String) throws -> String { + var res: Result? + let utf8Length = json.utf8.count + Task { + do { + let out = try await webView.callAsyncJavaScript(""" + await initPromise; + const inAddr = api.wacro_malloc(\(utf8Length)); + const mem = api.memory; + const arr = new Uint8Array(mem.buffer, inAddr, \(utf8Length)); + enc.encodeInto(json, arr); + const outAddr = api.wacro_parse(inAddr, \(utf8Length)); + const len = new Uint32Array(mem.buffer, outAddr)[0]; + const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); + const text = dec.decode(outArr); + api.wacro_free(outAddr); + return text; + """, arguments: ["json": json], contentWorld: .defaultClient) + guard let str = out as? String else { throw PluginServerError(message: "invalid wasm output") } + res = .success(str) + } catch { + res = .failure(error) + } + } + // can't use a semaphore because it would block the main runloop + while true { + RunLoop.main.run(until: .now.addingTimeInterval(0.01)) + if let res { return try res.get() } + } + } +} + +@available(macOS 11, *) +private final class SchemeHandler: NSObject, WKURLSchemeHandler where Chunks.Element == Data { + typealias RequestHandler = (URLRequest) async throws -> (Output, URLResponse) + + private var tasks: [ObjectIdentifier: Task] = [:] + private let onRequest: RequestHandler + + init(onRequest: @escaping RequestHandler) { + self.onRequest = onRequest + } + + func webView(_ webView: WKWebView, start task: any WKURLSchemeTask) { + tasks[ObjectIdentifier(task)] = Task { + var err: Error? + do { + let (stream, response) = try await onRequest(task.request) + try Task.checkCancellation() + task.didReceive(response) + for try await data in stream { + try Task.checkCancellation() + task.didReceive(data) + } + } catch { + err = error + } + guard !Task.isCancelled else { return } + if let err { + task.didFailWithError(err) + } else { + task.didFinish() + } + } + } + + func webView(_ webView: WKWebView, stop urlSchemeTask: any WKURLSchemeTask) { + tasks.removeValue(forKey: ObjectIdentifier(urlSchemeTask))?.cancel() + } +} From e5342e7cc3ec9a3a146108e8c42eebf0ce37b869 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 14 Apr 2024 23:10:13 -0400 Subject: [PATCH 002/105] wip --- include/swift/AST/PluginLoader.h | 1 + lib/AST/PluginLoader.cpp | 12 ++++++++---- lib/AST/PluginRegistry.cpp | 5 +---- lib/Sema/TypeCheckMacros.cpp | 3 +-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/include/swift/AST/PluginLoader.h b/include/swift/AST/PluginLoader.h index 232f876c1a3e0..9c1456e73caa3 100644 --- a/include/swift/AST/PluginLoader.h +++ b/include/swift/AST/PluginLoader.h @@ -32,6 +32,7 @@ class PluginLoader { struct PluginEntry { StringRef libraryPath; StringRef executablePath; + bool forceDisableSandbox; }; private: diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index c6644ef890ab9..fb80a71581208 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -72,7 +72,7 @@ PluginLoader::getPluginMap() { // Helper function to try inserting an entry if there's no existing entry // associated with the module name. auto try_emplace = [&](StringRef moduleName, StringRef libPath, - StringRef execPath) { + StringRef execPath, bool forceDisableSandbox = false) { auto moduleNameIdentifier = Ctx.getIdentifier(moduleName); if (map.find(moduleNameIdentifier) != map.end()) { // Specified module name is already in the map. @@ -81,7 +81,9 @@ PluginLoader::getPluginMap() { libPath = libPath.empty() ? "" : Ctx.AllocateCopy(libPath); execPath = execPath.empty() ? "" : Ctx.AllocateCopy(execPath); - auto result = map.insert({moduleNameIdentifier, {libPath, execPath}}); + auto result = map.insert({moduleNameIdentifier, { + libPath, execPath, forceDisableSandbox + }}); assert(result.second); (void)result; }; @@ -107,11 +109,13 @@ PluginLoader::getPluginMap() { auto &val = entry.get(); assert(!val.ExecutablePath.empty() && "empty plugin path"); if (llvm::sys::path::filename(val.ExecutablePath).ends_with(".wasm")) { + // we treat wasm plugins like library plugins that can be loaded by an external + // "macro runner" executable SmallString<128> runner; // TODO: improve path resolution: we really want tools_dir llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-plugin-server"); for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, val.ExecutablePath, runner); + try_emplace(moduleName, val.ExecutablePath, runner, true); } } else { for (auto &moduleName : val.ModuleNames) { @@ -160,7 +164,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) { auto &map = getPluginMap(); auto found = map.find(moduleName); if (found == map.end()) { - static PluginEntry notFound{"", ""}; + static PluginEntry notFound{"", "", false}; return notFound; } diff --git a/lib/AST/PluginRegistry.cpp b/lib/AST/PluginRegistry.cpp index 5b1d4af90187e..3bd5d0e870291 100644 --- a/lib/AST/PluginRegistry.cpp +++ b/lib/AST/PluginRegistry.cpp @@ -93,10 +93,7 @@ PluginRegistry::loadExecutablePlugin(StringRef path, bool disableSandbox) { std::lock_guard lock(mtx); // See if the plugin is already loaded. - // include disableSandbox in the key so that we can have a plugin loaded - // as both sandboxed and unsandboxed at the same time. - const std::string key = (disableSandbox ? "0" : "1") + path.str(); - auto &storage = LoadedPluginExecutables[key]; + auto &storage = LoadedPluginExecutables[path]; if (storage) { // See if the loaded one is still usable. if (storage->getLastModificationTime() == stat.getLastModificationTime()) diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 212fa61a58d32..8a630e0f90afb 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -345,9 +345,8 @@ CompilerPluginLoadRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, SmallString<0> errorMessage; if (!entry.executablePath.empty()) { - bool forceDisableSandbox = entry.libraryPath.ends_with(".wasm"); llvm::Expected executablePlugin = - loader.loadExecutablePlugin(entry.executablePath, forceDisableSandbox); + loader.loadExecutablePlugin(entry.executablePath, entry.forceDisableSandbox); if (executablePlugin) { if (ctx->LangOpts.EnableMacroLoadingRemarks) { unsigned tag = entry.libraryPath.empty() ? 1 : 2; From d5d1cb74a732ffec94bcb0e8f004f07830ab76ef Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 00:10:07 -0400 Subject: [PATCH 003/105] wip --- lib/AST/PluginLoader.cpp | 2 +- tools/swift-plugin-server/CMakeLists.txt | 14 + tools/swift-plugin-server/Package.swift | 13 + .../swift-wasm-plugin-server.swift | 337 ++++++++++++++++++ 4 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index fb80a71581208..ed605e2f29930 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -113,7 +113,7 @@ PluginLoader::getPluginMap() { // "macro runner" executable SmallString<128> runner; // TODO: improve path resolution: we really want tools_dir - llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-plugin-server"); + llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-wasm-plugin-server"); for (auto &moduleName : val.ModuleNames) { try_emplace(moduleName, val.ExecutablePath, runner, true); } diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 34e8637089521..a6f0d95386f60 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -27,4 +27,18 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) target_include_directories(swift-plugin-server PRIVATE Sources/CSwiftPluginServer/include ) + + add_pure_swift_host_tool(swift-wasm-plugin-server + Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift + DEPENDENCIES + $ + SWIFT_COMPONENT + compiler + SWIFT_DEPENDENCIES + SwiftCompilerPluginMessageHandling + swiftLLVMJSON + ) + target_include_directories(swift-wasm-plugin-server PRIVATE + Sources/CSwiftPluginServer/include + ) endif() diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 7c2b3cba83d1f..1731850a9e7a1 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -7,6 +7,10 @@ let package = Package( platforms: [ .macOS(.v10_15) ], + products: [ + .library(name: "swift-plugin-server", targets: ["swift-plugin-server"]), + .library(name: "swift-wasm-plugin-server", targets: ["swift-wasm-plugin-server"]), + ], dependencies: [ .package(path: "../../../swift-syntax"), .package(path: "../../lib/ASTGen"), @@ -38,6 +42,15 @@ let package = Package( ], swiftSettings: [.interoperabilityMode(.Cxx)] ), + .target( + name: "swift-wasm-plugin-server", + dependencies: [ + .product(name: "swiftLLVMJSON", package: "ASTGen"), + .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), + "CSwiftPluginServer" + ], + swiftSettings: [.interoperabilityMode(.Cxx)] + ), ], cxxLanguageStandard: .cxx17 ) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift new file mode 100644 index 0000000000000..f876256ce667b --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -0,0 +1,337 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling +import swiftLLVMJSON +import Foundation +import CSwiftPluginServer + +@main +final class SwiftPluginServer { + let connection: PluginHostConnection + var loadedWasmPlugins: [String: WasmPlugin] = [:] + var hostCapability: PluginMessage.HostCapability? + + init(connection: PluginHostConnection) { + self.connection = connection + } + + private func sendMessage(_ message: PluginToHostMessage) throws { + try connection.sendMessage(message) + } + + @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { + guard #available(macOS 12, *) else { throw PluginServerError(message: "Wasm support requires macOS 12+") } + guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } + let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) + return try WebKitWasmPlugin(wasm: wasm) + } + + private func expandMacro( + moduleName module: String, + message: HostToPluginMessage + ) async throws { + let response: PluginToHostMessage + do { + guard let plugin = loadedWasmPlugins[module] else { throw PluginServerError(message: "Could not find module \(module)") } + let request = try LLVMJSON.encoding(message) { String(decoding: UnsafeRawBufferPointer($0), as: UTF8.self) } + var responseRaw = try plugin.handleMessage(request) + response = try responseRaw.withUTF8 { + try $0.withMemoryRebound(to: Int8.self) { + try LLVMJSON.decode(PluginToHostMessage.self, from: $0) + } + } + } catch { + try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [ + ])) + return + } + try sendMessage(response) + } + + func run() async throws { + while let message = try connection.waitForNextMessage(HostToPluginMessage.self) { + switch message { + case .getCapability(let hostCapability): + // Remember the peer capability if provided. + if let hostCapability = hostCapability { + self.hostCapability = .init(hostCapability) + } + + // Return the plugin capability. + let capability = PluginMessage.PluginCapability( + protocolVersion: PluginMessage.PROTOCOL_VERSION_NUMBER, + features: [PluginFeature.loadPluginLibrary.rawValue] + ) + try sendMessage(.getCapabilityResult(capability: capability)) + case .loadPluginLibrary(let libraryPath, let moduleName): + do { + loadedWasmPlugins[moduleName] = try await loadPluginLibrary(path: libraryPath, moduleName: moduleName) + } catch { + try sendMessage(.loadPluginLibraryResult(loaded: false, diagnostics: [])) + continue + } + try sendMessage(.loadPluginLibraryResult(loaded: true, diagnostics: [])) + case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), + .expandFreestandingMacro(let macro, _, _, _, _): + try await expandMacro(moduleName: macro.moduleName, message: message) + } + } + } + + /// @main entry point. + static func main() async throws { + let connection = try PluginHostConnection() + try await Self(connection: connection).run() + } +} + +protocol WasmPlugin { + func handleMessage(_ json: String) throws -> String +} + +final class PluginHostConnection { + let handle: UnsafeRawPointer + init() throws { + var errorMessage: UnsafePointer? = nil + guard let handle = PluginServer_createConnection(&errorMessage) else { + throw PluginServerError(message: String(cString: errorMessage!)) + } + self.handle = handle + } + + deinit { + PluginServer_destroyConnection(self.handle) + } + + func sendMessage(_ message: TX) throws { + try LLVMJSON.encoding(message) { buffer in + try self.sendMessageData(buffer) + } + } + + func waitForNextMessage(_ type: RX.Type) throws -> RX? { + try self.withReadingMessageData { buffer in + try LLVMJSON.decode(RX.self, from: buffer) + } + } + + /// Send a serialized message to the message channel. + func sendMessageData(_ data: UnsafeBufferPointer) throws { + // Write the header (a 64-bit length field in little endian byte order). + var header: UInt64 = UInt64(data.count).littleEndian + let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in + try self.write(buffer: UnsafeRawBufferPointer(buffer)) + } + guard writtenSize == MemoryLayout.size(ofValue: header) else { + throw PluginServerError(message: "failed to write message header") + } + + // Write the body. + guard try self.write(buffer: UnsafeRawBufferPointer(data)) == data.count else { + throw PluginServerError(message: "failed to write message body") + } + } + + /// Read a serialized message from the message channel and call the 'body' + /// with the data. + private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { + // Read the header (a 64-bit length field in little endian byte order). + var header: UInt64 = 0 + let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in + try self.read(into: UnsafeMutableRawBufferPointer(buffer)) + } + guard readSize == MemoryLayout.size(ofValue: header) else { + if readSize == 0 { + // The host closed the pipe. + return nil + } + // Otherwise, some error happened. + throw PluginServerError(message: "failed to read message header") + } + + // Read the body. + let count = Int(UInt64(littleEndian: header)) + let data = UnsafeMutableBufferPointer.allocate(capacity: count) + defer { data.deallocate() } + guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { + throw PluginServerError(message: "failed to read message body") + } + + // Invoke the handler. + return try body(UnsafeBufferPointer(data)) + } + + /// Write the 'buffer' to the message channel. + /// Returns the number of bytes succeeded to write. + private func write(buffer: UnsafeRawBufferPointer) throws -> Int { + var bytesToWrite = buffer.count + guard bytesToWrite > 0 else { + return 0 + } + var ptr = buffer.baseAddress! + + while (bytesToWrite > 0) { + let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) + if (writtenSize <= 0) { + // error e.g. broken pipe. + break + } + ptr = ptr.advanced(by: writtenSize) + bytesToWrite -= Int(writtenSize) + } + return buffer.count - bytesToWrite + } + + /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. + /// Returns the number of bytes succeeded to read. + private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { + var bytesToRead = buffer.count + guard bytesToRead > 0 else { + return 0 + } + var ptr = buffer.baseAddress! + + while bytesToRead > 0 { + let readSize = PluginServer_read(handle, ptr, bytesToRead) + if (readSize <= 0) { + // 0: EOF (the host closed), -1: Broken pipe (the host crashed?) + break; + } + ptr = ptr.advanced(by: readSize) + bytesToRead -= readSize + } + return buffer.count - bytesToRead + } +} + +struct PluginServerError: Error, CustomStringConvertible { + var description: String + init(message: String) { + self.description = message + } +} + +import WebKit + +@available(macOS 12, *) +final class WebKitWasmPlugin: WasmPlugin { + private let webView: WKWebView + + private init(_ data: Chunks) throws where Chunks.Element == Data { + let configuration = WKWebViewConfiguration() + configuration.setURLSchemeHandler(SchemeHandler { request in + let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: [ + "access-control-allow-origin": "*", + "content-type": "application/wasm" + ])! + return (data, response) + }, forURLScheme: "wasm-runner-data") + webView = WKWebView(frame: .zero, configuration: configuration) + webView.evaluateJavaScript(""" + const initPromise = (async () => { + // somehow compile(await fetch.arrayBuf) is faster than + // compileStreaming(fetch). Beats me. + const data = await (await fetch("wasm-runner-data://")).arrayBuffer(); + const mod = await WebAssembly.compile(data); + // stub WASI imports + const imports = WebAssembly.Module.imports(mod) + .filter(x => x.module === "wasi_snapshot_preview1") + .map(x => [x.name, () => {}]); + const instance = await WebAssembly.instantiate(mod, { + wasi_snapshot_preview1: Object.fromEntries(imports) + }); + api = instance.exports; + enc = new TextEncoder(); + dec = new TextDecoder(); + api._start(); + })() + """, in: nil, in: .defaultClient) + } + + package convenience init(wasm: Data) throws { + try self.init(AsyncStream { + $0.yield(wasm) + $0.finish() + }) + } + + @MainActor func handleMessage(_ json: String) throws -> String { + var res: Result? + let utf8Length = json.utf8.count + Task { + do { + let out = try await webView.callAsyncJavaScript(""" + await initPromise; + const inAddr = api.wacro_malloc(\(utf8Length)); + const mem = api.memory; + const arr = new Uint8Array(mem.buffer, inAddr, \(utf8Length)); + enc.encodeInto(json, arr); + const outAddr = api.wacro_parse(inAddr, \(utf8Length)); + const len = new Uint32Array(mem.buffer, outAddr)[0]; + const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); + const text = dec.decode(outArr); + api.wacro_free(outAddr); + return text; + """, arguments: ["json": json], contentWorld: .defaultClient) + guard let str = out as? String else { throw PluginServerError(message: "invalid wasm output") } + res = .success(str) + } catch { + res = .failure(error) + } + } + // can't use a semaphore because it would block the main runloop + while true { + RunLoop.main.run(until: .now.addingTimeInterval(0.01)) + if let res { return try res.get() } + } + } +} + +@available(macOS 11, *) +private final class SchemeHandler: NSObject, WKURLSchemeHandler where Chunks.Element == Data { + typealias RequestHandler = (URLRequest) async throws -> (Output, URLResponse) + + private var tasks: [ObjectIdentifier: Task] = [:] + private let onRequest: RequestHandler + + init(onRequest: @escaping RequestHandler) { + self.onRequest = onRequest + } + + func webView(_ webView: WKWebView, start task: any WKURLSchemeTask) { + tasks[ObjectIdentifier(task)] = Task { + var err: Error? + do { + let (stream, response) = try await onRequest(task.request) + try Task.checkCancellation() + task.didReceive(response) + for try await data in stream { + try Task.checkCancellation() + task.didReceive(data) + } + } catch { + err = error + } + guard !Task.isCancelled else { return } + if let err { + task.didFailWithError(err) + } else { + task.didFinish() + } + } + } + + func webView(_ webView: WKWebView, stop urlSchemeTask: any WKURLSchemeTask) { + tasks.removeValue(forKey: ObjectIdentifier(urlSchemeTask))?.cancel() + } +} From f776c287af7431635ba8694000bd9599c94b0766 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 00:10:52 -0400 Subject: [PATCH 004/105] revert swift-plugin-server --- .../swift-plugin-server.swift | 161 +----------------- 1 file changed, 5 insertions(+), 156 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 28d5ab5b7a57c..383a1b99491ed 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -14,7 +14,6 @@ import SwiftSyntaxMacros import swiftLLVMJSON import CSwiftPluginServer -import Foundation @main final class SwiftPluginServer { @@ -35,37 +34,23 @@ final class SwiftPluginServer { /// Loaded dylib handles associated with the module name. var loadedLibraryPlugins: [String: LoadedLibraryPlugin] = [:] - var loadedWasmPlugins: [String: WasmPlugin] = [:] - /// Resolved cached macros. var resolvedMacros: [MacroRef: Macro.Type] = [:] /// @main entry point. static func main() throws { - let provider = self.init() - let connection = try PluginHostConnection(provider: provider) + let connection = try PluginHostConnection() let messageHandler = CompilerPluginMessageHandler( connection: connection, - provider: provider + provider: self.init() ) try messageHandler.main() } } -protocol WasmPlugin { - func handleMessage(_ json: String) throws -> String -} - extension SwiftPluginServer: PluginProvider { /// Load a macro implementation from the dynamic link library. func loadPluginLibrary(libraryPath: String, moduleName: String) throws { - if libraryPath.hasSuffix(".wasm") { - guard #available(macOS 12, *) else { throw PluginServerError(message: "Wasm support requires macOS 11+") } - let wasm = try Data(contentsOf: URL(fileURLWithPath: libraryPath)) - let plugin = try WebKitWasmPlugin(wasm: wasm) - loadedWasmPlugins[moduleName] = plugin - return - } var errorMessage: UnsafePointer? guard let dlHandle = PluginServer_load(libraryPath, &errorMessage) else { throw PluginServerError(message: "loader error: " + String(cString: errorMessage!)) @@ -115,15 +100,13 @@ extension SwiftPluginServer: PluginProvider { } final class PluginHostConnection: MessageConnection { - let provider: SwiftPluginServer let handle: UnsafeRawPointer - init(provider: SwiftPluginServer) throws { + init() throws { var errorMessage: UnsafePointer? = nil guard let handle = PluginServer_createConnection(&errorMessage) else { throw PluginServerError(message: String(cString: errorMessage!)) } self.handle = handle - self.provider = provider } deinit { @@ -137,27 +120,8 @@ final class PluginHostConnection: MessageConnection { } func waitForNextMessage(_ type: RX.Type) throws -> RX? { - while true { - let result = try self.withReadingMessageData { jsonData -> RX? in - let hostMessage = try LLVMJSON.decode(HostToPluginMessage.self, from: jsonData) - switch hostMessage { - case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), - .expandFreestandingMacro(let macro, _, _, _, _): - if let plugin = provider.loadedWasmPlugins[macro.moduleName] { - var response = try plugin.handleMessage(String(decoding: UnsafeRawBufferPointer(jsonData), as: UTF8.self)) - try response.withUTF8 { - try $0.withMemoryRebound(to: Int8.self) { - try self.sendMessageData($0) - } - } - return nil - } - default: - break - } - return try LLVMJSON.decode(RX.self, from: jsonData) - } ?? nil - if let result { return result } + return try self.withReadingMessageData { jsonData in + try LLVMJSON.decode(RX.self, from: jsonData) } } @@ -256,118 +220,3 @@ struct PluginServerError: Error, CustomStringConvertible { self.description = message } } - -import WebKit - -@available(macOS 12, *) -final class WebKitWasmPlugin: WasmPlugin { - private let webView: WKWebView - - private init(_ data: Chunks) throws where Chunks.Element == Data { - let configuration = WKWebViewConfiguration() - configuration.setURLSchemeHandler(SchemeHandler { request in - let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: [ - "access-control-allow-origin": "*", - "content-type": "application/wasm" - ])! - return (data, response) - }, forURLScheme: "wasm-runner-data") - webView = WKWebView(frame: .zero, configuration: configuration) - webView.evaluateJavaScript(""" - const initPromise = (async () => { - // somehow compile(await fetch.arrayBuf) is faster than - // compileStreaming(fetch). Beats me. - const data = await (await fetch("wasm-runner-data://")).arrayBuffer(); - const mod = await WebAssembly.compile(data); - // stub WASI imports - const imports = WebAssembly.Module.imports(mod) - .filter(x => x.module === "wasi_snapshot_preview1") - .map(x => [x.name, () => {}]); - const instance = await WebAssembly.instantiate(mod, { - wasi_snapshot_preview1: Object.fromEntries(imports) - }); - api = instance.exports; - enc = new TextEncoder(); - dec = new TextDecoder(); - api._start(); - })() - """, in: nil, in: .defaultClient) - } - - package convenience init(wasm: Data) throws { - try self.init(AsyncStream { - $0.yield(wasm) - $0.finish() - }) - } - - @MainActor func handleMessage(_ json: String) throws -> String { - var res: Result? - let utf8Length = json.utf8.count - Task { - do { - let out = try await webView.callAsyncJavaScript(""" - await initPromise; - const inAddr = api.wacro_malloc(\(utf8Length)); - const mem = api.memory; - const arr = new Uint8Array(mem.buffer, inAddr, \(utf8Length)); - enc.encodeInto(json, arr); - const outAddr = api.wacro_parse(inAddr, \(utf8Length)); - const len = new Uint32Array(mem.buffer, outAddr)[0]; - const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); - const text = dec.decode(outArr); - api.wacro_free(outAddr); - return text; - """, arguments: ["json": json], contentWorld: .defaultClient) - guard let str = out as? String else { throw PluginServerError(message: "invalid wasm output") } - res = .success(str) - } catch { - res = .failure(error) - } - } - // can't use a semaphore because it would block the main runloop - while true { - RunLoop.main.run(until: .now.addingTimeInterval(0.01)) - if let res { return try res.get() } - } - } -} - -@available(macOS 11, *) -private final class SchemeHandler: NSObject, WKURLSchemeHandler where Chunks.Element == Data { - typealias RequestHandler = (URLRequest) async throws -> (Output, URLResponse) - - private var tasks: [ObjectIdentifier: Task] = [:] - private let onRequest: RequestHandler - - init(onRequest: @escaping RequestHandler) { - self.onRequest = onRequest - } - - func webView(_ webView: WKWebView, start task: any WKURLSchemeTask) { - tasks[ObjectIdentifier(task)] = Task { - var err: Error? - do { - let (stream, response) = try await onRequest(task.request) - try Task.checkCancellation() - task.didReceive(response) - for try await data in stream { - try Task.checkCancellation() - task.didReceive(data) - } - } catch { - err = error - } - guard !Task.isCancelled else { return } - if let err { - task.didFailWithError(err) - } else { - task.didFinish() - } - } - } - - func webView(_ webView: WKWebView, stop urlSchemeTask: any WKURLSchemeTask) { - tasks.removeValue(forKey: ObjectIdentifier(urlSchemeTask))?.cancel() - } -} From 43fc23304909a51ddc949140d3cc477e306052bf Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 01:29:31 -0400 Subject: [PATCH 005/105] use jsc in-proc --- tools/swift-plugin-server/CMakeLists.txt | 5 +- .../swift-wasm-plugin-server.swift | 205 ++++++++---------- tools/swift-plugin-server/allow-jit.plist | 8 + 3 files changed, 104 insertions(+), 114 deletions(-) create mode 100644 tools/swift-plugin-server/allow-jit.plist diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index a6f0d95386f60..bbd2fbacd9b8e 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -39,6 +39,9 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) swiftLLVMJSON ) target_include_directories(swift-wasm-plugin-server PRIVATE - Sources/CSwiftPluginServer/include + Sources/CSwiftPluginServer/include + ) + add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND + codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist ) endif() diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index f876256ce667b..4a2dd2f2de84b 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -30,10 +30,10 @@ final class SwiftPluginServer { } @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - guard #available(macOS 12, *) else { throw PluginServerError(message: "Wasm support requires macOS 12+") } + guard #available(macOS 10.15, *) else { throw PluginServerError(message: "Wasm support requires macOS 12+") } guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - return try WebKitWasmPlugin(wasm: wasm) + return try await JSCWasmPlugin(wasm: wasm) } private func expandMacro( @@ -43,16 +43,15 @@ final class SwiftPluginServer { let response: PluginToHostMessage do { guard let plugin = loadedWasmPlugins[module] else { throw PluginServerError(message: "Could not find module \(module)") } - let request = try LLVMJSON.encoding(message) { String(decoding: UnsafeRawBufferPointer($0), as: UTF8.self) } - var responseRaw = try plugin.handleMessage(request) - response = try responseRaw.withUTF8 { + let request = try LLVMJSON.encoding(message) { Data(UnsafeRawBufferPointer($0)) } + let responseRaw = try await plugin.handleMessage(request) + response = try responseRaw.withUnsafeBytes { try $0.withMemoryRebound(to: Int8.self) { try LLVMJSON.decode(PluginToHostMessage.self, from: $0) } } } catch { - try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [ - ])) + try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [])) return } try sendMessage(response) @@ -96,7 +95,7 @@ final class SwiftPluginServer { } protocol WasmPlugin { - func handleMessage(_ json: String) throws -> String + func handleMessage(_ json: Data) async throws -> Data } final class PluginHostConnection { @@ -221,117 +220,97 @@ struct PluginServerError: Error, CustomStringConvertible { } } -import WebKit - -@available(macOS 12, *) -final class WebKitWasmPlugin: WasmPlugin { - private let webView: WKWebView - - private init(_ data: Chunks) throws where Chunks.Element == Data { - let configuration = WKWebViewConfiguration() - configuration.setURLSchemeHandler(SchemeHandler { request in - let response = HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: "HTTP/1.1", headerFields: [ - "access-control-allow-origin": "*", - "content-type": "application/wasm" - ])! - return (data, response) - }, forURLScheme: "wasm-runner-data") - webView = WKWebView(frame: .zero, configuration: configuration) - webView.evaluateJavaScript(""" - const initPromise = (async () => { - // somehow compile(await fetch.arrayBuf) is faster than - // compileStreaming(fetch). Beats me. - const data = await (await fetch("wasm-runner-data://")).arrayBuffer(); - const mod = await WebAssembly.compile(data); - // stub WASI imports - const imports = WebAssembly.Module.imports(mod) - .filter(x => x.module === "wasi_snapshot_preview1") - .map(x => [x.name, () => {}]); - const instance = await WebAssembly.instantiate(mod, { - wasi_snapshot_preview1: Object.fromEntries(imports) - }); - api = instance.exports; - enc = new TextEncoder(); - dec = new TextDecoder(); - api._start(); - })() - """, in: nil, in: .defaultClient) +import JavaScriptCore + +@available(macOS 10.15, *) +final class JSCWasmPlugin: WasmPlugin { + private let context: JSContext + private let fn: JSValue + + @MainActor init(wasm data: Data) async throws { + guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } + self.context = context + + let jsBuf = try JSValue(newBufferWithData: data, in: context) + context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") + + let promise = context.evaluateScript(""" + (async () => { + const mod = await WebAssembly.compile(wasmData); + // stub WASI imports + const imports = WebAssembly.Module.imports(mod) + .filter(x => x.module === "wasi_snapshot_preview1") + .map(x => [x.name, () => {}]); + const instance = await WebAssembly.instantiate(mod, { + wasi_snapshot_preview1: Object.fromEntries(imports) + }); + const api = instance.exports; + api._start(); + return ((json) => { + const inAddr = api.wacro_malloc(json.byteLength); + const mem = api.memory; + const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); + arr.set(new Uint8Array(json)); + const outAddr = api.wacro_parse(inAddr, json.byteLength); + const len = new Uint32Array(mem.buffer, outAddr)[0]; + const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); + const copy = new Uint8Array(outArr); + api.wacro_free(outAddr); + return copy.buffer; + }) + })() + """)! + + if let error = context.exception { + throw PluginServerError(message: "Failed to load plugin: \(error)") } - package convenience init(wasm: Data) throws { - try self.init(AsyncStream { - $0.yield(wasm) - $0.finish() - }) - } + var result: Result? - @MainActor func handleMessage(_ json: String) throws -> String { - var res: Result? - let utf8Length = json.utf8.count - Task { - do { - let out = try await webView.callAsyncJavaScript(""" - await initPromise; - const inAddr = api.wacro_malloc(\(utf8Length)); - const mem = api.memory; - const arr = new Uint8Array(mem.buffer, inAddr, \(utf8Length)); - enc.encodeInto(json, arr); - const outAddr = api.wacro_parse(inAddr, \(utf8Length)); - const len = new Uint32Array(mem.buffer, outAddr)[0]; - const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); - const text = dec.decode(outArr); - api.wacro_free(outAddr); - return text; - """, arguments: ["json": json], contentWorld: .defaultClient) - guard let str = out as? String else { throw PluginServerError(message: "invalid wasm output") } - res = .success(str) - } catch { - res = .failure(error) - } - } - // can't use a semaphore because it would block the main runloop - while true { - RunLoop.main.run(until: .now.addingTimeInterval(0.01)) - if let res { return try res.get() } - } - } -} + promise + .invokeMethod("then", withArguments: [JSValue(object: { val in + result = .success(val) + } as @convention(block) (JSValue) -> Void, in: context)!]) + .invokeMethod("catch", withArguments: [JSValue(object: { err in + result = .failure(PluginServerError(message: "\(err)")) + } as @convention(block) (JSValue) -> Void, in: context)!]) -@available(macOS 11, *) -private final class SchemeHandler: NSObject, WKURLSchemeHandler where Chunks.Element == Data { - typealias RequestHandler = (URLRequest) async throws -> (Output, URLResponse) + if let error = context.exception { + throw PluginServerError(message: "Failed to load plugin: \(error)") + } - private var tasks: [ObjectIdentifier: Task] = [:] - private let onRequest: RequestHandler + fn = try { + while true { + RunLoop.main.run(until: Date().addingTimeInterval(0.01)) + if let result { return result } + } + }().get() + } - init(onRequest: @escaping RequestHandler) { - self.onRequest = onRequest - } + @MainActor func handleMessage(_ json: Data) async throws -> Data { + let jsonJS = try JSValue(newBufferWithData: json, in: context) + let res = fn.call(withArguments: [jsonJS]) + return res!.toData() + } +} - func webView(_ webView: WKWebView, start task: any WKURLSchemeTask) { - tasks[ObjectIdentifier(task)] = Task { - var err: Error? - do { - let (stream, response) = try await onRequest(task.request) - try Task.checkCancellation() - task.didReceive(response) - for try await data in stream { - try Task.checkCancellation() - task.didReceive(data) - } - } catch { - err = error - } - guard !Task.isCancelled else { return } - if let err { - task.didFailWithError(err) - } else { - task.didFinish() - } - } - } +extension JSValue { + convenience init(newBufferWithData data: Data, in context: JSContext) throws { + let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) + _ = copy.initialize(from: data) + let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( + context.jsGlobalContextRef, + copy.baseAddress, copy.count, + { buf, _ in buf?.deallocate() }, nil, + nil + ) + self.init(jsValueRef: rawBuf, in: context) + } - func webView(_ webView: WKWebView, stop urlSchemeTask: any WKURLSchemeTask) { - tasks.removeValue(forKey: ObjectIdentifier(urlSchemeTask))?.cancel() - } + func toData() -> Data { + let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) + let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) + let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) + return Data(buf) + } } diff --git a/tools/swift-plugin-server/allow-jit.plist b/tools/swift-plugin-server/allow-jit.plist new file mode 100644 index 0000000000000..005d584f6fa8b --- /dev/null +++ b/tools/swift-plugin-server/allow-jit.plist @@ -0,0 +1,8 @@ + + + + +com.apple.security.cs.allow-jit + + + From 961fd54a0c621af56563b67d8bb98210fa7a4458 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 01:39:54 -0400 Subject: [PATCH 006/105] =?UTF-8?q?Allow=20wasm=20in=20sandbox,=20don?= =?UTF-8?q?=E2=80=99t=20disable=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/swift/AST/PluginLoader.h | 3 +-- lib/AST/PluginLoader.cpp | 14 ++++++-------- lib/Basic/Sandbox.cpp | 4 ++-- lib/Sema/TypeCheckMacros.cpp | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/include/swift/AST/PluginLoader.h b/include/swift/AST/PluginLoader.h index 9c1456e73caa3..f35d5adea7fb8 100644 --- a/include/swift/AST/PluginLoader.h +++ b/include/swift/AST/PluginLoader.h @@ -32,7 +32,6 @@ class PluginLoader { struct PluginEntry { StringRef libraryPath; StringRef executablePath; - bool forceDisableSandbox; }; private: @@ -87,7 +86,7 @@ class PluginLoader { /// NOTE: This method is idempotent. If the plugin is already loaded, the same /// instance is simply returned. llvm::Expected - loadExecutablePlugin(llvm::StringRef path, bool forceDisableSandbox); + loadExecutablePlugin(llvm::StringRef path); /// Add the specified plugin associated with the module name to the dependency /// tracker if needed. diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index ed605e2f29930..e6a6a6e5d87fd 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -72,7 +72,7 @@ PluginLoader::getPluginMap() { // Helper function to try inserting an entry if there's no existing entry // associated with the module name. auto try_emplace = [&](StringRef moduleName, StringRef libPath, - StringRef execPath, bool forceDisableSandbox = false) { + StringRef execPath) { auto moduleNameIdentifier = Ctx.getIdentifier(moduleName); if (map.find(moduleNameIdentifier) != map.end()) { // Specified module name is already in the map. @@ -81,9 +81,7 @@ PluginLoader::getPluginMap() { libPath = libPath.empty() ? "" : Ctx.AllocateCopy(libPath); execPath = execPath.empty() ? "" : Ctx.AllocateCopy(execPath); - auto result = map.insert({moduleNameIdentifier, { - libPath, execPath, forceDisableSandbox - }}); + auto result = map.insert({moduleNameIdentifier, {libPath, execPath}}); assert(result.second); (void)result; }; @@ -115,7 +113,7 @@ PluginLoader::getPluginMap() { // TODO: improve path resolution: we really want tools_dir llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-wasm-plugin-server"); for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, val.ExecutablePath, runner, true); + try_emplace(moduleName, val.ExecutablePath, runner); } } else { for (auto &moduleName : val.ModuleNames) { @@ -164,7 +162,7 @@ PluginLoader::lookupPluginByModuleName(Identifier moduleName) { auto &map = getPluginMap(); auto found = map.find(moduleName); if (found == map.end()) { - static PluginEntry notFound{"", "", false}; + static PluginEntry notFound{"", ""}; return notFound; } @@ -198,7 +196,7 @@ PluginLoader::loadLibraryPlugin(StringRef path) { } llvm::Expected -PluginLoader::loadExecutablePlugin(StringRef path, bool forceDisableSandbox) { +PluginLoader::loadExecutablePlugin(StringRef path) { auto fs = Ctx.SourceMgr.getFileSystem(); SmallString<128> resolvedPath; if (auto err = fs->getRealPath(path, resolvedPath)) { @@ -207,7 +205,7 @@ PluginLoader::loadExecutablePlugin(StringRef path, bool forceDisableSandbox) { // Load the plugin. auto plugin = - getRegistry()->loadExecutablePlugin(resolvedPath, disableSandbox || forceDisableSandbox); + getRegistry()->loadExecutablePlugin(resolvedPath, disableSandbox); if (!plugin) { resolvedPath.push_back(0); return llvm::handleErrors( diff --git a/lib/Basic/Sandbox.cpp b/lib/Basic/Sandbox.cpp index 3c60ef9c10101..f0055feaeb1f6 100644 --- a/lib/Basic/Sandbox.cpp +++ b/lib/Basic/Sandbox.cpp @@ -24,8 +24,8 @@ static StringRef sandboxProfile(llvm::BumpPtrAllocator &Alloc) { // Allow reading file metadata of any files. contents += "(allow file-read-metadata)\n"; - // Allow reading dylibs. - contents += "(allow file-read* (regex #\"\\.dylib$\"))\n"; + // Allow reading dylibs and WebAssembly macros. + contents += "(allow file-read* (regex #\"\\.(dylib|wasm)$\"))\n"; // This is required to launch any processes (execve(2)). contents += "(allow process-exec*)\n"; diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 8a630e0f90afb..84a0d273f150f 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -346,7 +346,7 @@ CompilerPluginLoadRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, if (!entry.executablePath.empty()) { llvm::Expected executablePlugin = - loader.loadExecutablePlugin(entry.executablePath, entry.forceDisableSandbox); + loader.loadExecutablePlugin(entry.executablePath); if (executablePlugin) { if (ctx->LangOpts.EnableMacroLoadingRemarks) { unsigned tag = entry.libraryPath.empty() ? 1 : 2; From a7716315791b697edbb03304cbdd9a17c73dee79 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 02:08:46 -0400 Subject: [PATCH 007/105] fix warning, prune version check --- .../swift-wasm-plugin-server.swift | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 4a2dd2f2de84b..bb776d9f74405 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -30,10 +30,9 @@ final class SwiftPluginServer { } @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - guard #available(macOS 10.15, *) else { throw PluginServerError(message: "Wasm support requires macOS 12+") } guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - return try await JSCWasmPlugin(wasm: wasm) + return try JSCWasmPlugin(wasm: wasm) } private func expandMacro( @@ -83,6 +82,10 @@ final class SwiftPluginServer { case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), .expandFreestandingMacro(let macro, _, _, _, _): try await expandMacro(moduleName: macro.moduleName, message: message) + #if !SWIFT_PACKAGE + @unknown default: + break + #endif } } } @@ -222,12 +225,11 @@ struct PluginServerError: Error, CustomStringConvertible { import JavaScriptCore -@available(macOS 10.15, *) final class JSCWasmPlugin: WasmPlugin { private let context: JSContext private let fn: JSValue - @MainActor init(wasm data: Data) async throws { + @MainActor init(wasm data: Data) throws { guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } self.context = context @@ -287,7 +289,7 @@ final class JSCWasmPlugin: WasmPlugin { }().get() } - @MainActor func handleMessage(_ json: Data) async throws -> Data { + @MainActor func handleMessage(_ json: Data) throws -> Data { let jsonJS = try JSValue(newBufferWithData: json, in: context) let res = fn.call(withArguments: [jsonJS]) return res!.toData() From 3f8ef819180340cbf1f3759f435cadc8db683b06 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 02:10:46 -0400 Subject: [PATCH 008/105] Factor out JSCWasmPlugin --- tools/swift-plugin-server/CMakeLists.txt | 1 + .../JSCWasmPlugin.swift | 93 ++++++++++++++++++ .../swift-wasm-plugin-server.swift | 94 ------------------- 3 files changed, 94 insertions(+), 94 deletions(-) create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index bbd2fbacd9b8e..3220ec79527e9 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -30,6 +30,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) add_pure_swift_host_tool(swift-wasm-plugin-server Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift + Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift DEPENDENCIES $ SWIFT_COMPONENT diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift new file mode 100644 index 0000000000000..873ce303fed75 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -0,0 +1,93 @@ +import JavaScriptCore + +final class JSCWasmPlugin: WasmPlugin { + private let context: JSContext + private let fn: JSValue + + @MainActor init(wasm data: Data) throws { + guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } + self.context = context + + let jsBuf = try JSValue(newBufferWithData: data, in: context) + context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") + + let promise = context.evaluateScript(""" + (async () => { + const mod = await WebAssembly.compile(wasmData); + // stub WASI imports + const imports = WebAssembly.Module.imports(mod) + .filter(x => x.module === "wasi_snapshot_preview1") + .map(x => [x.name, () => {}]); + const instance = await WebAssembly.instantiate(mod, { + wasi_snapshot_preview1: Object.fromEntries(imports) + }); + const api = instance.exports; + api._start(); + return ((json) => { + const inAddr = api.wacro_malloc(json.byteLength); + const mem = api.memory; + const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); + arr.set(new Uint8Array(json)); + const outAddr = api.wacro_parse(inAddr, json.byteLength); + const len = new Uint32Array(mem.buffer, outAddr)[0]; + const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); + const copy = new Uint8Array(outArr); + api.wacro_free(outAddr); + return copy.buffer; + }) + })() + """)! + + if let error = context.exception { + throw PluginServerError(message: "Failed to load plugin: \(error)") + } + + var result: Result? + + promise + .invokeMethod("then", withArguments: [JSValue(object: { val in + result = .success(val) + } as @convention(block) (JSValue) -> Void, in: context)!]) + .invokeMethod("catch", withArguments: [JSValue(object: { err in + result = .failure(PluginServerError(message: "\(err)")) + } as @convention(block) (JSValue) -> Void, in: context)!]) + + if let error = context.exception { + throw PluginServerError(message: "Failed to load plugin: \(error)") + } + + fn = try { + while true { + RunLoop.main.run(until: Date().addingTimeInterval(0.01)) + if let result { return result } + } + }().get() + } + + @MainActor func handleMessage(_ json: Data) throws -> Data { + let jsonJS = try JSValue(newBufferWithData: json, in: context) + let res = fn.call(withArguments: [jsonJS]) + return res!.toData() + } +} + +extension JSValue { + convenience init(newBufferWithData data: Data, in context: JSContext) throws { + let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) + _ = copy.initialize(from: data) + let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( + context.jsGlobalContextRef, + copy.baseAddress, copy.count, + { buf, _ in buf?.deallocate() }, nil, + nil + ) + self.init(jsValueRef: rawBuf, in: context) + } + + func toData() -> Data { + let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) + let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) + let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) + return Data(buf) + } +} diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index bb776d9f74405..682de899064ab 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -222,97 +222,3 @@ struct PluginServerError: Error, CustomStringConvertible { self.description = message } } - -import JavaScriptCore - -final class JSCWasmPlugin: WasmPlugin { - private let context: JSContext - private let fn: JSValue - - @MainActor init(wasm data: Data) throws { - guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } - self.context = context - - let jsBuf = try JSValue(newBufferWithData: data, in: context) - context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") - - let promise = context.evaluateScript(""" - (async () => { - const mod = await WebAssembly.compile(wasmData); - // stub WASI imports - const imports = WebAssembly.Module.imports(mod) - .filter(x => x.module === "wasi_snapshot_preview1") - .map(x => [x.name, () => {}]); - const instance = await WebAssembly.instantiate(mod, { - wasi_snapshot_preview1: Object.fromEntries(imports) - }); - const api = instance.exports; - api._start(); - return ((json) => { - const inAddr = api.wacro_malloc(json.byteLength); - const mem = api.memory; - const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); - arr.set(new Uint8Array(json)); - const outAddr = api.wacro_parse(inAddr, json.byteLength); - const len = new Uint32Array(mem.buffer, outAddr)[0]; - const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); - const copy = new Uint8Array(outArr); - api.wacro_free(outAddr); - return copy.buffer; - }) - })() - """)! - - if let error = context.exception { - throw PluginServerError(message: "Failed to load plugin: \(error)") - } - - var result: Result? - - promise - .invokeMethod("then", withArguments: [JSValue(object: { val in - result = .success(val) - } as @convention(block) (JSValue) -> Void, in: context)!]) - .invokeMethod("catch", withArguments: [JSValue(object: { err in - result = .failure(PluginServerError(message: "\(err)")) - } as @convention(block) (JSValue) -> Void, in: context)!]) - - if let error = context.exception { - throw PluginServerError(message: "Failed to load plugin: \(error)") - } - - fn = try { - while true { - RunLoop.main.run(until: Date().addingTimeInterval(0.01)) - if let result { return result } - } - }().get() - } - - @MainActor func handleMessage(_ json: Data) throws -> Data { - let jsonJS = try JSValue(newBufferWithData: json, in: context) - let res = fn.call(withArguments: [jsonJS]) - return res!.toData() - } -} - -extension JSValue { - convenience init(newBufferWithData data: Data, in context: JSContext) throws { - let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) - _ = copy.initialize(from: data) - let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( - context.jsGlobalContextRef, - copy.baseAddress, copy.count, - { buf, _ in buf?.deallocate() }, nil, - nil - ) - self.init(jsValueRef: rawBuf, in: context) - } - - func toData() -> Data { - let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) - let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) - let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) - return Data(buf) - } -} From 23426e014468375097123b4e9687385d2c91aaf8 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 02:23:12 -0400 Subject: [PATCH 009/105] the continuation just... works now? --- .../JSCWasmPlugin.swift | 113 ++++++++++-------- .../swift-wasm-plugin-server.swift | 7 +- 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 873ce303fed75..3bd57cfe30d1f 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -1,67 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + import JavaScriptCore +private let js = """ +(async () => { + const mod = await WebAssembly.compile(wasmData); + // stub WASI imports + const imports = WebAssembly.Module.imports(mod) + .filter(x => x.module === "wasi_snapshot_preview1") + .map(x => [x.name, () => {}]); + const instance = await WebAssembly.instantiate(mod, { + wasi_snapshot_preview1: Object.fromEntries(imports) + }); + const api = instance.exports; + api._start(); + return (json) => { + const inAddr = api.wacro_malloc(json.byteLength); + const mem = api.memory; + const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); + arr.set(new Uint8Array(json)); + const outAddr = api.wacro_parse(inAddr, json.byteLength); + const len = new Uint32Array(mem.buffer, outAddr)[0]; + const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); + const copy = new Uint8Array(outArr); + api.wacro_free(outAddr); + return copy.buffer; + } +})() +""" + +@available(macOS 10.15, *) final class JSCWasmPlugin: WasmPlugin { private let context: JSContext private let fn: JSValue - @MainActor init(wasm data: Data) throws { + @MainActor init(wasm data: Data) async throws { guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } self.context = context let jsBuf = try JSValue(newBufferWithData: data, in: context) context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") - let promise = context.evaluateScript(""" - (async () => { - const mod = await WebAssembly.compile(wasmData); - // stub WASI imports - const imports = WebAssembly.Module.imports(mod) - .filter(x => x.module === "wasi_snapshot_preview1") - .map(x => [x.name, () => {}]); - const instance = await WebAssembly.instantiate(mod, { - wasi_snapshot_preview1: Object.fromEntries(imports) - }); - const api = instance.exports; - api._start(); - return ((json) => { - const inAddr = api.wacro_malloc(json.byteLength); - const mem = api.memory; - const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); - arr.set(new Uint8Array(json)); - const outAddr = api.wacro_parse(inAddr, json.byteLength); - const len = new Uint32Array(mem.buffer, outAddr)[0]; - const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); - const copy = new Uint8Array(outArr); - api.wacro_free(outAddr); - return copy.buffer; - }) - })() - """)! - - if let error = context.exception { - throw PluginServerError(message: "Failed to load plugin: \(error)") + guard let promise = context.evaluateScript(js) else { + throw PluginServerError(message: "Failed to load plugin") } - var result: Result? - - promise - .invokeMethod("then", withArguments: [JSValue(object: { val in - result = .success(val) - } as @convention(block) (JSValue) -> Void, in: context)!]) - .invokeMethod("catch", withArguments: [JSValue(object: { err in - result = .failure(PluginServerError(message: "\(err)")) - } as @convention(block) (JSValue) -> Void, in: context)!]) - if let error = context.exception { throw PluginServerError(message: "Failed to load plugin: \(error)") } - fn = try { - while true { - RunLoop.main.run(until: Date().addingTimeInterval(0.01)) - if let result { return result } - } - }().get() + fn = try await promise.promiseValue } @MainActor func handleMessage(_ json: Data) throws -> Data { @@ -72,7 +70,7 @@ final class JSCWasmPlugin: WasmPlugin { } extension JSValue { - convenience init(newBufferWithData data: Data, in context: JSContext) throws { + fileprivate convenience init(newBufferWithData data: Data, in context: JSContext) throws { let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) _ = copy.initialize(from: data) let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( @@ -84,10 +82,29 @@ extension JSValue { self.init(jsValueRef: rawBuf, in: context) } - func toData() -> Data { + fileprivate func toData() -> Data { let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) return Data(buf) } + + @available(macOS 10.15, *) + fileprivate var promiseValue: JSValue { + get async throws { + try await withCheckedThrowingContinuation { continuation in + invokeMethod("then", withArguments: [ + JSValue(object: { val in + continuation.resume(returning: val) + } as @convention(block) (JSValue) -> Void, in: context)!, + JSValue(object: { err in + continuation.resume(throwing: PluginServerError(message: "\(err)")) + } as @convention(block) (JSValue) -> Void, in: context)! + ]) + if let exception = context.exception { + continuation.resume(throwing: PluginServerError(message: "\(exception)")) + } + } + } + } } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 682de899064ab..7bdef32d4544d 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -10,11 +10,12 @@ // //===----------------------------------------------------------------------===// -@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling +import SwiftCompilerPluginMessageHandling import swiftLLVMJSON import Foundation import CSwiftPluginServer +@available(macOS 10.15, *) @main final class SwiftPluginServer { let connection: PluginHostConnection @@ -32,7 +33,7 @@ final class SwiftPluginServer { @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - return try JSCWasmPlugin(wasm: wasm) + return try await JSCWasmPlugin(wasm: wasm) } private func expandMacro( From 9fe505f885c784debb13e3625e3e787f8bbc83b7 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 02:32:25 -0400 Subject: [PATCH 010/105] more tweaks to JSCWasmPlugin --- .../JSCWasmPlugin.swift | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 3bd57cfe30d1f..5de0cf1460d21 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -12,6 +12,7 @@ import JavaScriptCore +// returns: Promise<(ArrayBuffer) => ArrayBuffer> private let js = """ (async () => { const mod = await WebAssembly.compile(wasmData); @@ -41,12 +42,10 @@ private let js = """ @available(macOS 10.15, *) final class JSCWasmPlugin: WasmPlugin { - private let context: JSContext - private let fn: JSValue + private let handler: JSValue @MainActor init(wasm data: Data) async throws { guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } - self.context = context let jsBuf = try JSValue(newBufferWithData: data, in: context) context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") @@ -59,13 +58,15 @@ final class JSCWasmPlugin: WasmPlugin { throw PluginServerError(message: "Failed to load plugin: \(error)") } - fn = try await promise.promiseValue + handler = try await promise.promiseValue } @MainActor func handleMessage(_ json: Data) throws -> Data { - let jsonJS = try JSValue(newBufferWithData: json, in: context) - let res = fn.call(withArguments: [jsonJS]) - return res!.toData() + let jsonJS = try JSValue(newBufferWithData: json, in: handler.context) + guard let result = handler.call(withArguments: [jsonJS]) else { + throw PluginServerError(message: "Wasm plugin did not provide a valid response") + } + return result.arrayBufferData() } } @@ -82,7 +83,7 @@ extension JSValue { self.init(jsValueRef: rawBuf, in: context) } - fileprivate func toData() -> Data { + fileprivate func arrayBufferData() -> Data { let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) @@ -93,14 +94,26 @@ extension JSValue { fileprivate var promiseValue: JSValue { get async throws { try await withCheckedThrowingContinuation { continuation in - invokeMethod("then", withArguments: [ - JSValue(object: { val in - continuation.resume(returning: val) - } as @convention(block) (JSValue) -> Void, in: context)!, - JSValue(object: { err in - continuation.resume(throwing: PluginServerError(message: "\(err)")) - } as @convention(block) (JSValue) -> Void, in: context)! - ]) + typealias Handler = @convention(block) (JSValue) -> Void + let successFunc = JSValue( + object: { + continuation.resume(returning: $0) + } as Handler, + in: context + ) + let rejectFunc = JSValue( + object: { error in + continuation.resume( + throwing: PluginServerError(message: "\(error)") + ) + } as @convention(block) (JSValue) -> Void, + in: context + ) + guard let successFunc, let rejectFunc else { + continuation.resume(throwing: PluginServerError(message: "Could not await promise")) + return + } + invokeMethod("then", withArguments: [successFunc, rejectFunc]) if let exception = context.exception { continuation.resume(throwing: PluginServerError(message: "\(exception)")) } From 89158e207e1dc6766e772699a156baaaea79dc63 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 02:57:12 -0400 Subject: [PATCH 011/105] Create SwiftPluginServerSupport --- tools/swift-plugin-server/CMakeLists.txt | 11 ++ tools/swift-plugin-server/Package.swift | 20 ++- .../SwiftPluginServerSupport.swift | 137 ++++++++++++++++++ .../swift-plugin-server.swift | 124 +--------------- .../JSCWasmPlugin.swift | 1 + .../swift-wasm-plugin-server.swift | 126 +--------------- 6 files changed, 165 insertions(+), 254 deletions(-) create mode 100644 tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 3220ec79527e9..1d8e2dc63680e 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -11,6 +11,15 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/CSwiftPluginServer/include ) + add_pure_swift_host_library(SwiftPluginServerSupport EMIT_MODULE + Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift + DEPENDENCIES + $ + ) + target_include_directories(SwiftPluginServerSupport PRIVATE + Sources/CSwiftPluginServer/include + ) + add_pure_swift_host_tool(swift-plugin-server Sources/swift-plugin-server/swift-plugin-server.swift DEPENDENCIES @@ -23,6 +32,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SwiftSyntaxMacroExpansion SwiftCompilerPluginMessageHandling swiftLLVMJSON + SwiftPluginServerSupport ) target_include_directories(swift-plugin-server PRIVATE Sources/CSwiftPluginServer/include @@ -38,6 +48,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SWIFT_DEPENDENCIES SwiftCompilerPluginMessageHandling swiftLLVMJSON + SwiftPluginServerSupport ) target_include_directories(swift-wasm-plugin-server PRIVATE Sources/CSwiftPluginServer/include diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 1731850a9e7a1..af18c447740cb 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -29,25 +29,31 @@ let package = Package( ] ), .target( - name: "swift-plugin-server", + name: "SwiftPluginServerSupport", dependencies: [ .product(name: "swiftLLVMJSON", package: "ASTGen"), .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), - .product(name: "SwiftDiagnostics", package: "swift-syntax"), - .product(name: "SwiftSyntax", package: "swift-syntax"), - .product(name: "SwiftOperators", package: "swift-syntax"), - .product(name: "SwiftParser", package: "swift-syntax"), - .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), "CSwiftPluginServer" ], swiftSettings: [.interoperabilityMode(.Cxx)] ), + .target( + name: "swift-plugin-server", + dependencies: [ + .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + "CSwiftPluginServer", + "SwiftPluginServerSupport", + ], + swiftSettings: [.interoperabilityMode(.Cxx)] + ), .target( name: "swift-wasm-plugin-server", dependencies: [ .product(name: "swiftLLVMJSON", package: "ASTGen"), .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), - "CSwiftPluginServer" + "CSwiftPluginServer", + "SwiftPluginServerSupport", ], swiftSettings: [.interoperabilityMode(.Cxx)] ), diff --git a/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift b/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift new file mode 100644 index 0000000000000..bc8b72767f052 --- /dev/null +++ b/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift @@ -0,0 +1,137 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftCompilerPluginMessageHandling +import swiftLLVMJSON +import CSwiftPluginServer + +public final class PluginHostConnection: MessageConnection { + let handle: UnsafeRawPointer + public init() throws { + var errorMessage: UnsafePointer? = nil + guard let handle = PluginServer_createConnection(&errorMessage) else { + throw PluginServerError(message: String(cString: errorMessage!)) + } + self.handle = handle + } + + deinit { + PluginServer_destroyConnection(self.handle) + } + + public func sendMessage(_ message: TX) throws { + try LLVMJSON.encoding(message) { buffer in + try self.sendMessageData(buffer) + } + } + + public func waitForNextMessage(_ type: RX.Type) throws -> RX? { + return try self.withReadingMessageData { jsonData in + try LLVMJSON.decode(RX.self, from: jsonData) + } + } + + /// Send a serialized message to the message channel. + private func sendMessageData(_ data: UnsafeBufferPointer) throws { + // Write the header (a 64-bit length field in little endian byte order). + var header: UInt64 = UInt64(data.count).littleEndian + let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in + try self.write(buffer: UnsafeRawBufferPointer(buffer)) + } + guard writtenSize == MemoryLayout.size(ofValue: header) else { + throw PluginServerError(message: "failed to write message header") + } + + // Write the body. + guard try self.write(buffer: UnsafeRawBufferPointer(data)) == data.count else { + throw PluginServerError(message: "failed to write message body") + } + } + + /// Read a serialized message from the message channel and call the 'body' + /// with the data. + private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { + // Read the header (a 64-bit length field in little endian byte order). + var header: UInt64 = 0 + let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in + try self.read(into: UnsafeMutableRawBufferPointer(buffer)) + } + guard readSize == MemoryLayout.size(ofValue: header) else { + if readSize == 0 { + // The host closed the pipe. + return nil + } + // Otherwise, some error happened. + throw PluginServerError(message: "failed to read message header") + } + + // Read the body. + let count = Int(UInt64(littleEndian: header)) + let data = UnsafeMutableBufferPointer.allocate(capacity: count) + defer { data.deallocate() } + guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { + throw PluginServerError(message: "failed to read message body") + } + + // Invoke the handler. + return try body(UnsafeBufferPointer(data)) + } + + /// Write the 'buffer' to the message channel. + /// Returns the number of bytes succeeded to write. + private func write(buffer: UnsafeRawBufferPointer) throws -> Int { + var bytesToWrite = buffer.count + guard bytesToWrite > 0 else { + return 0 + } + var ptr = buffer.baseAddress! + + while (bytesToWrite > 0) { + let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) + if (writtenSize <= 0) { + // error e.g. broken pipe. + break + } + ptr = ptr.advanced(by: writtenSize) + bytesToWrite -= Int(writtenSize) + } + return buffer.count - bytesToWrite + } + + /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. + /// Returns the number of bytes succeeded to read. + private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { + var bytesToRead = buffer.count + guard bytesToRead > 0 else { + return 0 + } + var ptr = buffer.baseAddress! + + while bytesToRead > 0 { + let readSize = PluginServer_read(handle, ptr, bytesToRead) + if (readSize <= 0) { + // 0: EOF (the host closed), -1: Broken pipe (the host crashed?) + break; + } + ptr = ptr.advanced(by: readSize) + bytesToRead -= readSize + } + return buffer.count - bytesToRead + } +} + +public struct PluginServerError: Error, CustomStringConvertible { + public var description: String + public init(message: String) { + self.description = message + } +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 383a1b99491ed..1b3789c201175 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -12,7 +12,7 @@ @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling import SwiftSyntaxMacros -import swiftLLVMJSON +import SwiftPluginServerSupport import CSwiftPluginServer @main @@ -98,125 +98,3 @@ extension SwiftPluginServer: PluginProvider { [.loadPluginLibrary] } } - -final class PluginHostConnection: MessageConnection { - let handle: UnsafeRawPointer - init() throws { - var errorMessage: UnsafePointer? = nil - guard let handle = PluginServer_createConnection(&errorMessage) else { - throw PluginServerError(message: String(cString: errorMessage!)) - } - self.handle = handle - } - - deinit { - PluginServer_destroyConnection(self.handle) - } - - func sendMessage(_ message: TX) throws { - try LLVMJSON.encoding(message) { buffer in - try self.sendMessageData(buffer) - } - } - - func waitForNextMessage(_ type: RX.Type) throws -> RX? { - return try self.withReadingMessageData { jsonData in - try LLVMJSON.decode(RX.self, from: jsonData) - } - } - - /// Send a serialized message to the message channel. - private func sendMessageData(_ data: UnsafeBufferPointer) throws { - // Write the header (a 64-bit length field in little endian byte order). - var header: UInt64 = UInt64(data.count).littleEndian - let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in - try self.write(buffer: UnsafeRawBufferPointer(buffer)) - } - guard writtenSize == MemoryLayout.size(ofValue: header) else { - throw PluginServerError(message: "failed to write message header") - } - - // Write the body. - guard try self.write(buffer: UnsafeRawBufferPointer(data)) == data.count else { - throw PluginServerError(message: "failed to write message body") - } - } - - /// Read a serialized message from the message channel and call the 'body' - /// with the data. - private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { - // Read the header (a 64-bit length field in little endian byte order). - var header: UInt64 = 0 - let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in - try self.read(into: UnsafeMutableRawBufferPointer(buffer)) - } - guard readSize == MemoryLayout.size(ofValue: header) else { - if readSize == 0 { - // The host closed the pipe. - return nil - } - // Otherwise, some error happened. - throw PluginServerError(message: "failed to read message header") - } - - // Read the body. - let count = Int(UInt64(littleEndian: header)) - let data = UnsafeMutableBufferPointer.allocate(capacity: count) - defer { data.deallocate() } - guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { - throw PluginServerError(message: "failed to read message body") - } - - // Invoke the handler. - return try body(UnsafeBufferPointer(data)) - } - - /// Write the 'buffer' to the message channel. - /// Returns the number of bytes succeeded to write. - private func write(buffer: UnsafeRawBufferPointer) throws -> Int { - var bytesToWrite = buffer.count - guard bytesToWrite > 0 else { - return 0 - } - var ptr = buffer.baseAddress! - - while (bytesToWrite > 0) { - let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) - if (writtenSize <= 0) { - // error e.g. broken pipe. - break - } - ptr = ptr.advanced(by: writtenSize) - bytesToWrite -= Int(writtenSize) - } - return buffer.count - bytesToWrite - } - - /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. - /// Returns the number of bytes succeeded to read. - private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { - var bytesToRead = buffer.count - guard bytesToRead > 0 else { - return 0 - } - var ptr = buffer.baseAddress! - - while bytesToRead > 0 { - let readSize = PluginServer_read(handle, ptr, bytesToRead) - if (readSize <= 0) { - // 0: EOF (the host closed), -1: Broken pipe (the host crashed?) - break; - } - ptr = ptr.advanced(by: readSize) - bytesToRead -= readSize - } - return buffer.count - bytesToRead - } -} - -struct PluginServerError: Error, CustomStringConvertible { - var description: String - init(message: String) { - self.description = message - } -} diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 5de0cf1460d21..584c5cae5687b 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import JavaScriptCore +import SwiftPluginServerSupport // returns: Promise<(ArrayBuffer) => ArrayBuffer> private let js = """ diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 7bdef32d4544d..4be25c29e9b5e 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -11,9 +11,9 @@ //===----------------------------------------------------------------------===// import SwiftCompilerPluginMessageHandling -import swiftLLVMJSON import Foundation -import CSwiftPluginServer +import swiftLLVMJSON +import SwiftPluginServerSupport @available(macOS 10.15, *) @main @@ -101,125 +101,3 @@ final class SwiftPluginServer { protocol WasmPlugin { func handleMessage(_ json: Data) async throws -> Data } - -final class PluginHostConnection { - let handle: UnsafeRawPointer - init() throws { - var errorMessage: UnsafePointer? = nil - guard let handle = PluginServer_createConnection(&errorMessage) else { - throw PluginServerError(message: String(cString: errorMessage!)) - } - self.handle = handle - } - - deinit { - PluginServer_destroyConnection(self.handle) - } - - func sendMessage(_ message: TX) throws { - try LLVMJSON.encoding(message) { buffer in - try self.sendMessageData(buffer) - } - } - - func waitForNextMessage(_ type: RX.Type) throws -> RX? { - try self.withReadingMessageData { buffer in - try LLVMJSON.decode(RX.self, from: buffer) - } - } - - /// Send a serialized message to the message channel. - func sendMessageData(_ data: UnsafeBufferPointer) throws { - // Write the header (a 64-bit length field in little endian byte order). - var header: UInt64 = UInt64(data.count).littleEndian - let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in - try self.write(buffer: UnsafeRawBufferPointer(buffer)) - } - guard writtenSize == MemoryLayout.size(ofValue: header) else { - throw PluginServerError(message: "failed to write message header") - } - - // Write the body. - guard try self.write(buffer: UnsafeRawBufferPointer(data)) == data.count else { - throw PluginServerError(message: "failed to write message body") - } - } - - /// Read a serialized message from the message channel and call the 'body' - /// with the data. - private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { - // Read the header (a 64-bit length field in little endian byte order). - var header: UInt64 = 0 - let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in - try self.read(into: UnsafeMutableRawBufferPointer(buffer)) - } - guard readSize == MemoryLayout.size(ofValue: header) else { - if readSize == 0 { - // The host closed the pipe. - return nil - } - // Otherwise, some error happened. - throw PluginServerError(message: "failed to read message header") - } - - // Read the body. - let count = Int(UInt64(littleEndian: header)) - let data = UnsafeMutableBufferPointer.allocate(capacity: count) - defer { data.deallocate() } - guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { - throw PluginServerError(message: "failed to read message body") - } - - // Invoke the handler. - return try body(UnsafeBufferPointer(data)) - } - - /// Write the 'buffer' to the message channel. - /// Returns the number of bytes succeeded to write. - private func write(buffer: UnsafeRawBufferPointer) throws -> Int { - var bytesToWrite = buffer.count - guard bytesToWrite > 0 else { - return 0 - } - var ptr = buffer.baseAddress! - - while (bytesToWrite > 0) { - let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) - if (writtenSize <= 0) { - // error e.g. broken pipe. - break - } - ptr = ptr.advanced(by: writtenSize) - bytesToWrite -= Int(writtenSize) - } - return buffer.count - bytesToWrite - } - - /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. - /// Returns the number of bytes succeeded to read. - private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { - var bytesToRead = buffer.count - guard bytesToRead > 0 else { - return 0 - } - var ptr = buffer.baseAddress! - - while bytesToRead > 0 { - let readSize = PluginServer_read(handle, ptr, bytesToRead) - if (readSize <= 0) { - // 0: EOF (the host closed), -1: Broken pipe (the host crashed?) - break; - } - ptr = ptr.advanced(by: readSize) - bytesToRead -= readSize - } - return buffer.count - bytesToRead - } -} - -struct PluginServerError: Error, CustomStringConvertible { - var description: String - init(message: String) { - self.description = message - } -} From f084e25a37c54911cdef8d557f1c2c00ec24a7ce Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 13:53:37 -0400 Subject: [PATCH 012/105] JSCWasmPlugin: memoize, improve errors --- .../JSCWasmPlugin.swift | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 584c5cae5687b..1b288f25dfbe8 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -11,11 +11,10 @@ //===----------------------------------------------------------------------===// import JavaScriptCore -import SwiftPluginServerSupport -// returns: Promise<(ArrayBuffer) => ArrayBuffer> +// returns: (wasm: ArrayBuffer) => Promise<(input: ArrayBuffer) => ArrayBuffer> private let js = """ -(async () => { +async (wasmData) => { const mod = await WebAssembly.compile(wasmData); // stub WASI imports const imports = WebAssembly.Module.imports(mod) @@ -38,39 +37,49 @@ private let js = """ api.wacro_free(outAddr); return copy.buffer; } -})() +} """ @available(macOS 10.15, *) final class JSCWasmPlugin: WasmPlugin { + private static let factory = JSContext()?.evaluateScript(js) + private let handler: JSValue @MainActor init(wasm data: Data) async throws { - guard let context = JSContext() else { throw PluginServerError(message: "Could not create JSContext") } - - let jsBuf = try JSValue(newBufferWithData: data, in: context) - context.globalObject.setObject(jsBuf, forKeyedSubscript: "wasmData") - - guard let promise = context.evaluateScript(js) else { - throw PluginServerError(message: "Failed to load plugin") + guard let factory = Self.factory, let context = factory.context else { + throw JSCWasmError(message: "Failed to load plugin") } - if let error = context.exception { - throw PluginServerError(message: "Failed to load plugin: \(error)") + let jsBuf = try JSValue(newBufferWithData: data, in: context) + guard let promise = factory.call(withArguments: [jsBuf]), context.exception == nil else { + throw JSCWasmError(message: "Failed to load plugin", value: context.exception) } handler = try await promise.promiseValue } @MainActor func handleMessage(_ json: Data) throws -> Data { - let jsonJS = try JSValue(newBufferWithData: json, in: handler.context) + guard let context = handler.context else { + throw JSCWasmError(message: "Failed to invoke plugin") + } + let jsonJS = try JSValue(newBufferWithData: json, in: context) guard let result = handler.call(withArguments: [jsonJS]) else { - throw PluginServerError(message: "Wasm plugin did not provide a valid response") + throw JSCWasmError(message: "Wasm plugin did not provide a valid response", value: context.exception) } return result.arrayBufferData() } } +public struct JSCWasmError: Error, CustomStringConvertible { + public let value: JSValue? + public let description: String + public init(message: String, value: JSValue? = nil) { + self.value = value + self.description = "\(message)\(value.map { ": \($0)" } ?? "")" + } +} + extension JSValue { fileprivate convenience init(newBufferWithData data: Data, in context: JSContext) throws { let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) @@ -105,18 +114,18 @@ extension JSValue { let rejectFunc = JSValue( object: { error in continuation.resume( - throwing: PluginServerError(message: "\(error)") + throwing: JSCWasmError(message: "Promise rejected", value: error) ) } as @convention(block) (JSValue) -> Void, in: context ) guard let successFunc, let rejectFunc else { - continuation.resume(throwing: PluginServerError(message: "Could not await promise")) + continuation.resume(throwing: JSCWasmError(message: "Could not await promise")) return } invokeMethod("then", withArguments: [successFunc, rejectFunc]) if let exception = context.exception { - continuation.resume(throwing: PluginServerError(message: "\(exception)")) + continuation.resume(throwing: JSCWasmError(message: "Promise.then threw", value: exception)) } } } From 5ef86da855d68295ae42116b8c696845c36af5cd Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 15:22:06 -0400 Subject: [PATCH 013/105] steps towards x-plat --- tools/swift-plugin-server/CMakeLists.txt | 8 +++++--- .../swift-wasm-plugin-server.swift | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 1d8e2dc63680e..d74a4814a055b 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -53,7 +53,9 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) target_include_directories(swift-wasm-plugin-server PRIVATE Sources/CSwiftPluginServer/include ) - add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND - codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist - ) + if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) + add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND + codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist + ) + endif() endif() diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 4be25c29e9b5e..9c7c33d830a19 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -15,7 +15,6 @@ import Foundation import swiftLLVMJSON import SwiftPluginServerSupport -@available(macOS 10.15, *) @main final class SwiftPluginServer { let connection: PluginHostConnection @@ -31,6 +30,8 @@ final class SwiftPluginServer { } @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { + // TODO: fall back to WasmKit for non-macOS and macOS < 10.15 + guard #available(macOS 10.15, *) else { throw PluginServerError(message: "Wasm plugins currently require macOS 10.15+") } guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) return try await JSCWasmPlugin(wasm: wasm) From 02f4324214cc13f35b82b102573742f9e602e221 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Apr 2024 16:15:32 -0400 Subject: [PATCH 014/105] more modularity --- .../swift-wasm-plugin-server/JSCWasmPlugin.swift | 11 ++++++++++- .../swift-wasm-plugin-server.swift | 9 +++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 1b288f25dfbe8..1de4a1edc6b87 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -10,8 +10,12 @@ // //===----------------------------------------------------------------------===// +#if os(macOS) + import JavaScriptCore +typealias DefaultWasmPlugin = JSCWasmPlugin + // returns: (wasm: ArrayBuffer) => Promise<(input: ArrayBuffer) => ArrayBuffer> private let js = """ async (wasmData) => { @@ -40,13 +44,16 @@ async (wasmData) => { } """ -@available(macOS 10.15, *) final class JSCWasmPlugin: WasmPlugin { private static let factory = JSContext()?.evaluateScript(js) private let handler: JSValue @MainActor init(wasm data: Data) async throws { + guard #available(macOS 10.15, *) else { + throw JSCWasmError(message: "JSC Wasm plugins currently require macOS 10.15+") + } + guard let factory = Self.factory, let context = factory.context else { throw JSCWasmError(message: "Failed to load plugin") } @@ -131,3 +138,5 @@ extension JSValue { } } } + +#endif diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 9c7c33d830a19..2fc622a233703 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -29,12 +29,10 @@ final class SwiftPluginServer { try connection.sendMessage(message) } - @MainActor private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - // TODO: fall back to WasmKit for non-macOS and macOS < 10.15 - guard #available(macOS 10.15, *) else { throw PluginServerError(message: "Wasm plugins currently require macOS 10.15+") } + private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - return try await JSCWasmPlugin(wasm: wasm) + return try await defaultWasmPlugin.init(wasm: wasm) } private func expandMacro( @@ -100,5 +98,8 @@ final class SwiftPluginServer { } protocol WasmPlugin { + init(wasm: Data) async throws func handleMessage(_ json: Data) async throws -> Data } + +private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } From bddde66935c98d485e290712a28711682193b026 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 16 Apr 2024 01:02:58 -0400 Subject: [PATCH 015/105] Add wasm-plugin-server-path frontend opt --- include/swift/AST/SearchPathOptions.h | 3 +++ include/swift/Option/FrontendOptions.td | 5 +++++ lib/AST/PluginLoader.cpp | 9 ++++----- lib/Frontend/CompilerInvocation.cpp | 4 ++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index dd7debcce1351..8f0242314ae6e 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -468,6 +468,9 @@ class SearchPathOptions { /// Plugin search path options. std::vector PluginSearchOpts; + /// Path to swift-wasm-plugin-server executable. + std::string PluginWasmServerPath; + /// Don't look in for compiler-provided modules. bool SkipRuntimeLibraryImportPaths = false; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index ed82e9cba1c5a..fd08b2e31769a 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -277,6 +277,11 @@ def Raccess_note_EQ : Joined<["-"], "Raccess-note=">, def block_list_file : Separate<["-"], "blocklist-file">, MetaVarName<"">, HelpText<"The path to a blocklist configuration file">; + +def wasm_plugin_server_path : Separate<["-"], "wasm-plugin-server-path">, + Flags<[DoesNotAffectIncrementalBuild, ArgumentIsPath]>, + HelpText<"Path to the swift-wasm-plugin-server executable">, + MetaVarName<"">; } // end let Flags = [FrontendOption, NoDriverOption] def debug_crash_Group : OptionGroup<"">; diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index e6a6a6e5d87fd..fc9f6f81084cd 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -108,12 +108,11 @@ PluginLoader::getPluginMap() { assert(!val.ExecutablePath.empty() && "empty plugin path"); if (llvm::sys::path::filename(val.ExecutablePath).ends_with(".wasm")) { // we treat wasm plugins like library plugins that can be loaded by an external - // "macro runner" executable - SmallString<128> runner; - // TODO: improve path resolution: we really want tools_dir - llvm::sys::path::append(runner, Ctx.SearchPathOpts.RuntimeResourcePath, "../../bin/swift-wasm-plugin-server"); + // "wasm server" that in turn invokes the wasm runtime. + const auto &wasmServerPath = Ctx.SearchPathOpts.PluginWasmServerPath; + assert(!wasmServerPath.empty() && "wasm load requested but got empty wasm server path"); for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, val.ExecutablePath, runner); + try_emplace(moduleName, val.ExecutablePath, wasmServerPath); } } else { for (auto &moduleName : val.ModuleNames) { diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 5bcbaaed78815..3d46ab5356243 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1962,6 +1962,10 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, } } + if (const Arg *A = Args.getLastArg(OPT_wasm_plugin_server_path)) { + Opts.PluginWasmServerPath = A->getValue(); + } + for (const Arg *A : Args.filtered(OPT_L)) { Opts.LibrarySearchPaths.push_back(resolveSearchPath(A->getValue())); } From ca22e2db06f4e33f88cbca957f52d5583310e406 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 16 Apr 2024 02:06:59 -0400 Subject: [PATCH 016/105] Start adding WasmKit support --- tools/swift-plugin-server/.gitignore | 1 + tools/swift-plugin-server/CMakeLists.txt | 2 + tools/swift-plugin-server/Package.swift | 3 + .../JSCWasmPlugin.swift | 2 +- .../WasmKitPlugin.swift | 63 +++++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tools/swift-plugin-server/.gitignore create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift diff --git a/tools/swift-plugin-server/.gitignore b/tools/swift-plugin-server/.gitignore new file mode 100644 index 0000000000000..f4096fee41368 --- /dev/null +++ b/tools/swift-plugin-server/.gitignore @@ -0,0 +1 @@ +Package.resolved diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index d74a4814a055b..53596169dac4b 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -57,5 +57,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist ) + else() + # TODO: Add dependency on WasmKit endif() endif() diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index af18c447740cb..b078eb1fc79b1 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -14,6 +14,7 @@ let package = Package( dependencies: [ .package(path: "../../../swift-syntax"), .package(path: "../../lib/ASTGen"), + .package(path: "../../../wasmkit"), ], targets: [ .target( @@ -54,6 +55,8 @@ let package = Package( .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), "CSwiftPluginServer", "SwiftPluginServerSupport", + .product(name: "WasmKit", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), + .product(name: "WASI", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), ], swiftSettings: [.interoperabilityMode(.Cxx)] ), diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 1de4a1edc6b87..331c2dd800ddd 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -44,7 +44,7 @@ async (wasmData) => { } """ -final class JSCWasmPlugin: WasmPlugin { +struct JSCWasmPlugin: WasmPlugin { private static let factory = JSContext()?.evaluateScript(js) private let handler: JSValue diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift new file mode 100644 index 0000000000000..e5de34f588e1c --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if !os(macOS) + +import Foundation +import WasmKit +import WASI + +typealias DefaultWasmPlugin = WasmKitPlugin + +struct WasmKitMacroRunner: WasmPlugin { + let instance: ModuleInstance + let runtime: Runtime + + init(wasm: Data) throws { + let module = try parseWasm(bytes: Array(wasm)) + let bridge = try WASIBridgeToHost() + runtime = Runtime(hostModules: bridge.hostModules) + instance = try runtime.instantiate(module: module) + _ = try bridge.start(instance, runtime: runtime) + } + + func handleMessage(_ json: Data) throws -> Data { + let exports = instance.exports + guard case let .memory(memoryAddr) = exports["memory"] else { fatalError("bad memory") } + guard case let .function(malloc) = exports["wacro_malloc"] else { fatalError("bad wacro_malloc") } + guard case let .function(parse) = exports["wacro_parse"] else { fatalError("bad wacro_parse") } + guard case let .function(free) = exports["wacro_free"] else { fatalError("bad wacro_free") } + + let inAddr = try malloc.invoke([.i32(UInt32(json.count))], runtime: runtime)[0].i32 + + runtime.store.withMemory(at: memoryAddr) { mem in + mem.data.replaceSubrange(Int(inAddr)..<(Int(inAddr) + json.count), with: json) + } + + let outAddr = try parse.invoke([.i32(inAddr), .i32(UInt32(json.count))], runtime: runtime)[0].i32 + let out = runtime.store.withMemory(at: memoryAddr) { mem in + let bytes = Array(mem.data[Int(outAddr)..<(Int(outAddr) + 4)]) + let len = + (UInt32(bytes[0]) << 0) | + (UInt32(bytes[1]) << 8) | + (UInt32(bytes[2]) << 16) | + (UInt32(bytes[3]) << 24) + return Data(mem.data[(Int(outAddr) + 4)...].prefix(Int(len))) + } + + _ = try free.invoke([.i32(outAddr)], runtime: runtime) + + return out + } +} + +#endif From 6a9bbe7e5eaf067b25fc2c2187c0cb706ccb2ceb Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 16 Apr 2024 02:26:36 -0400 Subject: [PATCH 017/105] Update WasmKitPlugin --- .../WasmKitPlugin.swift | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift index e5de34f588e1c..f0c8cfe77c44e 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift @@ -18,12 +18,13 @@ import WASI typealias DefaultWasmPlugin = WasmKitPlugin -struct WasmKitMacroRunner: WasmPlugin { +struct WasmKitPlugin: WasmPlugin { let instance: ModuleInstance let runtime: Runtime init(wasm: Data) throws { let module = try parseWasm(bytes: Array(wasm)) + // TODO: we should air-gap this bridge. Wasm macros don't need IO. let bridge = try WASIBridgeToHost() runtime = Runtime(hostModules: bridge.hostModules) instance = try runtime.instantiate(module: module) @@ -36,23 +37,18 @@ struct WasmKitMacroRunner: WasmPlugin { guard case let .function(malloc) = exports["wacro_malloc"] else { fatalError("bad wacro_malloc") } guard case let .function(parse) = exports["wacro_parse"] else { fatalError("bad wacro_parse") } guard case let .function(free) = exports["wacro_free"] else { fatalError("bad wacro_free") } - - let inAddr = try malloc.invoke([.i32(UInt32(json.count))], runtime: runtime)[0].i32 - - runtime.store.withMemory(at: memoryAddr) { mem in - mem.data.replaceSubrange(Int(inAddr)..<(Int(inAddr) + json.count), with: json) - } - - let outAddr = try parse.invoke([.i32(inAddr), .i32(UInt32(json.count))], runtime: runtime)[0].i32 - let out = runtime.store.withMemory(at: memoryAddr) { mem in - let bytes = Array(mem.data[Int(outAddr)..<(Int(outAddr) + 4)]) - let len = - (UInt32(bytes[0]) << 0) | - (UInt32(bytes[1]) << 8) | - (UInt32(bytes[2]) << 16) | - (UInt32(bytes[3]) << 24) - return Data(mem.data[(Int(outAddr) + 4)...].prefix(Int(len))) - } + let memory = GuestMemory(store: runtime.store, address: memoryAddr) + + let jsonLen = UInt32(json.count) + let inAddr = try malloc.invoke([.i32(jsonLen)], runtime: runtime)[0].i32 + let rawInAddr = UnsafeGuestPointer(memorySpace: memory, offset: inAddr) + _ = UnsafeGuestBufferPointer(baseAddress: rawInAddr, count: jsonLen) + .withHostPointer { $0.initialize(from: json) } + + let outAddr = try parse.invoke([.i32(inAddr), .i32(jsonLen)], runtime: runtime)[0].i32 + let outLen = UnsafeGuestPointer(memorySpace: memory, offset: outAddr).pointee + let outBase = UnsafeGuestPointer(memorySpace: memory, offset: outAddr + 4) + let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen).withHostPointer { Data($0) } _ = try free.invoke([.i32(outAddr)], runtime: runtime) From e598064595dc22a6ff87aee4b2b6925c15fa8227 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 16 Apr 2024 02:44:32 -0400 Subject: [PATCH 018/105] safety, tweaks --- .../WasmKitPlugin.swift | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift index f0c8cfe77c44e..01597cfbe2350 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift @@ -21,6 +21,7 @@ typealias DefaultWasmPlugin = WasmKitPlugin struct WasmKitPlugin: WasmPlugin { let instance: ModuleInstance let runtime: Runtime + let memory: GuestMemory init(wasm: Data) throws { let module = try parseWasm(bytes: Array(wasm)) @@ -29,31 +30,38 @@ struct WasmKitPlugin: WasmPlugin { runtime = Runtime(hostModules: bridge.hostModules) instance = try runtime.instantiate(module: module) _ = try bridge.start(instance, runtime: runtime) - } - func handleMessage(_ json: Data) throws -> Data { let exports = instance.exports - guard case let .memory(memoryAddr) = exports["memory"] else { fatalError("bad memory") } - guard case let .function(malloc) = exports["wacro_malloc"] else { fatalError("bad wacro_malloc") } - guard case let .function(parse) = exports["wacro_parse"] else { fatalError("bad wacro_parse") } - guard case let .function(free) = exports["wacro_free"] else { fatalError("bad wacro_free") } - let memory = GuestMemory(store: runtime.store, address: memoryAddr) + guard case let .memory(memoryAddr) = exports["memory"] else { + throw WasmKitPluginError(message: "Wasm plugin does not export a valid memory.") + } + self.memory = GuestMemory(store: runtime.store, address: memoryAddr) + } + func handleMessage(_ json: Data) throws -> Data { let jsonLen = UInt32(json.count) - let inAddr = try malloc.invoke([.i32(jsonLen)], runtime: runtime)[0].i32 + let inAddr = try runtime.invoke(instance, function: "wacro_malloc", with: [.i32(jsonLen)])[0].i32 let rawInAddr = UnsafeGuestPointer(memorySpace: memory, offset: inAddr) _ = UnsafeGuestBufferPointer(baseAddress: rawInAddr, count: jsonLen) .withHostPointer { $0.initialize(from: json) } - let outAddr = try parse.invoke([.i32(inAddr), .i32(jsonLen)], runtime: runtime)[0].i32 + let outAddr = try runtime.invoke(instance, function: "wacro_parse", with: [.i32(inAddr), .i32(jsonLen)])[0].i32 let outLen = UnsafeGuestPointer(memorySpace: memory, offset: outAddr).pointee let outBase = UnsafeGuestPointer(memorySpace: memory, offset: outAddr + 4) - let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen).withHostPointer { Data($0) } + let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen) + .withHostPointer { Data($0) } - _ = try free.invoke([.i32(outAddr)], runtime: runtime) + _ = try runtime.invoke(instance, function: "wacro_free", with: [.i32(outAddr)]) return out } } +struct WasmKitPluginError: Error, CustomStringConvertible { + let description: String + init(message: String) { + self.description = message + } +} + #endif From 7d1510ec42d8c9f3902fefba9a11f1510ab3a781 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 16 Apr 2024 18:10:02 -0400 Subject: [PATCH 019/105] tweaks to JSC --- .../JSCWasmPlugin.swift | 62 +++++++++++++++---- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift index 331c2dd800ddd..89baa152aa3f1 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift @@ -17,6 +17,10 @@ import JavaScriptCore typealias DefaultWasmPlugin = JSCWasmPlugin // returns: (wasm: ArrayBuffer) => Promise<(input: ArrayBuffer) => ArrayBuffer> +// FIXME: stubbing WASI by always returning 0 triggers UB in the guest. +// (e.g. if we return 0 from args_sizes_get we must populate the out pointers.) +// we could adapt WasmKit.WASI to work with the JSC engine, since the host +// function code should be runtime-agnostic. private let js = """ async (wasmData) => { const mod = await WebAssembly.compile(wasmData); @@ -45,28 +49,25 @@ async (wasmData) => { """ struct JSCWasmPlugin: WasmPlugin { - private static let factory = JSContext()?.evaluateScript(js) - private let handler: JSValue - @MainActor init(wasm data: Data) async throws { + init(wasm data: Data) async throws { guard #available(macOS 10.15, *) else { throw JSCWasmError(message: "JSC Wasm plugins currently require macOS 10.15+") } - guard let factory = Self.factory, let context = factory.context else { - throw JSCWasmError(message: "Failed to load plugin") - } + let factory = try await JSCWasmFactory.shared + let context = factory.context let jsBuf = try JSValue(newBufferWithData: data, in: context) - guard let promise = factory.call(withArguments: [jsBuf]), context.exception == nil else { + guard let promise = factory.value.call(withArguments: [jsBuf]), context.exception == nil else { throw JSCWasmError(message: "Failed to load plugin", value: context.exception) } handler = try await promise.promiseValue } - @MainActor func handleMessage(_ json: Data) throws -> Data { + func handleMessage(_ json: Data) throws -> Data { guard let context = handler.context else { throw JSCWasmError(message: "Failed to invoke plugin") } @@ -78,6 +79,32 @@ struct JSCWasmPlugin: WasmPlugin { } } +@available(macOS 10.15, *) +private struct JSCWasmFactory { + let context: JSContext + let value: JSValue + + private init() async throws { + // the VM must be created on the MainActor for async methods to work. + let vm = await MainActor.run { JSVirtualMachine() } + guard let context = JSContext(virtualMachine: vm) else { + throw JSCWasmError(message: "Failed to load plugin") + } + self.context = context + self.value = context.evaluateScript(js) + } + + private static let _shared = Task { + try await JSCWasmFactory() + } + + static var shared: JSCWasmFactory { + get async throws { + try await _shared.value + } + } +} + public struct JSCWasmError: Error, CustomStringConvertible { public let value: JSValue? public let description: String @@ -100,11 +127,22 @@ extension JSValue { self.init(jsValueRef: rawBuf, in: context) } + // it's unsafe to call JavaScriptCore APIs inside the `perform` block as this + // might invalidate the buffer. + fileprivate func withUnsafeArrayBuffer( + _ perform: (UnsafeRawBufferPointer) throws -> Result + ) rethrows -> Result { + let rawContext = context.jsGlobalContextRef + guard let base = JSObjectGetArrayBufferBytesPtr(rawContext, jsValueRef, nil) else { + return try perform(UnsafeRawBufferPointer(start: nil, count: 0)) + } + let count = JSObjectGetArrayBufferByteLength(rawContext, jsValueRef, nil) + let buffer = UnsafeRawBufferPointer(start: base, count: count) + return try perform(buffer) + } + fileprivate func arrayBufferData() -> Data { - let base = JSObjectGetArrayBufferBytesPtr(context.jsGlobalContextRef, jsValueRef, nil) - let count = JSObjectGetArrayBufferByteLength(context.jsGlobalContextRef, jsValueRef, nil) - let buf = UnsafeBufferPointer(start: base?.assumingMemoryBound(to: UInt8.self), count: count) - return Data(buf) + withUnsafeArrayBuffer { Data($0) } } @available(macOS 10.15, *) From da74f386ed5c7a474f0b3ad9ea827998d441920d Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 21 Apr 2024 22:50:11 -0400 Subject: [PATCH 020/105] Proper WasmKit integration --- tools/swift-plugin-server/CMakeLists.txt | 15 +- tools/swift-plugin-server/Package.swift | 8 +- .../SwiftPluginServerSupport.swift | 4 +- .../swift-plugin-server.swift | 2 +- .../JSCWasmEngine.swift | 236 ++++++++++++++++++ .../JSCWasmPlugin.swift | 180 ------------- .../swift-wasm-plugin-server/WasmEngine.swift | 58 +++++ .../WasmKitEngine.swift | 47 ++++ .../WasmKitPlugin.swift | 67 ----- .../swift-wasm-plugin-server.swift | 4 +- 10 files changed, 364 insertions(+), 257 deletions(-) create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift delete mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift create mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift delete mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 53596169dac4b..33830afe75164 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,4 +1,14 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) + # hack: override macOS deployment target to 10.15.4 + # since that's the minimum required by WasmKit + set(SWIFT_DARWIN_DEPLOYMENT_VERSION_OSX "10.15.4") + include(DarwinSDKs) + swift_get_host_triple(SWIFT_HOST_TRIPLE) + + file(TO_CMAKE_PATH "${SWIFT_PATH_TO_WASMKIT_SOURCE}" wasmkit_path) + FetchContent_Declare(WasmKit SOURCE_DIR "${wasmkit_path}") + FetchContent_MakeAvailable(WasmKit) + # _swiftCSwiftPluginServer is just a C support library for swift-plugin-server # Don't bother to create '.a' for that. add_swift_host_library(_swiftCSwiftPluginServer OBJECT @@ -40,7 +50,9 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) add_pure_swift_host_tool(swift-wasm-plugin-server Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift - Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift + Sources/swift-wasm-plugin-server/JSCWasmEngine.swift + Sources/swift-wasm-plugin-server/WasmEngine.swift + Sources/swift-wasm-plugin-server/WasmKitEngine.swift DEPENDENCIES $ SWIFT_COMPONENT @@ -49,6 +61,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SwiftCompilerPluginMessageHandling swiftLLVMJSON SwiftPluginServerSupport + WASI WasmKit WasmKitWASI WasmTypes ) target_include_directories(swift-wasm-plugin-server PRIVATE Sources/CSwiftPluginServer/include diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index b078eb1fc79b1..6f3d3e3a9ce34 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -5,7 +5,7 @@ import PackageDescription let package = Package( name: "swift-plugin-server", platforms: [ - .macOS(.v10_15) + .macOS(.v11), ], products: [ .library(name: "swift-plugin-server", targets: ["swift-plugin-server"]), @@ -34,7 +34,7 @@ let package = Package( dependencies: [ .product(name: "swiftLLVMJSON", package: "ASTGen"), .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), - "CSwiftPluginServer" + "CSwiftPluginServer", ], swiftSettings: [.interoperabilityMode(.Cxx)] ), @@ -55,8 +55,8 @@ let package = Package( .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), "CSwiftPluginServer", "SwiftPluginServerSupport", - .product(name: "WasmKit", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), - .product(name: "WASI", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), + .product(name: "WASI", package: "WasmKit"), + .product(name: "WasmKitWASI", package: "WasmKit"), ], swiftSettings: [.interoperabilityMode(.Cxx)] ), diff --git a/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift b/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift index bc8b72767f052..2750196560424 100644 --- a/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift +++ b/tools/swift-plugin-server/Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift @@ -10,11 +10,11 @@ // //===----------------------------------------------------------------------===// -import SwiftCompilerPluginMessageHandling +@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling import swiftLLVMJSON import CSwiftPluginServer -public final class PluginHostConnection: MessageConnection { +@_spi(PluginMessage) public final class PluginHostConnection: MessageConnection { let handle: UnsafeRawPointer public init() throws { var errorMessage: UnsafePointer? = nil diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 1b3789c201175..91a1c9e9002ef 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -11,8 +11,8 @@ //===----------------------------------------------------------------------===// @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling +@_spi(PluginMessage) import SwiftPluginServerSupport import SwiftSyntaxMacros -import SwiftPluginServerSupport import CSwiftPluginServer @main diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift new file mode 100644 index 0000000000000..0a1e186927f1c --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -0,0 +1,236 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if os(macOS) + +import JavaScriptCore +import WASI +import WasmTypes + +// typealias DefaultWasmEngine = JSCWasmEngine + +// (wasm: ArrayBuffer, imports: Object) => Promise<{ ... }> +private let js = """ +async (wasmData, imports) => { + const mod = await WebAssembly.compile(wasmData); + const instance = await WebAssembly.instantiate(mod, imports); + const api = instance.exports; + return { + api, + read: (off, size) => new Uint8Array(new Uint8Array(api.memory.buffer, off, size)).buffer, + write: (buf, off, size) => new Uint8Array(api.memory.buffer, off, size).set(new Uint8Array(buf)), + }; +} +""" + +final class JSCWasmEngine: WasmEngine { + private let _memory: JSCGuestMemory + private let api: JSValue + + var memory: some GuestMemory { _memory } + + init(wasm data: Data, imports: WASIBridgeToHost) async throws { + let factory = try await JSCWasmFactory.shared + let context = factory.context + + var memory: JSCGuestMemory? + let imports: [String: [String: JSValue]] = imports.wasiHostModules.mapValues { module in + module.functions.mapValues { function in + function.asJSFunction(in: context) { + guard let memory else { throw JSCWasmError(message: "Missing GuestMemory") } + return memory + } + } + } + + let jsBuf = try JSValue(newArrayBufferWithData: data, in: context) + let promise = factory.value.call(withArguments: [jsBuf, imports]) + guard let promise, context.exception == nil else { + throw JSCWasmError(message: "Failed to load plugin", value: context.exception) + } + + let runner = try await promise.promiseValue + + api = runner.objectForKeyedSubscript("api")! + + let getMemory = runner.objectForKeyedSubscript("read")! + let setMemory = runner.objectForKeyedSubscript("write")! + self._memory = JSCGuestMemory(getMemory: getMemory, setMemory: setMemory) + memory = self._memory + } + + func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] { + let result = api.invokeMethod(method, withArguments: args)! + if let exception = api.context.exception { + throw JSCWasmError(message: "Call to \(method) failed", value: exception) + } + return result.isUndefined ? [] : [result.toUInt32()] + } +} + +private struct JSCGuestMemory: GuestMemory { + let getMemory: JSValue + let setMemory: JSValue + + func withUnsafeMutableBufferPointer( + offset: UInt, + count: Int, + _ body: (UnsafeMutableRawBufferPointer) throws -> T + ) rethrows -> T { + // note that we can't get a shared pointer to the original wasm memory + // because JSC doesn't allow JSObjectGetArrayBufferBytesPtr on Wasm memory. + // we perform targeted copies instead. + let memory = getMemory.call(withArguments: [offset, count])! + defer { setMemory.call(withArguments: [memory, offset, count]) } + return try memory.withUnsafeArrayBuffer(body) + } +} + +private struct JSCWasmFactory { + let context: JSContext + let value: JSValue + + private init() async throws { + // the VM must be created on the MainActor for async methods to work. + let vm = await MainActor.run { JSVirtualMachine() } + guard let context = JSContext(virtualMachine: vm) else { + throw JSCWasmError(message: "Failed to load plugin") + } + self.context = context + self.value = context.evaluateScript(js) + } + + private static let _shared = Task { + try await JSCWasmFactory() + } + + static var shared: JSCWasmFactory { + get async throws { + try await _shared.value + } + } +} + +public struct JSCWasmError: Error, CustomStringConvertible { + public let value: JSValue? + public let description: String + public init(message: String, value: JSValue? = nil) { + self.value = value + self.description = "\(message)\(value.map { ": \($0)" } ?? "")" + } +} + +extension WASIHostFunction { + func asJSFunction(in context: JSContext, memory: @escaping () throws -> GuestMemory) -> JSValue { + JSValue(object: { + let arguments = JSContext.currentArguments() as? [JSValue] ?? [] + let types: [Value] = zip(arguments, type.parameters).map { argument, type in + switch type { + case .i32: .i32(argument.toUInt32()) + case .i64: .i64(UInt64(argument.toString()) ?? 0) + case .f32, .f64, .ref: .i32(0) // WASI doesn't support these + } + } + do { + let memory = try memory() + if let result = try self.implementation(memory, types).first { + return JSValue(uInt32: result.i32, in: context) + } + } catch { + context.exception = JSValue(object: "\(error)", in: context) + } + return JSValue(undefinedIn: context) + } as @convention(block) () -> JSValue, in: context)! + } +} + +extension JSValue { + fileprivate convenience init( + newArrayBufferWithData data: Data, + in context: JSContext + ) throws { + let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) + _ = copy.initialize(from: data) + let address = UInt(bitPattern: copy.baseAddress) + try self.init( + newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer(copy), + deallocator: { UnsafeMutableRawPointer(bitPattern: address)?.deallocate() }, + in: context + ) + } + + fileprivate convenience init( + newArrayBufferWithBytesNoCopy bytes: UnsafeMutableRawBufferPointer, + deallocator: @escaping @Sendable () -> Void, + in context: JSContext + ) throws { + let box = Unmanaged.passRetained(deallocator as AnyObject).toOpaque() + let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( + context.jsGlobalContextRef, + bytes.baseAddress, bytes.count, + { _, box in (Unmanaged.fromOpaque(box!).takeRetainedValue() as! () -> Void)() }, + box, + nil + ) + self.init(jsValueRef: rawBuf, in: context) + } + + // it's unsafe to call JavaScriptCore APIs inside the `perform` block as this + // might invalidate the buffer. + fileprivate func withUnsafeArrayBuffer( + _ perform: (UnsafeMutableRawBufferPointer) throws -> Result + ) rethrows -> Result { + let rawContext = context.jsGlobalContextRef + guard let base = JSObjectGetArrayBufferBytesPtr(rawContext, jsValueRef, nil) else { + return try perform(UnsafeMutableRawBufferPointer(start: nil, count: 0)) + } + let count = JSObjectGetArrayBufferByteLength(rawContext, jsValueRef, nil) + let buffer = UnsafeMutableRawBufferPointer(start: base, count: count) + return try perform(buffer) + } + + fileprivate func arrayBufferData() -> Data { + withUnsafeArrayBuffer { Data($0) } + } + + fileprivate var promiseValue: JSValue { + get async throws { + try await withCheckedThrowingContinuation { continuation in + typealias Handler = @convention(block) (JSValue) -> Void + let successFunc = JSValue( + object: { + continuation.resume(returning: $0) + } as Handler, + in: context + ) + let rejectFunc = JSValue( + object: { error in + continuation.resume( + throwing: JSCWasmError(message: "Promise rejected", value: error) + ) + } as @convention(block) (JSValue) -> Void, + in: context + ) + guard let successFunc, let rejectFunc else { + continuation.resume(throwing: JSCWasmError(message: "Could not await promise")) + return + } + invokeMethod("then", withArguments: [successFunc, rejectFunc]) + if let exception = context.exception { + continuation.resume(throwing: JSCWasmError(message: "Promise.then threw", value: exception)) + } + } + } + } +} + +#endif diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift deleted file mode 100644 index 89baa152aa3f1..0000000000000 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmPlugin.swift +++ /dev/null @@ -1,180 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#if os(macOS) - -import JavaScriptCore - -typealias DefaultWasmPlugin = JSCWasmPlugin - -// returns: (wasm: ArrayBuffer) => Promise<(input: ArrayBuffer) => ArrayBuffer> -// FIXME: stubbing WASI by always returning 0 triggers UB in the guest. -// (e.g. if we return 0 from args_sizes_get we must populate the out pointers.) -// we could adapt WasmKit.WASI to work with the JSC engine, since the host -// function code should be runtime-agnostic. -private let js = """ -async (wasmData) => { - const mod = await WebAssembly.compile(wasmData); - // stub WASI imports - const imports = WebAssembly.Module.imports(mod) - .filter(x => x.module === "wasi_snapshot_preview1") - .map(x => [x.name, () => {}]); - const instance = await WebAssembly.instantiate(mod, { - wasi_snapshot_preview1: Object.fromEntries(imports) - }); - const api = instance.exports; - api._start(); - return (json) => { - const inAddr = api.wacro_malloc(json.byteLength); - const mem = api.memory; - const arr = new Uint8Array(mem.buffer, inAddr, json.byteLength); - arr.set(new Uint8Array(json)); - const outAddr = api.wacro_parse(inAddr, json.byteLength); - const len = new Uint32Array(mem.buffer, outAddr)[0]; - const outArr = new Uint8Array(mem.buffer, outAddr + 4, len); - const copy = new Uint8Array(outArr); - api.wacro_free(outAddr); - return copy.buffer; - } -} -""" - -struct JSCWasmPlugin: WasmPlugin { - private let handler: JSValue - - init(wasm data: Data) async throws { - guard #available(macOS 10.15, *) else { - throw JSCWasmError(message: "JSC Wasm plugins currently require macOS 10.15+") - } - - let factory = try await JSCWasmFactory.shared - let context = factory.context - - let jsBuf = try JSValue(newBufferWithData: data, in: context) - guard let promise = factory.value.call(withArguments: [jsBuf]), context.exception == nil else { - throw JSCWasmError(message: "Failed to load plugin", value: context.exception) - } - - handler = try await promise.promiseValue - } - - func handleMessage(_ json: Data) throws -> Data { - guard let context = handler.context else { - throw JSCWasmError(message: "Failed to invoke plugin") - } - let jsonJS = try JSValue(newBufferWithData: json, in: context) - guard let result = handler.call(withArguments: [jsonJS]) else { - throw JSCWasmError(message: "Wasm plugin did not provide a valid response", value: context.exception) - } - return result.arrayBufferData() - } -} - -@available(macOS 10.15, *) -private struct JSCWasmFactory { - let context: JSContext - let value: JSValue - - private init() async throws { - // the VM must be created on the MainActor for async methods to work. - let vm = await MainActor.run { JSVirtualMachine() } - guard let context = JSContext(virtualMachine: vm) else { - throw JSCWasmError(message: "Failed to load plugin") - } - self.context = context - self.value = context.evaluateScript(js) - } - - private static let _shared = Task { - try await JSCWasmFactory() - } - - static var shared: JSCWasmFactory { - get async throws { - try await _shared.value - } - } -} - -public struct JSCWasmError: Error, CustomStringConvertible { - public let value: JSValue? - public let description: String - public init(message: String, value: JSValue? = nil) { - self.value = value - self.description = "\(message)\(value.map { ": \($0)" } ?? "")" - } -} - -extension JSValue { - fileprivate convenience init(newBufferWithData data: Data, in context: JSContext) throws { - let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) - _ = copy.initialize(from: data) - let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( - context.jsGlobalContextRef, - copy.baseAddress, copy.count, - { buf, _ in buf?.deallocate() }, nil, - nil - ) - self.init(jsValueRef: rawBuf, in: context) - } - - // it's unsafe to call JavaScriptCore APIs inside the `perform` block as this - // might invalidate the buffer. - fileprivate func withUnsafeArrayBuffer( - _ perform: (UnsafeRawBufferPointer) throws -> Result - ) rethrows -> Result { - let rawContext = context.jsGlobalContextRef - guard let base = JSObjectGetArrayBufferBytesPtr(rawContext, jsValueRef, nil) else { - return try perform(UnsafeRawBufferPointer(start: nil, count: 0)) - } - let count = JSObjectGetArrayBufferByteLength(rawContext, jsValueRef, nil) - let buffer = UnsafeRawBufferPointer(start: base, count: count) - return try perform(buffer) - } - - fileprivate func arrayBufferData() -> Data { - withUnsafeArrayBuffer { Data($0) } - } - - @available(macOS 10.15, *) - fileprivate var promiseValue: JSValue { - get async throws { - try await withCheckedThrowingContinuation { continuation in - typealias Handler = @convention(block) (JSValue) -> Void - let successFunc = JSValue( - object: { - continuation.resume(returning: $0) - } as Handler, - in: context - ) - let rejectFunc = JSValue( - object: { error in - continuation.resume( - throwing: JSCWasmError(message: "Promise rejected", value: error) - ) - } as @convention(block) (JSValue) -> Void, - in: context - ) - guard let successFunc, let rejectFunc else { - continuation.resume(throwing: JSCWasmError(message: "Could not await promise")) - return - } - invokeMethod("then", withArguments: [successFunc, rejectFunc]) - if let exception = context.exception { - continuation.resume(throwing: JSCWasmError(message: "Promise.then threw", value: exception)) - } - } - } - } -} - -#endif diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift new file mode 100644 index 0000000000000..39cbca1cc3095 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import WASI +import WasmTypes +import Foundation + +protocol WasmEngine { + associatedtype GuestMemoryType: GuestMemory + var memory: GuestMemoryType { get } + + init(wasm: Data, imports: WASIBridgeToHost) async throws + + func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] +} + +typealias DefaultWasmPlugin = WasmEnginePlugin + +// a WasmPlugin implementation that delegates to a WasmEngine +struct WasmEnginePlugin: WasmPlugin { + let engine: Engine + + init(wasm: Data) async throws { + // TODO: we should air-gap this bridge. Wasm macros don't need IO. + let bridge = try WASIBridgeToHost() + engine = try await Engine(wasm: wasm, imports: bridge) + _ = try engine.invoke("_start", []) + } + + func handleMessage(_ json: Data) async throws -> Data { + let memory = engine.memory + + let jsonLen = UInt32(json.count) + let inAddr = try engine.invoke("wacro_malloc", [jsonLen])[0] + let rawInAddr = UnsafeGuestPointer(memorySpace: memory, offset: inAddr) + _ = UnsafeGuestBufferPointer(baseAddress: rawInAddr, count: jsonLen) + .withHostPointer { $0.initialize(from: json) } + + let outAddr = try engine.invoke("wacro_parse", [inAddr, jsonLen])[0] + let outLen = UnsafeGuestPointer(memorySpace: memory, offset: outAddr).pointee + let outBase = UnsafeGuestPointer(memorySpace: memory, offset: outAddr + 4) + let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen) + .withHostPointer { Data($0) } + + _ = try engine.invoke("wacro_free", [outAddr]) + + return out + } +} diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift new file mode 100644 index 0000000000000..feca199f56095 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import WASI +import WasmKit +import WasmKitWASI +import Foundation + +typealias DefaultWasmEngine = WasmKitEngine + +struct WasmKitEngine: WasmEngine { + private let instance: ModuleInstance + private let runtime: Runtime + let memory: WasmKitGuestMemory + + init(wasm: Data, imports: WASIBridgeToHost) throws { + let module = try parseWasm(bytes: Array(wasm)) + runtime = Runtime(hostModules: imports.hostModules) + instance = try runtime.instantiate(module: module) + + let exports = instance.exports + guard case let .memory(memoryAddr) = exports["memory"] else { + throw WasmKitPluginError(message: "Wasm plugin does not export a valid memory.") + } + self.memory = WasmKitGuestMemory(store: runtime.store, address: memoryAddr) + } + + func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] { + try runtime.invoke(instance, function: method, with: args.map(Value.i32)).map(\.i32) + } +} + +struct WasmKitPluginError: Error, CustomStringConvertible { + let description: String + init(message: String) { + self.description = message + } +} diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift deleted file mode 100644 index 01597cfbe2350..0000000000000 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitPlugin.swift +++ /dev/null @@ -1,67 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#if !os(macOS) - -import Foundation -import WasmKit -import WASI - -typealias DefaultWasmPlugin = WasmKitPlugin - -struct WasmKitPlugin: WasmPlugin { - let instance: ModuleInstance - let runtime: Runtime - let memory: GuestMemory - - init(wasm: Data) throws { - let module = try parseWasm(bytes: Array(wasm)) - // TODO: we should air-gap this bridge. Wasm macros don't need IO. - let bridge = try WASIBridgeToHost() - runtime = Runtime(hostModules: bridge.hostModules) - instance = try runtime.instantiate(module: module) - _ = try bridge.start(instance, runtime: runtime) - - let exports = instance.exports - guard case let .memory(memoryAddr) = exports["memory"] else { - throw WasmKitPluginError(message: "Wasm plugin does not export a valid memory.") - } - self.memory = GuestMemory(store: runtime.store, address: memoryAddr) - } - - func handleMessage(_ json: Data) throws -> Data { - let jsonLen = UInt32(json.count) - let inAddr = try runtime.invoke(instance, function: "wacro_malloc", with: [.i32(jsonLen)])[0].i32 - let rawInAddr = UnsafeGuestPointer(memorySpace: memory, offset: inAddr) - _ = UnsafeGuestBufferPointer(baseAddress: rawInAddr, count: jsonLen) - .withHostPointer { $0.initialize(from: json) } - - let outAddr = try runtime.invoke(instance, function: "wacro_parse", with: [.i32(inAddr), .i32(jsonLen)])[0].i32 - let outLen = UnsafeGuestPointer(memorySpace: memory, offset: outAddr).pointee - let outBase = UnsafeGuestPointer(memorySpace: memory, offset: outAddr + 4) - let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen) - .withHostPointer { Data($0) } - - _ = try runtime.invoke(instance, function: "wacro_free", with: [.i32(outAddr)]) - - return out - } -} - -struct WasmKitPluginError: Error, CustomStringConvertible { - let description: String - init(message: String) { - self.description = message - } -} - -#endif diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 2fc622a233703..79702264233c8 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// -import SwiftCompilerPluginMessageHandling +@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling +@_spi(PluginMessage) import SwiftPluginServerSupport import Foundation import swiftLLVMJSON -import SwiftPluginServerSupport @main final class SwiftPluginServer { From 7a18da27a6b50de1c7892078be6c0d449b1c242a Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 21 Apr 2024 22:58:32 -0400 Subject: [PATCH 021/105] tweaks --- tools/swift-plugin-server/CMakeLists.txt | 2 -- .../Sources/swift-wasm-plugin-server/JSCWasmEngine.swift | 2 -- 2 files changed, 4 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 33830afe75164..26813b3104d4a 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -70,7 +70,5 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist ) - else() - # TODO: Add dependency on WasmKit endif() endif() diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 0a1e186927f1c..c82e0b5278525 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -16,8 +16,6 @@ import JavaScriptCore import WASI import WasmTypes -// typealias DefaultWasmEngine = JSCWasmEngine - // (wasm: ArrayBuffer, imports: Object) => Promise<{ ... }> private let js = """ async (wasmData, imports) => { From 1076b160d7522bd5ae0618f4d440bdec9a891972 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 21 Apr 2024 23:19:25 -0400 Subject: [PATCH 022/105] Check ABI version --- .../JSCWasmEngine.swift | 13 +++++-- .../swift-wasm-plugin-server/WasmEngine.swift | 34 +++++++++++++++++++ .../WasmKitEngine.swift | 7 +++- .../swift-wasm-plugin-server.swift | 10 +++++- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index c82e0b5278525..8efe3b7aef77a 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -24,6 +24,7 @@ async (wasmData, imports) => { const api = instance.exports; return { api, + customSections: (name) => WebAssembly.Module.customSections(mod, name), read: (off, size) => new Uint8Array(new Uint8Array(api.memory.buffer, off, size)).buffer, write: (buf, off, size) => new Uint8Array(api.memory.buffer, off, size).set(new Uint8Array(buf)), }; @@ -31,6 +32,7 @@ async (wasmData, imports) => { """ final class JSCWasmEngine: WasmEngine { + private let runner: JSValue private let _memory: JSCGuestMemory private let api: JSValue @@ -56,8 +58,7 @@ final class JSCWasmEngine: WasmEngine { throw JSCWasmError(message: "Failed to load plugin", value: context.exception) } - let runner = try await promise.promiseValue - + runner = try await promise.promiseValue api = runner.objectForKeyedSubscript("api")! let getMemory = runner.objectForKeyedSubscript("read")! @@ -66,6 +67,14 @@ final class JSCWasmEngine: WasmEngine { memory = self._memory } + func customSections(named name: String) throws -> [ArraySlice] { + guard let array = runner.invokeMethod("customSections", withArguments: [name]), + let length = array.objectForKeyedSubscript("length")?.toUInt32() else { return [] } + return (0.. [UInt32] { let result = api.invokeMethod(method, withArguments: args)! if let exception = api.context.exception { diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 39cbca1cc3095..b78bee7f49ca2 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -20,6 +20,8 @@ protocol WasmEngine { init(wasm: Data, imports: WASIBridgeToHost) async throws + func customSections(named name: String) throws -> [ArraySlice] + func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] } @@ -36,6 +38,30 @@ struct WasmEnginePlugin: WasmPlugin { _ = try engine.invoke("_start", []) } + func abiVersion() throws -> UInt32 { + let sectionName = "wacro_abi" + let sections = try engine.customSections(named: sectionName) + switch sections.count { + case 0: + throw WasmEngineError(message: "Wasm macro is missing a '\(sectionName)' section") + case 1: + break + default: + throw WasmEngineError(message: "Wasm macro has too many '\(sectionName)' sections. Expected one, got \(sections.count)") + } + let section = sections[0] + guard section.count == 4 else { + throw WasmEngineError(message: """ + Wasm macro has incorrect '\(sectionName)' section length. Expected 4 bytes, got \(section.count). + """) + } + return section.withUnsafeBufferPointer { buffer in + buffer.withMemoryRebound(to: UInt32.self) { + UInt32(littleEndian: $0.baseAddress!.pointee) + } + } + } + func handleMessage(_ json: Data) async throws -> Data { let memory = engine.memory @@ -56,3 +82,11 @@ struct WasmEnginePlugin: WasmPlugin { return out } } + +struct WasmEngineError: Error, CustomStringConvertible { + let description: String + + init(message: String) { + self.description = message + } +} diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index feca199f56095..9d424e6b7c353 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -18,12 +18,13 @@ import Foundation typealias DefaultWasmEngine = WasmKitEngine struct WasmKitEngine: WasmEngine { + private let module: Module private let instance: ModuleInstance private let runtime: Runtime let memory: WasmKitGuestMemory init(wasm: Data, imports: WASIBridgeToHost) throws { - let module = try parseWasm(bytes: Array(wasm)) + module = try parseWasm(bytes: Array(wasm)) runtime = Runtime(hostModules: imports.hostModules) instance = try runtime.instantiate(module: module) @@ -34,6 +35,10 @@ struct WasmKitEngine: WasmEngine { self.memory = WasmKitGuestMemory(store: runtime.store, address: memoryAddr) } + func customSections(named name: String) throws -> [ArraySlice] { + module.customSections.filter { $0.name == name }.map(\.bytes) + } + func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] { try runtime.invoke(instance, function: method, with: args.map(Value.i32)).map(\.i32) } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 79702264233c8..ddc15e4ade466 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -32,7 +32,12 @@ final class SwiftPluginServer { private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - return try await defaultWasmPlugin.init(wasm: wasm) + let plugin = try await defaultWasmPlugin.init(wasm: wasm) + let abiVersion = try plugin.abiVersion() + guard abiVersion == 1 else { + throw PluginServerError(message: "Wasm plugin has unsupported ABI version: \(abiVersion)") + } + return plugin } private func expandMacro( @@ -75,6 +80,7 @@ final class SwiftPluginServer { do { loadedWasmPlugins[moduleName] = try await loadPluginLibrary(path: libraryPath, moduleName: moduleName) } catch { + try? FileHandle.standardError.write(contentsOf: Data("Error: \(error)\n".utf8)) try sendMessage(.loadPluginLibraryResult(loaded: false, diagnostics: [])) continue } @@ -99,6 +105,8 @@ final class SwiftPluginServer { protocol WasmPlugin { init(wasm: Data) async throws + + func abiVersion() throws -> UInt32 func handleMessage(_ json: Data) async throws -> Data } From 45278424f225a013ca88ee8a4beffe8db8b698fc Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 21 Apr 2024 23:26:01 -0400 Subject: [PATCH 023/105] move ABI check into WasmEnginePlugin --- .../Sources/swift-wasm-plugin-server/WasmEngine.swift | 10 +++++++++- .../swift-wasm-plugin-server.swift | 8 +------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index b78bee7f49ca2..71ff490308ee3 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -35,10 +35,18 @@ struct WasmEnginePlugin: WasmPlugin { // TODO: we should air-gap this bridge. Wasm macros don't need IO. let bridge = try WASIBridgeToHost() engine = try await Engine(wasm: wasm, imports: bridge) + try checkABIVersion() _ = try engine.invoke("_start", []) } - func abiVersion() throws -> UInt32 { + private func checkABIVersion() throws { + let abiVersion = try abiVersion() + guard abiVersion == 1 else { + throw WasmEngineError(message: "Wasm plugin has unsupported ABI version: \(abiVersion)") + } + } + + private func abiVersion() throws -> UInt32 { let sectionName = "wacro_abi" let sections = try engine.customSections(named: sectionName) switch sections.count { diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index ddc15e4ade466..9423360a81bc8 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -32,12 +32,7 @@ final class SwiftPluginServer { private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) - let plugin = try await defaultWasmPlugin.init(wasm: wasm) - let abiVersion = try plugin.abiVersion() - guard abiVersion == 1 else { - throw PluginServerError(message: "Wasm plugin has unsupported ABI version: \(abiVersion)") - } - return plugin + return try await defaultWasmPlugin.init(wasm: wasm) } private func expandMacro( @@ -106,7 +101,6 @@ final class SwiftPluginServer { protocol WasmPlugin { init(wasm: Data) async throws - func abiVersion() throws -> UInt32 func handleMessage(_ json: Data) async throws -> Data } From f9dbbb1be300c9d86d61747e0c0c0abe6b63b032 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 22 Apr 2024 01:02:29 -0400 Subject: [PATCH 024/105] build fixes --- tools/swift-plugin-server/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 26813b3104d4a..a291316058b6b 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -5,8 +5,10 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) include(DarwinSDKs) swift_get_host_triple(SWIFT_HOST_TRIPLE) - file(TO_CMAKE_PATH "${SWIFT_PATH_TO_WASMKIT_SOURCE}" wasmkit_path) - FetchContent_Declare(WasmKit SOURCE_DIR "${wasmkit_path}") + # override the remote SwiftSystem dependency in WasmKit + FetchContent_Declare(SwiftSystem SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-system") + + FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") FetchContent_MakeAvailable(WasmKit) # _swiftCSwiftPluginServer is just a C support library for swift-plugin-server @@ -25,6 +27,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/SwiftPluginServerSupport/SwiftPluginServerSupport.swift DEPENDENCIES $ + SwiftCompilerPluginMessageHandling ) target_include_directories(SwiftPluginServerSupport PRIVATE Sources/CSwiftPluginServer/include From 6511672ad203e91907b3bebd389f000cd7d24234 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 22 Apr 2024 02:14:31 -0400 Subject: [PATCH 025/105] Elide copies of wasm blob --- .../JSCWasmEngine.swift | 11 ++++++-- .../swift-wasm-plugin-server/WasmEngine.swift | 4 +-- .../WasmKitEngine.swift | 7 ++--- .../swift-wasm-plugin-server.swift | 27 ++++++++++++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 8efe3b7aef77a..b8fd6c880c1a4 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -38,7 +38,7 @@ final class JSCWasmEngine: WasmEngine { var memory: some GuestMemory { _memory } - init(wasm data: Data, imports: WASIBridgeToHost) async throws { + init(wasm buffer: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws { let factory = try await JSCWasmFactory.shared let context = factory.context @@ -52,7 +52,14 @@ final class JSCWasmEngine: WasmEngine { } } - let jsBuf = try JSValue(newArrayBufferWithData: data, in: context) + let jsBuf = try JSValue( + // JavaScript doesn't have readonly ArrayBuffers but all the JS we're running + // is contained within this file and we know that we aren't mutating this buffer. + // If we did attempt to mutate it, it would be UB. + newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer(mutating: buffer.data), + deallocator: { _ = buffer }, + in: context + ) let promise = factory.value.call(withArguments: [jsBuf, imports]) guard let promise, context.exception == nil else { throw JSCWasmError(message: "Failed to load plugin", value: context.exception) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 71ff490308ee3..92881b429fe1e 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -18,7 +18,7 @@ protocol WasmEngine { associatedtype GuestMemoryType: GuestMemory var memory: GuestMemoryType { get } - init(wasm: Data, imports: WASIBridgeToHost) async throws + init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws func customSections(named name: String) throws -> [ArraySlice] @@ -31,7 +31,7 @@ typealias DefaultWasmPlugin = WasmEnginePlugin struct WasmEnginePlugin: WasmPlugin { let engine: Engine - init(wasm: Data) async throws { + init(wasm: UnsafeByteBuffer) async throws { // TODO: we should air-gap this bridge. Wasm macros don't need IO. let bridge = try WASIBridgeToHost() engine = try await Engine(wasm: wasm, imports: bridge) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 9d424e6b7c353..355b0e7e75b24 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -13,7 +13,6 @@ import WASI import WasmKit import WasmKitWASI -import Foundation typealias DefaultWasmEngine = WasmKitEngine @@ -23,8 +22,10 @@ struct WasmKitEngine: WasmEngine { private let runtime: Runtime let memory: WasmKitGuestMemory - init(wasm: Data, imports: WASIBridgeToHost) throws { - module = try parseWasm(bytes: Array(wasm)) + init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) throws { + // we never call wasm.deallocator, effectively leaking the data, + // but that's intentional because plugins can't be "unloaded" + module = try parseWasm(bytes: Array(wasm.data)) runtime = Runtime(hostModules: imports.hostModules) instance = try runtime.instantiate(module: module) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 9423360a81bc8..eaaf30279f10b 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -30,8 +30,14 @@ final class SwiftPluginServer { } private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - guard path.hasSuffix(".wasm") else { throw PluginServerError(message: "swift-wasm-plugin-server can only load wasm") } - let wasm = try Data(contentsOf: URL(fileURLWithPath: path)) + // it's worth the effort jumping through these hoops because + // wasm modules can be really large (30M+) so we want to avoid making + // copies if possible. + let data = try NSData(contentsOf: URL(fileURLWithPath: path), options: .mappedIfSafe) + let wasm = UnsafeByteBuffer( + data: UnsafeRawBufferPointer(start: data.bytes, count: data.count), + deallocator: { [box = data as AnyObject] in _ = box } + ) return try await defaultWasmPlugin.init(wasm: wasm) } @@ -99,9 +105,24 @@ final class SwiftPluginServer { } protocol WasmPlugin { - init(wasm: Data) async throws + init(wasm: UnsafeByteBuffer) async throws func handleMessage(_ json: Data) async throws -> Data } private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } + +// An immutable data buffer with a stable address. +// +// The pointer is valid until the receiver is deallocated. +final class UnsafeByteBuffer: @unchecked Sendable { + let data: UnsafeRawBufferPointer + private let deallocator: @Sendable () -> Void + + init(data: UnsafeRawBufferPointer, deallocator: @escaping @Sendable () -> Void) { + self.data = data + self.deallocator = deallocator + } + + deinit { deallocator() } +} From dd481fe67707318acd695d5ccba7d6c02d22590b Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 22 Apr 2024 20:15:11 -0400 Subject: [PATCH 026/105] fix duplicate libraries warning --- tools/swift-plugin-server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index a291316058b6b..6d950c0ee2075 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -64,7 +64,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SwiftCompilerPluginMessageHandling swiftLLVMJSON SwiftPluginServerSupport - WASI WasmKit WasmKitWASI WasmTypes + WasmKitWASI ) target_include_directories(swift-wasm-plugin-server PRIVATE Sources/CSwiftPluginServer/include From 482871c3c250d3898b1cd5b94f0dcd30e79def33 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 22 Apr 2024 20:59:11 -0400 Subject: [PATCH 027/105] add SWIFT_WASM_USE_JSC --- tools/swift-plugin-server/CMakeLists.txt | 21 +++++++++++++++++-- tools/swift-plugin-server/Package.swift | 8 +++++-- .../JSCWasmEngine.swift | 6 ++++-- .../WasmKitEngine.swift | 4 ++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 6d950c0ee2075..c962fb97cbcfd 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -51,6 +51,22 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/CSwiftPluginServer/include ) + set(swift_wasm_allow_jsc ON) + + if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) + set(swift_wasm_use_jsc ${swift_wasm_allow_jsc}) + else() + set(swift_wasm_use_jsc OFF) + endif() + + if (swift_wasm_use_jsc) + set(wasi_dep WASI) + set(jsc_defines SWIFT_WASM_USE_JSC) + else() + set(wasi_dep WasmKitWASI) + set(jsc_defines) + endif() + add_pure_swift_host_tool(swift-wasm-plugin-server Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -64,12 +80,13 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SwiftCompilerPluginMessageHandling swiftLLVMJSON SwiftPluginServerSupport - WasmKitWASI + ${wasi_dep} ) + target_compile_definitions(swift-wasm-plugin-server PRIVATE ${jsc_defines}) target_include_directories(swift-wasm-plugin-server PRIVATE Sources/CSwiftPluginServer/include ) - if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) + if (swift_wasm_use_jsc) add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist ) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 6f3d3e3a9ce34..8dea58af4de0e 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -2,6 +2,8 @@ import PackageDescription +let allowJSC = true + let package = Package( name: "swift-plugin-server", platforms: [ @@ -56,9 +58,11 @@ let package = Package( "CSwiftPluginServer", "SwiftPluginServerSupport", .product(name: "WASI", package: "WasmKit"), - .product(name: "WasmKitWASI", package: "WasmKit"), + .product(name: "WasmKitWASI", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), ], - swiftSettings: [.interoperabilityMode(.Cxx)] + swiftSettings: [.interoperabilityMode(.Cxx)] + ( + allowJSC ? [.define("SWIFT_WASM_USE_JSC", .when(platforms: [.macOS]))] : [] + ) ), ], cxxLanguageStandard: .cxx17 diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index b8fd6c880c1a4..5a54de3ae83aa 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -10,12 +10,14 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) +#if SWIFT_WASM_USE_JSC import JavaScriptCore import WASI import WasmTypes +typealias DefaultWasmEngine = JSCWasmEngine + // (wasm: ArrayBuffer, imports: Object) => Promise<{ ... }> private let js = """ async (wasmData, imports) => { @@ -144,7 +146,7 @@ public struct JSCWasmError: Error, CustomStringConvertible { } extension WASIHostFunction { - func asJSFunction(in context: JSContext, memory: @escaping () throws -> GuestMemory) -> JSValue { + fileprivate func asJSFunction(in context: JSContext, memory: @escaping () throws -> GuestMemory) -> JSValue { JSValue(object: { let arguments = JSContext.currentArguments() as? [JSValue] ?? [] let types: [Value] = zip(arguments, type.parameters).map { argument, type in diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 355b0e7e75b24..38a204b8771f3 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +#if !SWIFT_WASM_USE_JSC + import WASI import WasmKit import WasmKitWASI @@ -51,3 +53,5 @@ struct WasmKitPluginError: Error, CustomStringConvertible { self.description = message } } + +#endif From 2460e9893fe8a02018a650e6c72a3a7b02edfbfa Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 04:09:47 -0400 Subject: [PATCH 028/105] Skip wasmkit CLI for now --- tools/swift-plugin-server/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index c962fb97cbcfd..3b0cdcfa96eff 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -8,6 +8,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) # override the remote SwiftSystem dependency in WasmKit FetchContent_Declare(SwiftSystem SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-system") + set(WASMKIT_BUILD_CLI OFF) FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") FetchContent_MakeAvailable(WasmKit) From 150f3d2241abeffbae8539b33e59db347bfbba84 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 04:13:31 -0400 Subject: [PATCH 029/105] Or actually build wasmkit-cli --- tools/swift-plugin-server/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 3b0cdcfa96eff..ab26f13be6f4f 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -5,10 +5,10 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) include(DarwinSDKs) swift_get_host_triple(SWIFT_HOST_TRIPLE) - # override the remote SwiftSystem dependency in WasmKit + # override the remote SwiftSystem and ArgParser dependencies in WasmKit FetchContent_Declare(SwiftSystem SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-system") + FetchContent_Declare(ArgumentParser SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-argument-parser") - set(WASMKIT_BUILD_CLI OFF) FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") FetchContent_MakeAvailable(WasmKit) From 34ed5b09b404bc52e34e95987096e58c0789229c Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 16:33:00 -0400 Subject: [PATCH 030/105] use stdio for ipc --- .../swift-wasm-plugin-server/WasmEngine.swift | 41 ++++++++++--------- .../swift-wasm-plugin-server.swift | 1 + 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 92881b429fe1e..21b19019fcd3d 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -12,6 +12,7 @@ import WASI import WasmTypes +import SystemPackage import Foundation protocol WasmEngine { @@ -29,11 +30,16 @@ typealias DefaultWasmPlugin = WasmEnginePlugin // a WasmPlugin implementation that delegates to a WasmEngine struct WasmEnginePlugin: WasmPlugin { + private let hostToPlugin = Pipe() + private let pluginToHost = Pipe() let engine: Engine init(wasm: UnsafeByteBuffer) async throws { - // TODO: we should air-gap this bridge. Wasm macros don't need IO. - let bridge = try WASIBridgeToHost() + let bridge = try WASIBridgeToHost( + stdin: FileDescriptor(rawValue: hostToPlugin.fileHandleForReading.fileDescriptor), + stdout: FileDescriptor(rawValue: pluginToHost.fileHandleForWriting.fileDescriptor), + stderr: .standardError + ) engine = try await Engine(wasm: wasm, imports: bridge) try checkABIVersion() _ = try engine.invoke("_start", []) @@ -47,7 +53,7 @@ struct WasmEnginePlugin: WasmPlugin { } private func abiVersion() throws -> UInt32 { - let sectionName = "wacro_abi" + let sectionName = "swift_wasm_macro_abi" let sections = try engine.customSections(named: sectionName) switch sections.count { case 0: @@ -71,23 +77,18 @@ struct WasmEnginePlugin: WasmPlugin { } func handleMessage(_ json: Data) async throws -> Data { - let memory = engine.memory - - let jsonLen = UInt32(json.count) - let inAddr = try engine.invoke("wacro_malloc", [jsonLen])[0] - let rawInAddr = UnsafeGuestPointer(memorySpace: memory, offset: inAddr) - _ = UnsafeGuestBufferPointer(baseAddress: rawInAddr, count: jsonLen) - .withHostPointer { $0.initialize(from: json) } - - let outAddr = try engine.invoke("wacro_parse", [inAddr, jsonLen])[0] - let outLen = UnsafeGuestPointer(memorySpace: memory, offset: outAddr).pointee - let outBase = UnsafeGuestPointer(memorySpace: memory, offset: outAddr + 4) - let out = UnsafeGuestBufferPointer(baseAddress: outBase, count: outLen) - .withHostPointer { Data($0) } - - _ = try engine.invoke("wacro_free", [outAddr]) - - return out + let writeHandle = hostToPlugin.fileHandleForWriting + let count = withUnsafeBytes(of: UInt64(json.count).littleEndian) { Data($0) } + try writeHandle.write(contentsOf: count) + try writeHandle.write(contentsOf: json) + + _ = try engine.invoke("swift_wasm_macro_pump", []) + + let readHandle = pluginToHost.fileHandleForReading + let lengthRaw = try readHandle.read(upToCount: 8) ?? Data() + let length = lengthRaw.withUnsafeBytes { $0.assumingMemoryBound(to: UInt64.self).baseAddress?.pointee } + guard let length else { throw WasmEngineError(message: "Bad byte length") } + return try readHandle.read(upToCount: Int(length)) ?? Data() } } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index eaaf30279f10b..005f0c0190c1f 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -56,6 +56,7 @@ final class SwiftPluginServer { } } } catch { + try? FileHandle.standardError.write(contentsOf: Data("Error: \(error)\n".utf8)) try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [])) return } From 94d525eaac98a0dc2072f1a18577ece5270d7618 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 17:10:35 -0400 Subject: [PATCH 031/105] No need for memory witness --- tools/swift-plugin-server/CMakeLists.txt | 4 ++-- .../Sources/swift-wasm-plugin-server/JSCWasmEngine.swift | 6 +----- .../Sources/swift-wasm-plugin-server/WasmEngine.swift | 3 --- .../Sources/swift-wasm-plugin-server/WasmKitEngine.swift | 7 ------- 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index ab26f13be6f4f..8d1b2c82b5091 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -52,10 +52,10 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/CSwiftPluginServer/include ) - set(swift_wasm_allow_jsc ON) + set(SWIFT_WASM_ALLOW_JSC ON CACHE BOOL "Use JavaScriptCore Wasm runtime if possible" FORCE) if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) - set(swift_wasm_use_jsc ${swift_wasm_allow_jsc}) + set(swift_wasm_use_jsc ${SWIFT_WASM_ALLOW_JSC}) else() set(swift_wasm_use_jsc OFF) endif() diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 5a54de3ae83aa..1be60cab7894e 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -35,11 +35,8 @@ async (wasmData, imports) => { final class JSCWasmEngine: WasmEngine { private let runner: JSValue - private let _memory: JSCGuestMemory private let api: JSValue - var memory: some GuestMemory { _memory } - init(wasm buffer: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws { let factory = try await JSCWasmFactory.shared let context = factory.context @@ -72,8 +69,7 @@ final class JSCWasmEngine: WasmEngine { let getMemory = runner.objectForKeyedSubscript("read")! let setMemory = runner.objectForKeyedSubscript("write")! - self._memory = JSCGuestMemory(getMemory: getMemory, setMemory: setMemory) - memory = self._memory + memory = JSCGuestMemory(getMemory: getMemory, setMemory: setMemory) } func customSections(named name: String) throws -> [ArraySlice] { diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 21b19019fcd3d..e9eab8b140349 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -16,9 +16,6 @@ import SystemPackage import Foundation protocol WasmEngine { - associatedtype GuestMemoryType: GuestMemory - var memory: GuestMemoryType { get } - init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws func customSections(named name: String) throws -> [ArraySlice] diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 38a204b8771f3..5d4a02a9e02a2 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -22,7 +22,6 @@ struct WasmKitEngine: WasmEngine { private let module: Module private let instance: ModuleInstance private let runtime: Runtime - let memory: WasmKitGuestMemory init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) throws { // we never call wasm.deallocator, effectively leaking the data, @@ -30,12 +29,6 @@ struct WasmKitEngine: WasmEngine { module = try parseWasm(bytes: Array(wasm.data)) runtime = Runtime(hostModules: imports.hostModules) instance = try runtime.instantiate(module: module) - - let exports = instance.exports - guard case let .memory(memoryAddr) = exports["memory"] else { - throw WasmKitPluginError(message: "Wasm plugin does not export a valid memory.") - } - self.memory = WasmKitGuestMemory(store: runtime.store, address: memoryAddr) } func customSections(named name: String) throws -> [ArraySlice] { From a99f6b3cdcaceda7c94880184bd68b6162d0333f Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 17:53:46 -0400 Subject: [PATCH 032/105] Byte-swap length if needed --- .../Sources/swift-wasm-plugin-server/WasmEngine.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index e9eab8b140349..95017eb3e0dc4 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -82,10 +82,11 @@ struct WasmEnginePlugin: WasmPlugin { _ = try engine.invoke("swift_wasm_macro_pump", []) let readHandle = pluginToHost.fileHandleForReading - let lengthRaw = try readHandle.read(upToCount: 8) ?? Data() - let length = lengthRaw.withUnsafeBytes { $0.assumingMemoryBound(to: UInt64.self).baseAddress?.pointee } - guard let length else { throw WasmEngineError(message: "Bad byte length") } - return try readHandle.read(upToCount: Int(length)) ?? Data() + let lengthData = try readHandle.read(upToCount: 8) ?? Data() + let lengthRaw = lengthData.withUnsafeBytes { $0.assumingMemoryBound(to: UInt64.self).baseAddress?.pointee } + guard let lengthRaw else { throw WasmEngineError(message: "Bad byte length") } + let length = Int(UInt64(littleEndian: lengthRaw)) + return try readHandle.read(upToCount: length) ?? Data() } } From c5dc6bdfa62c0a286244c9a6ae98fcad2b701be7 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 24 Apr 2024 17:59:10 -0400 Subject: [PATCH 033/105] Improve error handling --- .../Sources/swift-wasm-plugin-server/WasmEngine.swift | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 95017eb3e0dc4..cd9c5e5adc97d 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -82,9 +82,13 @@ struct WasmEnginePlugin: WasmPlugin { _ = try engine.invoke("swift_wasm_macro_pump", []) let readHandle = pluginToHost.fileHandleForReading - let lengthData = try readHandle.read(upToCount: 8) ?? Data() - let lengthRaw = lengthData.withUnsafeBytes { $0.assumingMemoryBound(to: UInt64.self).baseAddress?.pointee } - guard let lengthRaw else { throw WasmEngineError(message: "Bad byte length") } + let lengthData = try readHandle.read(upToCount: 8) + guard let lengthData, lengthData.count == 8 else { + throw WasmEngineError(message: "Wasm plugin sent invalid response") + } + let lengthRaw = lengthData.withUnsafeBytes { + $0.assumingMemoryBound(to: UInt64.self).baseAddress!.pointee + } let length = Int(UInt64(littleEndian: lengthRaw)) return try readHandle.read(upToCount: length) ?? Data() } From 826a80f997b5ecc4c6268372c746d1cb72a3b674 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 29 Apr 2024 21:45:21 -0400 Subject: [PATCH 034/105] Fix linkage --- tools/swift-plugin-server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 8b6079d3e83af..afc45ad19160b 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -62,7 +62,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/swift-wasm-plugin-server/WasmEngine.swift Sources/swift-wasm-plugin-server/WasmKitEngine.swift DEPENDENCIES - $ + _swiftCSwiftPluginServer SWIFT_COMPONENT compiler SWIFT_DEPENDENCIES From f053859a65e89b998b69f42a715cae7dec716e18 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 29 Apr 2024 21:50:04 -0400 Subject: [PATCH 035/105] No C dep needed for wasm --- tools/swift-plugin-server/CMakeLists.txt | 2 -- tools/swift-plugin-server/Package.swift | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index afc45ad19160b..08dad2da58cdd 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -61,8 +61,6 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) Sources/swift-wasm-plugin-server/JSCWasmEngine.swift Sources/swift-wasm-plugin-server/WasmEngine.swift Sources/swift-wasm-plugin-server/WasmKitEngine.swift - DEPENDENCIES - _swiftCSwiftPluginServer SWIFT_COMPONENT compiler SWIFT_DEPENDENCIES diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 37f76ec686e5d..46f6c1723f063 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -43,13 +43,10 @@ let package = Package( name: "swift-wasm-plugin-server", dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), - "CSwiftPluginServer", .product(name: "WASI", package: "WasmKit"), .product(name: "WasmKitWASI", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), ], - swiftSettings: [.interoperabilityMode(.Cxx)] + ( - allowJSC ? [.define("SWIFT_WASM_USE_JSC", .when(platforms: [.macOS]))] : [] - ) + swiftSettings: allowJSC ? [.define("SWIFT_WASM_USE_JSC", .when(platforms: [.macOS]))] : [] ), ], cxxLanguageStandard: .cxx17 From 6c0b40c0f537714c4453bf1c213cd0f3501866fc Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:37:15 -0400 Subject: [PATCH 036/105] Drop foundation (mostly); lower min os --- tools/swift-plugin-server/CMakeLists.txt | 6 --- .../JSCWasmEngine.swift | 4 ++ .../swift-wasm-plugin-server/WasmEngine.swift | 45 +++++++++++-------- .../swift-wasm-plugin-server.swift | 17 ++++--- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 08dad2da58cdd..bf5c56260f65c 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,10 +1,4 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) - # hack: override macOS deployment target to 10.15.4 - # since that's the minimum required by WasmKit - set(SWIFT_DARWIN_DEPLOYMENT_VERSION_OSX "10.15.4") - include(DarwinSDKs) - swift_get_host_triple(SWIFT_HOST_TRIPLE) - # override the remote SwiftSystem and ArgParser dependencies in WasmKit FetchContent_Declare(SwiftSystem SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-system") FetchContent_Declare(ArgumentParser SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-argument-parser") diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 1be60cab7894e..16e5f831fd57d 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -38,6 +38,8 @@ final class JSCWasmEngine: WasmEngine { private let api: JSValue init(wasm buffer: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws { + guard #available(macOS 10.15, *) else { fatalError("JSCWasmEngine requires macOS 10.15+") } + let factory = try await JSCWasmFactory.shared let context = factory.context @@ -107,6 +109,7 @@ private struct JSCGuestMemory: GuestMemory { } } +@available(macOS 10.15, *) private struct JSCWasmFactory { let context: JSContext let value: JSValue @@ -214,6 +217,7 @@ extension JSValue { withUnsafeArrayBuffer { Data($0) } } + @available(macOS 10.15, *) fileprivate var promiseValue: JSValue { get async throws { try await withCheckedThrowingContinuation { continuation in diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index cd9c5e5adc97d..1f6b755763355 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -13,7 +13,6 @@ import WASI import WasmTypes import SystemPackage -import Foundation protocol WasmEngine { init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws @@ -27,14 +26,19 @@ typealias DefaultWasmPlugin = WasmEnginePlugin // a WasmPlugin implementation that delegates to a WasmEngine struct WasmEnginePlugin: WasmPlugin { - private let hostToPlugin = Pipe() - private let pluginToHost = Pipe() + private let hostToPlugin: FileDescriptor + private let pluginToHost: FileDescriptor let engine: Engine init(wasm: UnsafeByteBuffer) async throws { + let hostToPluginPipes = try FileDescriptor.pipe() + let pluginToHostPipes = try FileDescriptor.pipe() + self.hostToPlugin = hostToPluginPipes.writeEnd + self.pluginToHost = pluginToHostPipes.readEnd + let bridge = try WASIBridgeToHost( - stdin: FileDescriptor(rawValue: hostToPlugin.fileHandleForReading.fileDescriptor), - stdout: FileDescriptor(rawValue: pluginToHost.fileHandleForWriting.fileDescriptor), + stdin: hostToPluginPipes.readEnd, + stdout: pluginToHostPipes.writeEnd, stderr: .standardError ) engine = try await Engine(wasm: wasm, imports: bridge) @@ -73,24 +77,29 @@ struct WasmEnginePlugin: WasmPlugin { } } - func handleMessage(_ json: Data) async throws -> Data { - let writeHandle = hostToPlugin.fileHandleForWriting - let count = withUnsafeBytes(of: UInt64(json.count).littleEndian) { Data($0) } - try writeHandle.write(contentsOf: count) - try writeHandle.write(contentsOf: json) + func handleMessage(_ json: [UInt8]) async throws -> [UInt8] { + try withUnsafeBytes(of: UInt64(json.count).littleEndian) { + _ = try hostToPlugin.writeAll($0) + } + try hostToPlugin.writeAll(json) _ = try engine.invoke("swift_wasm_macro_pump", []) - let readHandle = pluginToHost.fileHandleForReading - let lengthData = try readHandle.read(upToCount: 8) - guard let lengthData, lengthData.count == 8 else { - throw WasmEngineError(message: "Wasm plugin sent invalid response") - } - let lengthRaw = lengthData.withUnsafeBytes { - $0.assumingMemoryBound(to: UInt64.self).baseAddress!.pointee + let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in + let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) + guard lengthCount == 8 else { + throw WasmEngineError(message: "Wasm plugin sent invalid response") + } + return buffer.withMemoryRebound(to: UInt64.self, \.baseAddress!.pointee) } let length = Int(UInt64(littleEndian: lengthRaw)) - return try readHandle.read(upToCount: length) ?? Data() + return try [UInt8](unsafeUninitializedCapacity: length) { buffer, size in + let received = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) + guard received == length else { + throw WasmEngineError(message: "Wasm plugin sent truncated response") + } + size = received + } } } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index 9aff4b66c2790..bbb39e3ce03ea 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -11,7 +11,8 @@ //===----------------------------------------------------------------------===// @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling -import Foundation +import SystemPackage +import Foundation.NSData @main final class SwiftPluginServer { @@ -31,7 +32,7 @@ final class SwiftPluginServer { // it's worth the effort jumping through these hoops because // wasm modules can be really large (30M+) so we want to avoid making // copies if possible. - let data = try NSData(contentsOf: URL(fileURLWithPath: path), options: .mappedIfSafe) + let data = try NSData(contentsOfFile: path, options: .mappedIfSafe) let wasm = UnsafeByteBuffer( data: UnsafeRawBufferPointer(start: data.bytes, count: data.count), deallocator: { [box = data as AnyObject] in _ = box } @@ -46,7 +47,7 @@ final class SwiftPluginServer { let response: PluginToHostMessage do { guard let plugin = loadedWasmPlugins[module] else { throw PluginServerError(message: "Could not find module \(module)") } - let request = try JSON.encode(message).withUnsafeBufferPointer { Data($0) } + let request = try JSON.encode(message) let responseRaw = try await plugin.handleMessage(request) response = try responseRaw.withUnsafeBytes { try $0.withMemoryRebound(to: UInt8.self) { @@ -54,7 +55,7 @@ final class SwiftPluginServer { } } } catch { - try? FileHandle.standardError.write(contentsOf: Data("Error: \(error)\n".utf8)) + printError("Error: \(error)") try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [])) return } @@ -80,7 +81,7 @@ final class SwiftPluginServer { do { loadedWasmPlugins[moduleName] = try await loadPluginLibrary(path: libraryPath, moduleName: moduleName) } catch { - try? FileHandle.standardError.write(contentsOf: Data("Error: \(error)\n".utf8)) + printError("Error: \(error)") try sendMessage(.loadPluginLibraryResult(loaded: false, diagnostics: [])) continue } @@ -103,10 +104,14 @@ final class SwiftPluginServer { } } +private func printError(_ error: String) { + _ = try? FileDescriptor.standardError.writeAll("\(error)\n".utf8) +} + protocol WasmPlugin { init(wasm: UnsafeByteBuffer) async throws - func handleMessage(_ json: Data) async throws -> Data + func handleMessage(_ json: [UInt8]) async throws -> [UInt8] } private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } From d72e165945981a289aa5aa064b4b181f230547a5 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:49:33 -0400 Subject: [PATCH 037/105] More foundation pruning --- tools/swift-plugin-server/Package.swift | 4 ++- .../JSCWasmEngine.swift | 11 ++++++-- .../swift-wasm-plugin-server/WasmEngine.swift | 6 ++-- .../WasmKitEngine.swift | 5 ++-- .../swift-wasm-plugin-server.swift | 28 ++----------------- 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 46f6c1723f063..d1f76138d2c2d 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -44,7 +44,9 @@ let package = Package( dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "WASI", package: "WasmKit"), - .product(name: "WasmKitWASI", package: "WasmKit", condition: .when(platforms: [.linux, .windows])), + .product(name: "WasmKitWASI", package: "WasmKit", condition: .when( + platforms: [.linux, .windows] + (allowJSC ? [] : [.macOS]) + )), ], swiftSettings: allowJSC ? [.define("SWIFT_WASM_USE_JSC", .when(platforms: [.macOS]))] : [] ), diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 16e5f831fd57d..1ab07d0a3a192 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -15,6 +15,7 @@ import JavaScriptCore import WASI import WasmTypes +import SystemPackage typealias DefaultWasmEngine = JSCWasmEngine @@ -37,7 +38,7 @@ final class JSCWasmEngine: WasmEngine { private let runner: JSValue private let api: JSValue - init(wasm buffer: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws { + init(path: FilePath, imports: WASIBridgeToHost) async throws { guard #available(macOS 10.15, *) else { fatalError("JSCWasmEngine requires macOS 10.15+") } let factory = try await JSCWasmFactory.shared @@ -53,12 +54,16 @@ final class JSCWasmEngine: WasmEngine { } } + let buffer = try NSData(contentsOfFile: path.string, options: .mappedIfSafe) let jsBuf = try JSValue( // JavaScript doesn't have readonly ArrayBuffers but all the JS we're running // is contained within this file and we know that we aren't mutating this buffer. // If we did attempt to mutate it, it would be UB. - newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer(mutating: buffer.data), - deallocator: { _ = buffer }, + newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer( + start: UnsafeMutableRawPointer(mutating: buffer.bytes), + count: buffer.count + ), + deallocator: { [box = buffer as AnyObject] in _ = box }, in: context ) let promise = factory.value.call(withArguments: [jsBuf, imports]) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 1f6b755763355..249d27e2631ff 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -15,7 +15,7 @@ import WasmTypes import SystemPackage protocol WasmEngine { - init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) async throws + init(path: FilePath, imports: WASIBridgeToHost) async throws func customSections(named name: String) throws -> [ArraySlice] @@ -30,7 +30,7 @@ struct WasmEnginePlugin: WasmPlugin { private let pluginToHost: FileDescriptor let engine: Engine - init(wasm: UnsafeByteBuffer) async throws { + init(path: FilePath) async throws { let hostToPluginPipes = try FileDescriptor.pipe() let pluginToHostPipes = try FileDescriptor.pipe() self.hostToPlugin = hostToPluginPipes.writeEnd @@ -41,7 +41,7 @@ struct WasmEnginePlugin: WasmPlugin { stdout: pluginToHostPipes.writeEnd, stderr: .standardError ) - engine = try await Engine(wasm: wasm, imports: bridge) + engine = try await Engine(path: path, imports: bridge) try checkABIVersion() _ = try engine.invoke("_start", []) } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 5d4a02a9e02a2..65a5c33e48d96 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -15,6 +15,7 @@ import WASI import WasmKit import WasmKitWASI +import SystemPackage typealias DefaultWasmEngine = WasmKitEngine @@ -23,10 +24,10 @@ struct WasmKitEngine: WasmEngine { private let instance: ModuleInstance private let runtime: Runtime - init(wasm: UnsafeByteBuffer, imports: WASIBridgeToHost) throws { + init(path: FilePath, imports: WASIBridgeToHost) throws { // we never call wasm.deallocator, effectively leaking the data, // but that's intentional because plugins can't be "unloaded" - module = try parseWasm(bytes: Array(wasm.data)) + module = try parseWasm(filePath: path) runtime = Runtime(hostModules: imports.hostModules) instance = try runtime.instantiate(module: module) } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift index bbb39e3ce03ea..b76159957a9b3 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift @@ -12,7 +12,6 @@ @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling import SystemPackage -import Foundation.NSData @main final class SwiftPluginServer { @@ -29,15 +28,7 @@ final class SwiftPluginServer { } private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - // it's worth the effort jumping through these hoops because - // wasm modules can be really large (30M+) so we want to avoid making - // copies if possible. - let data = try NSData(contentsOfFile: path, options: .mappedIfSafe) - let wasm = UnsafeByteBuffer( - data: UnsafeRawBufferPointer(start: data.bytes, count: data.count), - deallocator: { [box = data as AnyObject] in _ = box } - ) - return try await defaultWasmPlugin.init(wasm: wasm) + return try await defaultWasmPlugin.init(path: FilePath(path)) } private func expandMacro( @@ -109,28 +100,13 @@ private func printError(_ error: String) { } protocol WasmPlugin { - init(wasm: UnsafeByteBuffer) async throws + init(path: FilePath) async throws func handleMessage(_ json: [UInt8]) async throws -> [UInt8] } private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } -// An immutable data buffer with a stable address. -// -// The pointer is valid until the receiver is deallocated. -final class UnsafeByteBuffer: @unchecked Sendable { - let data: UnsafeRawBufferPointer - private let deallocator: @Sendable () -> Void - - init(data: UnsafeRawBufferPointer, deallocator: @escaping @Sendable () -> Void) { - self.data = data - self.deallocator = deallocator - } - - deinit { deallocator() } -} - struct PluginServerError: Error, CustomStringConvertible { var description: String init(message: String) { From 564f84471f82396383a50b365512e1262beeefab Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:51:58 -0400 Subject: [PATCH 038/105] interop mode cxx -> c --- tools/swift-plugin-server/Package.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index d1f76138d2c2d..3f38126a596de 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -36,8 +36,7 @@ let package = Package( .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), "CSwiftPluginServer", - ], - swiftSettings: [.interoperabilityMode(.Cxx)] + ] ), .target( name: "swift-wasm-plugin-server", From a670f323eb7280b60cb3ed363ae2fc6856d5e696 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:53:10 -0400 Subject: [PATCH 039/105] Target --- tools/swift-plugin-server/Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 3f38126a596de..a440aecfa2f98 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -7,7 +7,7 @@ let allowJSC = true let package = Package( name: "swift-plugin-server", platforms: [ - .macOS(.v11), + .macOS(.v10_15), ], products: [ .library(name: "swift-plugin-server", targets: ["swift-plugin-server"]), From aaaaae3d1a4df2ff0da7398e30dfdf4c5389a849 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:54:56 -0400 Subject: [PATCH 040/105] tweaks --- tools/swift-plugin-server/Package.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index a440aecfa2f98..22b4aca280bc6 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -7,7 +7,7 @@ let allowJSC = true let package = Package( name: "swift-plugin-server", platforms: [ - .macOS(.v10_15), + .macOS(.v10_15) ], products: [ .library(name: "swift-plugin-server", targets: ["swift-plugin-server"]), @@ -35,7 +35,7 @@ let package = Package( dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), - "CSwiftPluginServer", + "CSwiftPluginServer" ] ), .target( From 7b16cbdac87db0b9c3d305c16e27e2a700fc00ef Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Tue, 30 Apr 2024 15:57:10 -0400 Subject: [PATCH 041/105] =?UTF-8?q?We=20don=E2=80=99t=20need=20WasmKitPlug?= =?UTF-8?q?inError?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/swift-wasm-plugin-server/WasmKitEngine.swift | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 65a5c33e48d96..3530ae9a0cc8f 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -41,11 +41,4 @@ struct WasmKitEngine: WasmEngine { } } -struct WasmKitPluginError: Error, CustomStringConvertible { - let description: String - init(message: String) { - self.description = message - } -} - #endif From 7d9d01527b9d0cda6cc4fe5b6a78675d7e3423fc Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 5 May 2024 13:58:14 -0500 Subject: [PATCH 042/105] Change wasm versioning approach --- .../JSCWasmEngine.swift | 25 +++++------ .../swift-wasm-plugin-server/WasmEngine.swift | 45 +++++-------------- .../WasmKitEngine.swift | 11 +++-- 3 files changed, 28 insertions(+), 53 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift index 1ab07d0a3a192..7d46120f36192 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift @@ -27,7 +27,6 @@ async (wasmData, imports) => { const api = instance.exports; return { api, - customSections: (name) => WebAssembly.Module.customSections(mod, name), read: (off, size) => new Uint8Array(new Uint8Array(api.memory.buffer, off, size)).buffer, write: (buf, off, size) => new Uint8Array(api.memory.buffer, off, size).set(new Uint8Array(buf)), }; @@ -79,20 +78,18 @@ final class JSCWasmEngine: WasmEngine { memory = JSCGuestMemory(getMemory: getMemory, setMemory: setMemory) } - func customSections(named name: String) throws -> [ArraySlice] { - guard let array = runner.invokeMethod("customSections", withArguments: [name]), - let length = array.objectForKeyedSubscript("length")?.toUInt32() else { return [] } - return (0.. [UInt32] { - let result = api.invokeMethod(method, withArguments: args)! - if let exception = api.context.exception { - throw JSCWasmError(message: "Call to \(method) failed", value: exception) + func function(named name: String) throws -> WasmFunction? { + guard let export = api.objectForKeyedSubscript(name), export.isObject else { + return nil + } + return { [api] args in + let result = export.call(withArguments: args) + if let exception = api.context.exception { + throw JSCWasmError(message: "Call to '\(name)' failed", value: exception) + } + guard let result else { return [] } + return result.isUndefined ? [] : [result.toUInt32()] } - return result.isUndefined ? [] : [result.toUInt32()] } } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift index 249d27e2631ff..15958cd46bf05 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift @@ -14,12 +14,12 @@ import WASI import WasmTypes import SystemPackage +typealias WasmFunction = ([UInt32]) throws -> [UInt32] + protocol WasmEngine { init(path: FilePath, imports: WASIBridgeToHost) async throws - func customSections(named name: String) throws -> [ArraySlice] - - func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] + func function(named name: String) throws -> WasmFunction? } typealias DefaultWasmPlugin = WasmEnginePlugin @@ -28,6 +28,7 @@ typealias DefaultWasmPlugin = WasmEnginePlugin struct WasmEnginePlugin: WasmPlugin { private let hostToPlugin: FileDescriptor private let pluginToHost: FileDescriptor + private let pumpFunction: WasmFunction let engine: Engine init(path: FilePath) async throws { @@ -42,39 +43,17 @@ struct WasmEnginePlugin: WasmPlugin { stderr: .standardError ) engine = try await Engine(path: path, imports: bridge) - try checkABIVersion() - _ = try engine.invoke("_start", []) - } - private func checkABIVersion() throws { - let abiVersion = try abiVersion() - guard abiVersion == 1 else { - throw WasmEngineError(message: "Wasm plugin has unsupported ABI version: \(abiVersion)") + let exportName = "swift_wasm_macro_v1_pump" + guard let pump = try engine.function(named: exportName) else { + throw WasmEngineError(message: "Wasm plugin has an unknown ABI (could not find '\(exportName)')") } - } + self.pumpFunction = pump - private func abiVersion() throws -> UInt32 { - let sectionName = "swift_wasm_macro_abi" - let sections = try engine.customSections(named: sectionName) - switch sections.count { - case 0: - throw WasmEngineError(message: "Wasm macro is missing a '\(sectionName)' section") - case 1: - break - default: - throw WasmEngineError(message: "Wasm macro has too many '\(sectionName)' sections. Expected one, got \(sections.count)") - } - let section = sections[0] - guard section.count == 4 else { - throw WasmEngineError(message: """ - Wasm macro has incorrect '\(sectionName)' section length. Expected 4 bytes, got \(section.count). - """) - } - return section.withUnsafeBufferPointer { buffer in - buffer.withMemoryRebound(to: UInt32.self) { - UInt32(littleEndian: $0.baseAddress!.pointee) - } + guard let start = try engine.function(named: "_start") else { + throw WasmEngineError(message: "Wasm plugin does not have a '_start' entrypoint") } + _ = try start([]) } func handleMessage(_ json: [UInt8]) async throws -> [UInt8] { @@ -83,7 +62,7 @@ struct WasmEnginePlugin: WasmPlugin { } try hostToPlugin.writeAll(json) - _ = try engine.invoke("swift_wasm_macro_pump", []) + _ = try pumpFunction([]) let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift index 3530ae9a0cc8f..d91110bded97d 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift @@ -32,12 +32,11 @@ struct WasmKitEngine: WasmEngine { instance = try runtime.instantiate(module: module) } - func customSections(named name: String) throws -> [ArraySlice] { - module.customSections.filter { $0.name == name }.map(\.bytes) - } - - func invoke(_ method: String, _ args: [UInt32]) throws -> [UInt32] { - try runtime.invoke(instance, function: method, with: args.map(Value.i32)).map(\.i32) + func function(named name: String) throws -> WasmFunction? { + guard case let .function(function) = instance.exportInstances.first(where: { $0.name == name })?.value else { + return nil + } + return { args in try function.invoke(args.map(Value.i32), runtime: runtime).map(\.i32) } } } From eee48e01d032e3ffc29c59ac3211d41bab522de0 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 19:21:53 -0400 Subject: [PATCH 043/105] swift-wasm-plugin-server => swift-plugin-server --- tools/swift-plugin-server/CMakeLists.txt | 32 ++--- tools/swift-plugin-server/Package.swift | 10 +- .../JSCWasmEngine.swift | 85 ++++++------- .../WasmEngine.swift | 8 +- .../WasmKitEngine.swift | 0 .../WasmMessageHandler.swift | 69 +++++++++++ .../swift-plugin-server.swift | 6 +- .../swift-wasm-plugin-server.swift | 115 ------------------ 8 files changed, 136 insertions(+), 189 deletions(-) rename tools/swift-plugin-server/Sources/{swift-wasm-plugin-server => swift-plugin-server}/JSCWasmEngine.swift (81%) rename tools/swift-plugin-server/Sources/{swift-wasm-plugin-server => swift-plugin-server}/WasmEngine.swift (92%) rename tools/swift-plugin-server/Sources/{swift-wasm-plugin-server => swift-plugin-server}/WasmKitEngine.swift (100%) create mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift delete mode 100644 tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index cc763c0bb6d7d..3dc15506225f5 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -6,15 +6,6 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") FetchContent_MakeAvailable(WasmKit) - add_pure_swift_host_tool(swift-plugin-server - Sources/swift-plugin-server/swift-plugin-server.swift - SWIFT_COMPONENT - compiler - SWIFT_DEPENDENCIES - SwiftCompilerPluginMessageHandling - SwiftLibraryPluginProvider - ) - set(SWIFT_WASM_ALLOW_JSC ON CACHE BOOL "Use JavaScriptCore Wasm runtime if possible" FORCE) if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) @@ -31,24 +22,25 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) set(jsc_defines) endif() - add_pure_swift_host_tool(swift-wasm-plugin-server - Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift - Sources/swift-wasm-plugin-server/JSCWasmEngine.swift - Sources/swift-wasm-plugin-server/WasmEngine.swift - Sources/swift-wasm-plugin-server/WasmKitEngine.swift + add_pure_swift_host_tool(swift-plugin-server + Sources/swift-plugin-server/swift-plugin-server.swift + Sources/swift-plugin-server/JSCWasmEngine.swift + Sources/swift-plugin-server/WasmEngine.swift + Sources/swift-plugin-server/WasmKitEngine.swift + Sources/swift-plugin-server/WasmMessageHandler.swift SWIFT_COMPONENT compiler SWIFT_DEPENDENCIES SwiftCompilerPluginMessageHandling + SwiftLibraryPluginProvider ${wasi_dep} ) - target_compile_definitions(swift-wasm-plugin-server PRIVATE ${jsc_defines}) - target_include_directories(swift-wasm-plugin-server PRIVATE - Sources/CSwiftPluginServer/include - ) + + target_compile_definitions(swift-plugin-server PRIVATE ${jsc_defines}) + if (swift_wasm_use_jsc) - add_custom_command(TARGET swift-wasm-plugin-server POST_BUILD COMMAND - codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist + add_custom_command(TARGET swift-plugin-server POST_BUILD COMMAND + codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist ) endif() endif() diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 5edde5452a91c..21bfd7116d94c 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -10,8 +10,7 @@ let package = Package( .macOS(.v10_15) ], products: [ - .library(name: "swift-plugin-server", targets: ["swift-plugin-server"]), - .library(name: "swift-wasm-plugin-server", targets: ["swift-wasm-plugin-server"]), + .executable(name: "swift-plugin-server", targets: ["swift-plugin-server"]), ], dependencies: [ .package(path: "../../../swift-syntax"), @@ -22,12 +21,7 @@ let package = Package( name: "swift-plugin-server", dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), - ] - ), - .target( - name: "swift-wasm-plugin-server", - dependencies: [ - .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), + .product(name: "SwiftLibraryPluginProvider", package: "swift-syntax"), .product(name: "WASI", package: "WasmKit"), .product(name: "WasmKitWASI", package: "WasmKit", condition: .when( platforms: [.linux, .windows] + (allowJSC ? [] : [.macOS]) diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift similarity index 81% rename from tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift rename to tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift index 7d46120f36192..f293c4fbf7979 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/JSCWasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift @@ -37,10 +37,8 @@ final class JSCWasmEngine: WasmEngine { private let runner: JSValue private let api: JSValue - init(path: FilePath, imports: WASIBridgeToHost) async throws { - guard #available(macOS 10.15, *) else { fatalError("JSCWasmEngine requires macOS 10.15+") } - - let factory = try await JSCWasmFactory.shared + init(path: FilePath, imports: WASIBridgeToHost) throws { + let factory = try JSCWasmFactory.shared let context = factory.context var memory: JSCGuestMemory? @@ -70,7 +68,7 @@ final class JSCWasmEngine: WasmEngine { throw JSCWasmError(message: "Failed to load plugin", value: context.exception) } - runner = try await promise.promiseValue + runner = try promise.synchronousPromiseValue() api = runner.objectForKeyedSubscript("api")! let getMemory = runner.objectForKeyedSubscript("read")! @@ -111,14 +109,13 @@ private struct JSCGuestMemory: GuestMemory { } } -@available(macOS 10.15, *) private struct JSCWasmFactory { let context: JSContext let value: JSValue - private init() async throws { - // the VM must be created on the MainActor for async methods to work. - let vm = await MainActor.run { JSVirtualMachine() } + // NB: the VM must be created on the MainActor for promises to work. + private init() throws { + let vm = JSVirtualMachine() guard let context = JSContext(virtualMachine: vm) else { throw JSCWasmError(message: "Failed to load plugin") } @@ -126,13 +123,13 @@ private struct JSCWasmFactory { self.value = context.evaluateScript(js) } - private static let _shared = Task { - try await JSCWasmFactory() + private static let _shared = Result { + try JSCWasmFactory() } static var shared: JSCWasmFactory { - get async throws { - try await _shared.value + get throws { + try _shared.get() } } } @@ -219,34 +216,40 @@ extension JSValue { withUnsafeArrayBuffer { Data($0) } } - @available(macOS 10.15, *) - fileprivate var promiseValue: JSValue { - get async throws { - try await withCheckedThrowingContinuation { continuation in - typealias Handler = @convention(block) (JSValue) -> Void - let successFunc = JSValue( - object: { - continuation.resume(returning: $0) - } as Handler, - in: context - ) - let rejectFunc = JSValue( - object: { error in - continuation.resume( - throwing: JSCWasmError(message: "Promise rejected", value: error) - ) - } as @convention(block) (JSValue) -> Void, - in: context - ) - guard let successFunc, let rejectFunc else { - continuation.resume(throwing: JSCWasmError(message: "Could not await promise")) - return - } - invokeMethod("then", withArguments: [successFunc, rejectFunc]) - if let exception = context.exception { - continuation.resume(throwing: JSCWasmError(message: "Promise.then threw", value: exception)) - } - } + private func getPromise(completion: @escaping (Result) -> Void) { + typealias Handler = @convention(block) (JSValue) -> Void + let successFunc = JSValue( + object: { + completion(.success($0)) + } as Handler, + in: context + ) + let rejectFunc = JSValue( + object: { error in + completion(.failure( + JSCWasmError(message: "Promise rejected", value: error) + )) + } as @convention(block) (JSValue) -> Void, + in: context + ) + guard let successFunc, let rejectFunc else { + completion(.failure(JSCWasmError(message: "Could not await promise"))) + return + } + invokeMethod("then", withArguments: [successFunc, rejectFunc]) + if let exception = context.exception { + completion(.failure(JSCWasmError(message: "Promise.then threw", value: exception))) + } + } + + func synchronousPromiseValue() throws -> JSValue { + var result: Result? + getPromise { + result = $0 + } + while true { + if let result { return try result.get() } + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.01)) } } } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift similarity index 92% rename from tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift rename to tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 15958cd46bf05..2f7d8a5174a98 100644 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -17,7 +17,7 @@ import SystemPackage typealias WasmFunction = ([UInt32]) throws -> [UInt32] protocol WasmEngine { - init(path: FilePath, imports: WASIBridgeToHost) async throws + init(path: FilePath, imports: WASIBridgeToHost) throws func function(named name: String) throws -> WasmFunction? } @@ -31,7 +31,7 @@ struct WasmEnginePlugin: WasmPlugin { private let pumpFunction: WasmFunction let engine: Engine - init(path: FilePath) async throws { + init(path: FilePath) throws { let hostToPluginPipes = try FileDescriptor.pipe() let pluginToHostPipes = try FileDescriptor.pipe() self.hostToPlugin = hostToPluginPipes.writeEnd @@ -42,7 +42,7 @@ struct WasmEnginePlugin: WasmPlugin { stdout: pluginToHostPipes.writeEnd, stderr: .standardError ) - engine = try await Engine(path: path, imports: bridge) + engine = try Engine(path: path, imports: bridge) let exportName = "swift_wasm_macro_v1_pump" guard let pump = try engine.function(named: exportName) else { @@ -56,7 +56,7 @@ struct WasmEnginePlugin: WasmPlugin { _ = try start([]) } - func handleMessage(_ json: [UInt8]) async throws -> [UInt8] { + func handleMessage(_ json: [UInt8]) throws -> [UInt8] { try withUnsafeBytes(of: UInt64(json.count).littleEndian) { _ = try hostToPlugin.writeAll($0) } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift similarity index 100% rename from tools/swift-plugin-server/Sources/swift-wasm-plugin-server/WasmKitEngine.swift rename to tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift new file mode 100644 index 0000000000000..85b17ee6260cf --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -0,0 +1,69 @@ +import SystemPackage +import Dispatch +@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling + +/// A `PluginMessageHandler` that intercepts messages intended for Wasm plugins. +final class WasmInterceptingMessageHandler: PluginMessageHandler { + private var loadedWasmPlugins: [String: WasmPlugin] = [:] + + let base: Base + init(base: Base) { + self.base = base + } + + func handleMessage(_ message: HostToPluginMessage) -> PluginToHostMessage { + switch message { + case .loadPluginLibrary(let libraryPath, let moduleName): + guard libraryPath.hasSuffix(".wasm") else { break } + do { + loadedWasmPlugins[moduleName] = try defaultWasmPlugin.init(path: FilePath(libraryPath)) + } catch { + printError("Error: \(error)") + return .loadPluginLibraryResult(loaded: false, diagnostics: []) + } + return .loadPluginLibraryResult(loaded: true, diagnostics: []) + case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), + .expandFreestandingMacro(let macro, _, _, _, _): + guard let wasmPlugin = loadedWasmPlugins[macro.moduleName] else { break } + return expandMacro(plugin: wasmPlugin, message: message) + case .getCapability: + break +#if !SWIFT_PACKAGE + @unknown default: + break +#endif + } + return base.handleMessage(message) + } + + private func expandMacro( + plugin: WasmPlugin, + message: HostToPluginMessage + ) -> PluginToHostMessage { + do { + let request = try JSON.encode(message) + let responseRaw = try plugin.handleMessage(request) + return try responseRaw.withUnsafeBytes { + try $0.withMemoryRebound(to: UInt8.self) { + try JSON.decode(PluginToHostMessage.self, from: $0) + } + } + } catch { + printError("Error: \(error)") + return .expandMacroResult(expandedSource: nil, diagnostics: []) + } + } +} + +protocol WasmPlugin { + init(path: FilePath) throws + + func handleMessage(_ json: [UInt8]) throws -> [UInt8] +} + +private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } + +// TODO: return actual diagnostics instead of using this +private func printError(_ error: String) { + _ = try? FileDescriptor.standardError.writeAll("\(error)\n".utf8) +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 1b97b4fd0af17..08be23e63a749 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -19,7 +19,11 @@ final class SwiftPluginServer { let connection = try StandardIOMessageConnection() let listener = CompilerPluginMessageListener( connection: connection, - provider: LibraryPluginProvider.shared + messageHandler: WasmInterceptingMessageHandler( + base: PluginProviderMessageHandler( + provider: LibraryPluginProvider.shared + ) + ) ) listener.main() } diff --git a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift deleted file mode 100644 index b76159957a9b3..0000000000000 --- a/tools/swift-plugin-server/Sources/swift-wasm-plugin-server/swift-wasm-plugin-server.swift +++ /dev/null @@ -1,115 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -@_spi(PluginMessage) import SwiftCompilerPluginMessageHandling -import SystemPackage - -@main -final class SwiftPluginServer { - let connection: StandardIOMessageConnection - var loadedWasmPlugins: [String: WasmPlugin] = [:] - var hostCapability: PluginMessage.HostCapability? - - init(connection: StandardIOMessageConnection) { - self.connection = connection - } - - private func sendMessage(_ message: PluginToHostMessage) throws { - try connection.sendMessage(message) - } - - private func loadPluginLibrary(path: String, moduleName: String) async throws -> WasmPlugin { - return try await defaultWasmPlugin.init(path: FilePath(path)) - } - - private func expandMacro( - moduleName module: String, - message: HostToPluginMessage - ) async throws { - let response: PluginToHostMessage - do { - guard let plugin = loadedWasmPlugins[module] else { throw PluginServerError(message: "Could not find module \(module)") } - let request = try JSON.encode(message) - let responseRaw = try await plugin.handleMessage(request) - response = try responseRaw.withUnsafeBytes { - try $0.withMemoryRebound(to: UInt8.self) { - try JSON.decode(PluginToHostMessage.self, from: $0) - } - } - } catch { - printError("Error: \(error)") - try sendMessage(.expandMacroResult(expandedSource: nil, diagnostics: [])) - return - } - try sendMessage(response) - } - - func run() async throws { - while let message = try connection.waitForNextMessage(HostToPluginMessage.self) { - switch message { - case .getCapability(let hostCapability): - // Remember the peer capability if provided. - if let hostCapability = hostCapability { - self.hostCapability = .init(hostCapability) - } - - // Return the plugin capability. - let capability = PluginMessage.PluginCapability( - protocolVersion: PluginMessage.PROTOCOL_VERSION_NUMBER, - features: [PluginFeature.loadPluginLibrary.rawValue] - ) - try sendMessage(.getCapabilityResult(capability: capability)) - case .loadPluginLibrary(let libraryPath, let moduleName): - do { - loadedWasmPlugins[moduleName] = try await loadPluginLibrary(path: libraryPath, moduleName: moduleName) - } catch { - printError("Error: \(error)") - try sendMessage(.loadPluginLibraryResult(loaded: false, diagnostics: [])) - continue - } - try sendMessage(.loadPluginLibraryResult(loaded: true, diagnostics: [])) - case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), - .expandFreestandingMacro(let macro, _, _, _, _): - try await expandMacro(moduleName: macro.moduleName, message: message) - #if !SWIFT_PACKAGE - @unknown default: - break - #endif - } - } - } - - /// @main entry point. - static func main() async throws { - let connection = try StandardIOMessageConnection() - try await Self(connection: connection).run() - } -} - -private func printError(_ error: String) { - _ = try? FileDescriptor.standardError.writeAll("\(error)\n".utf8) -} - -protocol WasmPlugin { - init(path: FilePath) async throws - - func handleMessage(_ json: [UInt8]) async throws -> [UInt8] -} - -private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } - -struct PluginServerError: Error, CustomStringConvertible { - var description: String - init(message: String) { - self.description = message - } -} From 4fca1028237d0d5541ff46482521e13033f0f969 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 19:24:21 -0400 Subject: [PATCH 044/105] =?UTF-8?q?swift-plugin-server=20doesn=E2=80=99t?= =?UTF-8?q?=20need=20a=20product?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/swift-plugin-server/Package.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 21bfd7116d94c..150d82a4101df 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -9,9 +9,6 @@ let package = Package( platforms: [ .macOS(.v10_15) ], - products: [ - .executable(name: "swift-plugin-server", targets: ["swift-plugin-server"]), - ], dependencies: [ .package(path: "../../../swift-syntax"), .package(path: "../../../wasmkit"), From d465d369528a1a0581d3959ab2ac2f5d4713c6a1 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 19:28:10 -0400 Subject: [PATCH 045/105] =?UTF-8?q?We=20don=E2=80=99t=20need=20Dispatch=20?= =?UTF-8?q?here?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Sources/swift-plugin-server/WasmMessageHandler.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index 85b17ee6260cf..9c8a905c5fc17 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -1,5 +1,4 @@ import SystemPackage -import Dispatch @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling /// A `PluginMessageHandler` that intercepts messages intended for Wasm plugins. From c4efcee689dd35549336051a859f2cd7d8817711 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 19:44:57 -0400 Subject: [PATCH 046/105] Improve diagnostics --- .../WasmMessageHandler.swift | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index 9c8a905c5fc17..8d18abbef034e 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -10,21 +10,27 @@ final class WasmInterceptingMessageHandler: PluginMe self.base = base } + /// Handle the message ourselves if it references a Wasm plugin. + /// Otherwise, forward it to `base`. func handleMessage(_ message: HostToPluginMessage) -> PluginToHostMessage { switch message { case .loadPluginLibrary(let libraryPath, let moduleName): guard libraryPath.hasSuffix(".wasm") else { break } + let libraryFilePath = FilePath(libraryPath) do { - loadedWasmPlugins[moduleName] = try defaultWasmPlugin.init(path: FilePath(libraryPath)) + loadedWasmPlugins[moduleName] = try defaultWasmPlugin.init(path: libraryFilePath) } catch { - printError("Error: \(error)") - return .loadPluginLibraryResult(loaded: false, diagnostics: []) + return .loadPluginLibraryResult( + loaded: false, + diagnostics: [PluginMessage.Diagnostic(errorMessage: "\(error)")] + ) } return .loadPluginLibraryResult(loaded: true, diagnostics: []) case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), .expandFreestandingMacro(let macro, _, _, _, _): - guard let wasmPlugin = loadedWasmPlugins[macro.moduleName] else { break } - return expandMacro(plugin: wasmPlugin, message: message) + if let response = expandMacro(macro, message: message) { + return response + } // else break case .getCapability: break #if !SWIFT_PACKAGE @@ -36,9 +42,10 @@ final class WasmInterceptingMessageHandler: PluginMe } private func expandMacro( - plugin: WasmPlugin, + _ macro: PluginMessage.MacroReference, message: HostToPluginMessage - ) -> PluginToHostMessage { + ) -> PluginToHostMessage? { + guard let plugin = loadedWasmPlugins[macro.moduleName] else { return nil } do { let request = try JSON.encode(message) let responseRaw = try plugin.handleMessage(request) @@ -48,12 +55,33 @@ final class WasmInterceptingMessageHandler: PluginMe } } } catch { - printError("Error: \(error)") - return .expandMacroResult(expandedSource: nil, diagnostics: []) + return .expandMacroResult( + expandedSource: nil, + diagnostics: [PluginMessage.Diagnostic( + errorMessage: """ + failed to communicate with external macro implementation type \ + '\(macro.moduleName).\(macro.typeName)' to expand macro '\(macro.name)()'; \ + \(error) + """ + )] + ) } } } +extension PluginMessage.Diagnostic { + fileprivate init(errorMessage: String) { + self.init( + message: errorMessage, + severity: .error, + position: .invalid, + highlights: [], + notes: [], + fixIts: [] + ) + } +} + protocol WasmPlugin { init(path: FilePath) throws @@ -61,8 +89,3 @@ protocol WasmPlugin { } private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } - -// TODO: return actual diagnostics instead of using this -private func printError(_ error: String) { - _ = try? FileDescriptor.standardError.writeAll("\(error)\n".utf8) -} From b3f53c2c12abf47e23cf5c785f7655383643b8fb Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 19:57:06 -0400 Subject: [PATCH 047/105] Remove JSC for now --- tools/swift-plugin-server/CMakeLists.txt | 26 +- tools/swift-plugin-server/Package.swift | 9 +- .../swift-plugin-server/JSCWasmEngine.swift | 257 ------------------ .../swift-plugin-server/WasmKitEngine.swift | 4 - tools/swift-plugin-server/allow-jit.plist | 8 - 5 files changed, 3 insertions(+), 301 deletions(-) delete mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift delete mode 100644 tools/swift-plugin-server/allow-jit.plist diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 3dc15506225f5..44edb4bdda37e 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -6,25 +6,8 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") FetchContent_MakeAvailable(WasmKit) - set(SWIFT_WASM_ALLOW_JSC ON CACHE BOOL "Use JavaScriptCore Wasm runtime if possible" FORCE) - - if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS) - set(swift_wasm_use_jsc ${SWIFT_WASM_ALLOW_JSC}) - else() - set(swift_wasm_use_jsc OFF) - endif() - - if (swift_wasm_use_jsc) - set(wasi_dep WASI) - set(jsc_defines SWIFT_WASM_USE_JSC) - else() - set(wasi_dep WasmKitWASI) - set(jsc_defines) - endif() - add_pure_swift_host_tool(swift-plugin-server Sources/swift-plugin-server/swift-plugin-server.swift - Sources/swift-plugin-server/JSCWasmEngine.swift Sources/swift-plugin-server/WasmEngine.swift Sources/swift-plugin-server/WasmKitEngine.swift Sources/swift-plugin-server/WasmMessageHandler.swift @@ -33,14 +16,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SWIFT_DEPENDENCIES SwiftCompilerPluginMessageHandling SwiftLibraryPluginProvider + SystemPackage ${wasi_dep} ) - - target_compile_definitions(swift-plugin-server PRIVATE ${jsc_defines}) - - if (swift_wasm_use_jsc) - add_custom_command(TARGET swift-plugin-server POST_BUILD COMMAND - codesign -fs - $ --entitlements ${CMAKE_CURRENT_SOURCE_DIR}/allow-jit.plist - ) - endif() endif() diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 150d82a4101df..35c4e410adb16 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -2,8 +2,6 @@ import PackageDescription -let allowJSC = true - let package = Package( name: "swift-plugin-server", platforms: [ @@ -20,11 +18,8 @@ let package = Package( .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "SwiftLibraryPluginProvider", package: "swift-syntax"), .product(name: "WASI", package: "WasmKit"), - .product(name: "WasmKitWASI", package: "WasmKit", condition: .when( - platforms: [.linux, .windows] + (allowJSC ? [] : [.macOS]) - )), - ], - swiftSettings: allowJSC ? [.define("SWIFT_WASM_USE_JSC", .when(platforms: [.macOS]))] : [] + .product(name: "WasmKitWASI", package: "WasmKit"), + ] ), ], cxxLanguageStandard: .cxx17 diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift deleted file mode 100644 index f293c4fbf7979..0000000000000 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/JSCWasmEngine.swift +++ /dev/null @@ -1,257 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See http://swift.org/LICENSE.txt for license information -// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#if SWIFT_WASM_USE_JSC - -import JavaScriptCore -import WASI -import WasmTypes -import SystemPackage - -typealias DefaultWasmEngine = JSCWasmEngine - -// (wasm: ArrayBuffer, imports: Object) => Promise<{ ... }> -private let js = """ -async (wasmData, imports) => { - const mod = await WebAssembly.compile(wasmData); - const instance = await WebAssembly.instantiate(mod, imports); - const api = instance.exports; - return { - api, - read: (off, size) => new Uint8Array(new Uint8Array(api.memory.buffer, off, size)).buffer, - write: (buf, off, size) => new Uint8Array(api.memory.buffer, off, size).set(new Uint8Array(buf)), - }; -} -""" - -final class JSCWasmEngine: WasmEngine { - private let runner: JSValue - private let api: JSValue - - init(path: FilePath, imports: WASIBridgeToHost) throws { - let factory = try JSCWasmFactory.shared - let context = factory.context - - var memory: JSCGuestMemory? - let imports: [String: [String: JSValue]] = imports.wasiHostModules.mapValues { module in - module.functions.mapValues { function in - function.asJSFunction(in: context) { - guard let memory else { throw JSCWasmError(message: "Missing GuestMemory") } - return memory - } - } - } - - let buffer = try NSData(contentsOfFile: path.string, options: .mappedIfSafe) - let jsBuf = try JSValue( - // JavaScript doesn't have readonly ArrayBuffers but all the JS we're running - // is contained within this file and we know that we aren't mutating this buffer. - // If we did attempt to mutate it, it would be UB. - newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer( - start: UnsafeMutableRawPointer(mutating: buffer.bytes), - count: buffer.count - ), - deallocator: { [box = buffer as AnyObject] in _ = box }, - in: context - ) - let promise = factory.value.call(withArguments: [jsBuf, imports]) - guard let promise, context.exception == nil else { - throw JSCWasmError(message: "Failed to load plugin", value: context.exception) - } - - runner = try promise.synchronousPromiseValue() - api = runner.objectForKeyedSubscript("api")! - - let getMemory = runner.objectForKeyedSubscript("read")! - let setMemory = runner.objectForKeyedSubscript("write")! - memory = JSCGuestMemory(getMemory: getMemory, setMemory: setMemory) - } - - func function(named name: String) throws -> WasmFunction? { - guard let export = api.objectForKeyedSubscript(name), export.isObject else { - return nil - } - return { [api] args in - let result = export.call(withArguments: args) - if let exception = api.context.exception { - throw JSCWasmError(message: "Call to '\(name)' failed", value: exception) - } - guard let result else { return [] } - return result.isUndefined ? [] : [result.toUInt32()] - } - } -} - -private struct JSCGuestMemory: GuestMemory { - let getMemory: JSValue - let setMemory: JSValue - - func withUnsafeMutableBufferPointer( - offset: UInt, - count: Int, - _ body: (UnsafeMutableRawBufferPointer) throws -> T - ) rethrows -> T { - // note that we can't get a shared pointer to the original wasm memory - // because JSC doesn't allow JSObjectGetArrayBufferBytesPtr on Wasm memory. - // we perform targeted copies instead. - let memory = getMemory.call(withArguments: [offset, count])! - defer { setMemory.call(withArguments: [memory, offset, count]) } - return try memory.withUnsafeArrayBuffer(body) - } -} - -private struct JSCWasmFactory { - let context: JSContext - let value: JSValue - - // NB: the VM must be created on the MainActor for promises to work. - private init() throws { - let vm = JSVirtualMachine() - guard let context = JSContext(virtualMachine: vm) else { - throw JSCWasmError(message: "Failed to load plugin") - } - self.context = context - self.value = context.evaluateScript(js) - } - - private static let _shared = Result { - try JSCWasmFactory() - } - - static var shared: JSCWasmFactory { - get throws { - try _shared.get() - } - } -} - -public struct JSCWasmError: Error, CustomStringConvertible { - public let value: JSValue? - public let description: String - public init(message: String, value: JSValue? = nil) { - self.value = value - self.description = "\(message)\(value.map { ": \($0)" } ?? "")" - } -} - -extension WASIHostFunction { - fileprivate func asJSFunction(in context: JSContext, memory: @escaping () throws -> GuestMemory) -> JSValue { - JSValue(object: { - let arguments = JSContext.currentArguments() as? [JSValue] ?? [] - let types: [Value] = zip(arguments, type.parameters).map { argument, type in - switch type { - case .i32: .i32(argument.toUInt32()) - case .i64: .i64(UInt64(argument.toString()) ?? 0) - case .f32, .f64, .ref: .i32(0) // WASI doesn't support these - } - } - do { - let memory = try memory() - if let result = try self.implementation(memory, types).first { - return JSValue(uInt32: result.i32, in: context) - } - } catch { - context.exception = JSValue(object: "\(error)", in: context) - } - return JSValue(undefinedIn: context) - } as @convention(block) () -> JSValue, in: context)! - } -} - -extension JSValue { - fileprivate convenience init( - newArrayBufferWithData data: Data, - in context: JSContext - ) throws { - let copy = UnsafeMutableBufferPointer.allocate(capacity: data.count) - _ = copy.initialize(from: data) - let address = UInt(bitPattern: copy.baseAddress) - try self.init( - newArrayBufferWithBytesNoCopy: UnsafeMutableRawBufferPointer(copy), - deallocator: { UnsafeMutableRawPointer(bitPattern: address)?.deallocate() }, - in: context - ) - } - - fileprivate convenience init( - newArrayBufferWithBytesNoCopy bytes: UnsafeMutableRawBufferPointer, - deallocator: @escaping @Sendable () -> Void, - in context: JSContext - ) throws { - let box = Unmanaged.passRetained(deallocator as AnyObject).toOpaque() - let rawBuf = JSObjectMakeArrayBufferWithBytesNoCopy( - context.jsGlobalContextRef, - bytes.baseAddress, bytes.count, - { _, box in (Unmanaged.fromOpaque(box!).takeRetainedValue() as! () -> Void)() }, - box, - nil - ) - self.init(jsValueRef: rawBuf, in: context) - } - - // it's unsafe to call JavaScriptCore APIs inside the `perform` block as this - // might invalidate the buffer. - fileprivate func withUnsafeArrayBuffer( - _ perform: (UnsafeMutableRawBufferPointer) throws -> Result - ) rethrows -> Result { - let rawContext = context.jsGlobalContextRef - guard let base = JSObjectGetArrayBufferBytesPtr(rawContext, jsValueRef, nil) else { - return try perform(UnsafeMutableRawBufferPointer(start: nil, count: 0)) - } - let count = JSObjectGetArrayBufferByteLength(rawContext, jsValueRef, nil) - let buffer = UnsafeMutableRawBufferPointer(start: base, count: count) - return try perform(buffer) - } - - fileprivate func arrayBufferData() -> Data { - withUnsafeArrayBuffer { Data($0) } - } - - private func getPromise(completion: @escaping (Result) -> Void) { - typealias Handler = @convention(block) (JSValue) -> Void - let successFunc = JSValue( - object: { - completion(.success($0)) - } as Handler, - in: context - ) - let rejectFunc = JSValue( - object: { error in - completion(.failure( - JSCWasmError(message: "Promise rejected", value: error) - )) - } as @convention(block) (JSValue) -> Void, - in: context - ) - guard let successFunc, let rejectFunc else { - completion(.failure(JSCWasmError(message: "Could not await promise"))) - return - } - invokeMethod("then", withArguments: [successFunc, rejectFunc]) - if let exception = context.exception { - completion(.failure(JSCWasmError(message: "Promise.then threw", value: exception))) - } - } - - func synchronousPromiseValue() throws -> JSValue { - var result: Result? - getPromise { - result = $0 - } - while true { - if let result { return try result.get() } - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.01)) - } - } -} - -#endif diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index d91110bded97d..01d7f8780504e 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -10,8 +10,6 @@ // //===----------------------------------------------------------------------===// -#if !SWIFT_WASM_USE_JSC - import WASI import WasmKit import WasmKitWASI @@ -39,5 +37,3 @@ struct WasmKitEngine: WasmEngine { return { args in try function.invoke(args.map(Value.i32), runtime: runtime).map(\.i32) } } } - -#endif diff --git a/tools/swift-plugin-server/allow-jit.plist b/tools/swift-plugin-server/allow-jit.plist deleted file mode 100644 index 005d584f6fa8b..0000000000000 --- a/tools/swift-plugin-server/allow-jit.plist +++ /dev/null @@ -1,8 +0,0 @@ - - - - -com.apple.security.cs.allow-jit - - - From b62948479579549f9a78b5a47c8da8b5460cc549 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:00:25 -0400 Subject: [PATCH 048/105] Explicitly depend on System --- tools/swift-plugin-server/Package.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 35c4e410adb16..96626ad6a1ff9 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -9,6 +9,7 @@ let package = Package( ], dependencies: [ .package(path: "../../../swift-syntax"), + .package(path: "../../../swift-system"), .package(path: "../../../wasmkit"), ], targets: [ @@ -17,6 +18,7 @@ let package = Package( dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "SwiftLibraryPluginProvider", package: "swift-syntax"), + .product(name: "SystemPackage", package: "swift-system"), .product(name: "WASI", package: "WasmKit"), .product(name: "WasmKitWASI", package: "WasmKit"), ] From af2b87ecfff9cde61886b9385931822e205fc7e6 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:01:45 -0400 Subject: [PATCH 049/105] =?UTF-8?q?Nevermind=20SwiftPM=20didn=E2=80=99t=20?= =?UTF-8?q?like=20that?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/swift-plugin-server/Package.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 96626ad6a1ff9..35c4e410adb16 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -9,7 +9,6 @@ let package = Package( ], dependencies: [ .package(path: "../../../swift-syntax"), - .package(path: "../../../swift-system"), .package(path: "../../../wasmkit"), ], targets: [ @@ -18,7 +17,6 @@ let package = Package( dependencies: [ .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), .product(name: "SwiftLibraryPluginProvider", package: "swift-syntax"), - .product(name: "SystemPackage", package: "swift-system"), .product(name: "WASI", package: "WasmKit"), .product(name: "WasmKitWASI", package: "WasmKit"), ] From bbb916c4a806f63b1f470dda7027841c55df8308 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:03:32 -0400 Subject: [PATCH 050/105] Lower swift-tools-version --- tools/swift-plugin-server/Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index 35c4e410adb16..752b26d6ab8cb 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.9 +// swift-tools-version: 5.6 import PackageDescription From 77c273f264ae7837f54b91e4c7d5c8bcaa573ba4 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:05:33 -0400 Subject: [PATCH 051/105] Fix WASI dep --- tools/swift-plugin-server/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 44edb4bdda37e..47aa4f672b6a1 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -16,7 +16,6 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) SWIFT_DEPENDENCIES SwiftCompilerPluginMessageHandling SwiftLibraryPluginProvider - SystemPackage - ${wasi_dep} + WasmKitWASI ) endif() From b56e28a89eb0de79b8d407e7b8175ad7a01126d2 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:17:54 -0400 Subject: [PATCH 052/105] Add missing header --- .../swift-plugin-server/WasmMessageHandler.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index 8d18abbef034e..baa6586833b3e 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -1,3 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + import SystemPackage @_spi(PluginMessage) import SwiftCompilerPluginMessageHandling From 14b5b79b2098c5b15161498629901070f1d21acf Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:32:27 -0400 Subject: [PATCH 053/105] Include position in diagnostics if possible --- .../WasmMessageHandler.swift | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index baa6586833b3e..1ec17d9638cc5 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -38,9 +38,9 @@ final class WasmInterceptingMessageHandler: PluginMe ) } return .loadPluginLibraryResult(loaded: true, diagnostics: []) - case .expandAttachedMacro(let macro, _, _, _, _, _, _, _, _), - .expandFreestandingMacro(let macro, _, _, _, _): - if let response = expandMacro(macro, message: message) { + case .expandAttachedMacro(let macro, _, _, let syntax, _, _, _, _, _), + .expandFreestandingMacro(let macro, _, _, let syntax, _): + if let response = expandMacro(macro, message: message, location: syntax.location) { return response } // else break case .getCapability: @@ -55,7 +55,8 @@ final class WasmInterceptingMessageHandler: PluginMe private func expandMacro( _ macro: PluginMessage.MacroReference, - message: HostToPluginMessage + message: HostToPluginMessage, + location: PluginMessage.SourceLocation? ) -> PluginToHostMessage? { guard let plugin = loadedWasmPlugins[macro.moduleName] else { return nil } do { @@ -72,9 +73,10 @@ final class WasmInterceptingMessageHandler: PluginMe diagnostics: [PluginMessage.Diagnostic( errorMessage: """ failed to communicate with external macro implementation type \ - '\(macro.moduleName).\(macro.typeName)' to expand macro '\(macro.name)()'; \ + '\(macro.moduleName).\(macro.typeName)' to expand macro '\(macro.name)'; \ \(error) - """ + """, + position: location?.position ?? .invalid )] ) } @@ -82,11 +84,14 @@ final class WasmInterceptingMessageHandler: PluginMe } extension PluginMessage.Diagnostic { - fileprivate init(errorMessage: String) { + fileprivate init( + errorMessage: String, + position: PluginMessage.Diagnostic.Position = .invalid + ) { self.init( message: errorMessage, severity: .error, - position: .invalid, + position: position, highlights: [], notes: [], fixIts: [] @@ -94,6 +99,12 @@ extension PluginMessage.Diagnostic { } } +extension PluginMessage.SourceLocation { + fileprivate var position: PluginMessage.Diagnostic.Position { + .init(fileName: fileName, offset: offset) + } +} + protocol WasmPlugin { init(path: FilePath) throws From 92aef4eb3d9de5631a6ce8debc4acc7a62fc9846 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 20:36:31 -0400 Subject: [PATCH 054/105] this comment is now stale --- .../Sources/swift-plugin-server/WasmKitEngine.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index 01d7f8780504e..c712a544d98e3 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -23,8 +23,6 @@ struct WasmKitEngine: WasmEngine { private let runtime: Runtime init(path: FilePath, imports: WASIBridgeToHost) throws { - // we never call wasm.deallocator, effectively leaking the data, - // but that's intentional because plugins can't be "unloaded" module = try parseWasm(filePath: path) runtime = Runtime(hostModules: imports.hostModules) instance = try runtime.instantiate(module: module) From 9b1e4ee063d7f8cbe8800b6511e4658526d97073 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 22:27:10 -0400 Subject: [PATCH 055/105] No wasm_plugin_server_path --- include/swift/AST/SearchPathOptions.h | 16 +++++++--- include/swift/Option/FrontendOptions.td | 5 --- include/swift/Option/Options.td | 7 +++++ lib/AST/ModuleDependencies.cpp | 8 +++++ lib/AST/PluginLoader.cpp | 23 +++++++------- lib/Frontend/CompilerInvocation.cpp | 31 ++++++++++++++++--- lib/Serialization/ModuleFileSharedCore.cpp | 3 ++ lib/Serialization/ModuleFormat.h | 3 +- lib/Serialization/Serialization.cpp | 11 +++++++ .../lldb-moduleimport-test.cpp | 3 ++ tools/swift-ide-test/swift-ide-test.cpp | 23 ++++++++++++++ 11 files changed, 107 insertions(+), 26 deletions(-) diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index 6cc9bc102ff17..a251d326dd4a2 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -213,17 +213,23 @@ class PluginSearchOption { std::string SearchPath; std::string ServerPath; }; + struct LoadPlugin { + std::string LibraryPath; + std::string ServerPath; + std::vector ModuleNames; + }; enum class Kind : uint8_t { LoadPluginLibrary, LoadPluginExecutable, PluginPath, ExternalPluginPath, + LoadPlugin, }; private: using Members = ExternalUnionMembers; + PluginPath, ExternalPluginPath, LoadPlugin>; static Members::Index getIndexForKind(Kind kind) { switch (kind) { case Kind::LoadPluginLibrary: @@ -234,6 +240,8 @@ class PluginSearchOption { return Members::indexOf(); case Kind::ExternalPluginPath: return Members::indexOf(); + case Kind::LoadPlugin: + return Members::indexOf(); } }; using Storage = ExternalUnion; @@ -257,6 +265,9 @@ class PluginSearchOption { : kind(Kind::ExternalPluginPath) { storage.emplace(kind, v); } + PluginSearchOption(const LoadPlugin &v) : kind(Kind::LoadPlugin) { + storage.emplace(kind, v); + } PluginSearchOption(const PluginSearchOption &o) : kind(o.kind) { storage.copyConstruct(o.kind, o.storage); } @@ -468,9 +479,6 @@ class SearchPathOptions { /// Plugin search path options. std::vector PluginSearchOpts; - /// Path to swift-wasm-plugin-server executable. - std::string PluginWasmServerPath; - /// Don't look in for compiler-provided modules. bool SkipRuntimeLibraryImportPaths = false; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index ff949d35efb6c..16081dee45216 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -284,11 +284,6 @@ def Raccess_note_EQ : Joined<["-"], "Raccess-note=">, def block_list_file : Separate<["-"], "blocklist-file">, MetaVarName<"">, HelpText<"The path to a blocklist configuration file">; - -def wasm_plugin_server_path : Separate<["-"], "wasm-plugin-server-path">, - Flags<[DoesNotAffectIncrementalBuild, ArgumentIsPath]>, - HelpText<"Path to the swift-wasm-plugin-server executable">, - MetaVarName<"">; } // end let Flags = [FrontendOption, NoDriverOption] def debug_crash_Group : OptionGroup<"">; diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index a232e65e7cff0..ed0eb37fe8128 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -2002,6 +2002,13 @@ def load_plugin_executable: "of module names where the macro types are declared">, MetaVarName<"#">; +def load_plugin: + Separate<["-"], "load-plugin">, Group, + Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>, + HelpText<"Path to a plugin library, a server to load it in, and a comma-separated " + "list of module names where the macro types are declared">, + MetaVarName<":#">; + def disable_sandbox: Flag<["-"], "disable-sandbox">, Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>, diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 618876627fb7d..b1620e1a0a254 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -584,6 +584,14 @@ void SwiftDependencyTracker::addCommonSearchPathDeps( recordFiles(val.SearchPath); break; } + + // '-load-plugin :#,...'. + case PluginSearchOption::Kind::LoadPlugin: { + auto &val = entry.get(); + FS->status(val.LibraryPath); + // TODO: what else do we need? + break; + } } } } diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index bb259ab56c06f..3b3e00f448772 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -116,18 +116,17 @@ PluginLoader::getPluginMap() { case PluginSearchOption::Kind::LoadPluginExecutable: { auto &val = entry.get(); assert(!val.ExecutablePath.empty() && "empty plugin path"); - if (llvm::sys::path::filename(val.ExecutablePath).ends_with(".wasm")) { - // we treat wasm plugins like library plugins that can be loaded by an external - // "wasm server" that in turn invokes the wasm runtime. - const auto &wasmServerPath = Ctx.SearchPathOpts.PluginWasmServerPath; - assert(!wasmServerPath.empty() && "wasm load requested but got empty wasm server path"); - for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, val.ExecutablePath, wasmServerPath); - } - } else { - for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, /*libraryPath=*/"", val.ExecutablePath); - } + for (auto &moduleName : val.ModuleNames) { + try_emplace(moduleName, /*libraryPath=*/"", val.ExecutablePath); + } + continue; + } + + case PluginSearchOption::Kind::LoadPlugin: { + auto &val = entry.get(); + std::vector moduleNames = val.ModuleNames; + for (auto &moduleName : moduleNames) { + try_emplace(moduleName, val.LibraryPath, val.ServerPath); } continue; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index b37ab4b477ad0..8e811f81fd794 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2027,6 +2027,30 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, } break; } + case OPT_load_plugin: { + // ':#' where the module names are + // comma separated. + StringRef pathAndServer; + StringRef modulesStr; + std::tie(pathAndServer, modulesStr) = StringRef(A->getValue()).rsplit('#'); + StringRef path; + StringRef server; + std::tie(path, server) = pathAndServer.rsplit(':'); + std::vector moduleNames; + for (auto name : llvm::split(modulesStr, ',')) { + moduleNames.emplace_back(name); + } + if (path.empty() || server.empty() || moduleNames.empty()) { + Diags.diagnose(SourceLoc(), diag::error_load_plugin_executable, + A->getValue()); + } else { + Opts.PluginSearchOpts.emplace_back( + PluginSearchOption::LoadPlugin{resolveSearchPath(path), + resolveSearchPath(server), + std::move(moduleNames)}); + } + break; + } case OPT_plugin_path: { Opts.PluginSearchOpts.emplace_back( PluginSearchOption::PluginPath{resolveSearchPath(A->getValue())}); @@ -2043,14 +2067,13 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, break; } default: + llvm::errs() << "option: "; + A->getOption().print(llvm::errs()); + llvm::errs() << "\n"; llvm_unreachable("unhandled plugin search option"); } } - if (const Arg *A = Args.getLastArg(OPT_wasm_plugin_server_path)) { - Opts.PluginWasmServerPath = A->getValue(); - } - for (const Arg *A : Args.filtered(OPT_L)) { Opts.LibrarySearchPaths.push_back(resolveSearchPath(A->getValue())); } diff --git a/lib/Serialization/ModuleFileSharedCore.cpp b/lib/Serialization/ModuleFileSharedCore.cpp index ee2cfc72f88fe..b27169d201216 100644 --- a/lib/Serialization/ModuleFileSharedCore.cpp +++ b/lib/Serialization/ModuleFileSharedCore.cpp @@ -144,6 +144,9 @@ static bool readOptionsBlock(llvm::BitstreamCursor &cursor, case PluginSearchOptionKind::LoadPluginExecutable: optKind = PluginSearchOption::Kind::LoadPluginExecutable; break; + case PluginSearchOptionKind::LoadPlugin: + optKind = PluginSearchOption::Kind::LoadPlugin; + break; } extendedInfo.addPluginSearchOption({optKind, blobData}); break; diff --git a/lib/Serialization/ModuleFormat.h b/lib/Serialization/ModuleFormat.h index 5f613242fc787..386550b47d8ee 100644 --- a/lib/Serialization/ModuleFormat.h +++ b/lib/Serialization/ModuleFormat.h @@ -59,7 +59,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0; /// it just ensures a conflict if two people change the module format. /// Don't worry about adhering to the 80-column limit for this line. const uint16_t SWIFTMODULE_VERSION_MINOR = - 875; // Add package field to SerializedKind_t + 876; // Add PluginSearchOptionKind::LoadPlugin /// A standard hash seed used for all string hashes in a serialized module. /// @@ -688,6 +688,7 @@ enum class PluginSearchOptionKind : uint8_t { ExternalPluginPath, LoadPluginLibrary, LoadPluginExecutable, + LoadPlugin, }; using PluginSearchOptionKindField = BCFixed<3>; diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp index 19686214eaaa2..ba99cb99612c2 100644 --- a/lib/Serialization/Serialization.cpp +++ b/lib/Serialization/Serialization.cpp @@ -1213,6 +1213,17 @@ void Serializer::writeHeader() { uint8_t(PluginSearchOptionKind::LoadPluginExecutable), optStr); continue; } + case PluginSearchOption::Kind::LoadPlugin: { + auto &opt = elem.get(); + std::string optStr = opt.LibraryPath + ":" + opt.ServerPath + "#"; + llvm::interleave( + opt.ModuleNames, [&](auto &name) { optStr += name; }, + [&]() { optStr += ","; }); + PluginSearchOpt.emit( + ScratchRecord, + uint8_t(PluginSearchOptionKind::LoadPlugin), optStr); + continue; + } } } } diff --git a/tools/lldb-moduleimport-test/lldb-moduleimport-test.cpp b/tools/lldb-moduleimport-test/lldb-moduleimport-test.cpp index 31b5f99a87cda..de0d860ebc8a4 100644 --- a/tools/lldb-moduleimport-test/lldb-moduleimport-test.cpp +++ b/tools/lldb-moduleimport-test/lldb-moduleimport-test.cpp @@ -105,6 +105,9 @@ static bool validateModule( case swift::PluginSearchOption::Kind::LoadPluginExecutable: optStr = "-load-plugin-executable"; break; + case swift::PluginSearchOption::Kind::LoadPlugin: + optStr = "-load-plugin"; + break; } llvm::outs() << " " << optStr << " " << opt.second << "\n"; } diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index 3577b6f132b72..0796ab8f4c9f9 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -347,6 +347,11 @@ LoadPluginExecutable("load-plugin-executable", llvm::cl::desc("load plugin executable"), llvm::cl::cat(Category)); +static llvm::cl::list +LoadPlugin("load-plugin", + llvm::cl::desc("load plugin"), + llvm::cl::cat(Category)); + static llvm::cl::opt EnableSourceImport("enable-source-import", llvm::cl::Hidden, @@ -4563,6 +4568,24 @@ int main(int argc, char *argv[]) { std::move(moduleNames)}); } } + if (!options::LoadPlugin.empty()) { + for (auto arg: options::LoadPlugin) { + StringRef pathAndServer; + StringRef modulesStr; + std::tie(pathAndServer, modulesStr) = StringRef(arg).rsplit('#'); + StringRef path; + StringRef server; + std::tie(path, server) = pathAndServer.rsplit(':'); + std::vector moduleNames; + for (auto name : llvm::split(modulesStr, ',')) { + moduleNames.emplace_back(name); + } + InitInvok.getSearchPathOptions().PluginSearchOpts.emplace_back( + PluginSearchOption::LoadPlugin{std::string(path), + std::string(server), + std::move(moduleNames)}); + } + } for (auto path : options::PluginPath) { InitInvok.getSearchPathOptions().PluginSearchOpts.emplace_back( PluginSearchOption::PluginPath{path}); From f93df4d446efb16f87ef3b96093206ea6c9a2cf0 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 25 May 2024 22:27:53 -0400 Subject: [PATCH 056/105] Remove debug code --- lib/Frontend/CompilerInvocation.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 8e811f81fd794..d081bc322349e 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2067,9 +2067,6 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, break; } default: - llvm::errs() << "option: "; - A->getOption().print(llvm::errs()); - llvm::errs() << "\n"; llvm_unreachable("unhandled plugin search option"); } } From 95f254c1d0bb60a4af3a518cc2f9e32bbf22023a Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 26 May 2024 13:59:45 -0400 Subject: [PATCH 057/105] error_load_plugin diagnostic --- include/swift/AST/DiagnosticsFrontend.def | 3 +++ lib/Frontend/CompilerInvocation.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 01125333b40e1..27012d89fd33a 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -132,6 +132,9 @@ ERROR(error_no_source_location_scope_map,none, ERROR(error_load_plugin_executable,none, "invalid value '%0' in '-load-plugin-executable'; " "make sure to use format '#'", (StringRef)) +ERROR(error_load_plugin,none, + "invalid value '%0' in '-load-plugin'; " + "make sure to use format ':#'", (StringRef)) NOTE(note_valid_swift_versions, none, "valid arguments to '-swift-version' are %0", (StringRef)) diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index d081bc322349e..b6c74ad88c810 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2041,7 +2041,7 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, moduleNames.emplace_back(name); } if (path.empty() || server.empty() || moduleNames.empty()) { - Diags.diagnose(SourceLoc(), diag::error_load_plugin_executable, + Diags.diagnose(SourceLoc(), diag::error_load_plugin, A->getValue()); } else { Opts.PluginSearchOpts.emplace_back( From caa1a8ccd01eb568bc1b65007882bf8840e08ad6 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 26 May 2024 14:17:17 -0400 Subject: [PATCH 058/105] Elide copy --- lib/AST/PluginLoader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 3b3e00f448772..162016de1afee 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -124,8 +124,7 @@ PluginLoader::getPluginMap() { case PluginSearchOption::Kind::LoadPlugin: { auto &val = entry.get(); - std::vector moduleNames = val.ModuleNames; - for (auto &moduleName : moduleNames) { + for (auto &moduleName : val.ModuleNames) { try_emplace(moduleName, val.LibraryPath, val.ServerPath); } continue; From fa52ecdce5fbc1873b7a77eb2f9b9d52aa600d04 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 12 Jun 2024 16:46:24 +0530 Subject: [PATCH 059/105] Feedback --- .../swift-plugin-server/WasmEngine.swift | 6 +++--- .../swift-plugin-server/WasmKitEngine.swift | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 2f7d8a5174a98..a57064ca3b82f 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -14,7 +14,7 @@ import WASI import WasmTypes import SystemPackage -typealias WasmFunction = ([UInt32]) throws -> [UInt32] +typealias WasmFunction = () throws -> Void protocol WasmEngine { init(path: FilePath, imports: WASIBridgeToHost) throws @@ -53,7 +53,7 @@ struct WasmEnginePlugin: WasmPlugin { guard let start = try engine.function(named: "_start") else { throw WasmEngineError(message: "Wasm plugin does not have a '_start' entrypoint") } - _ = try start([]) + try start() } func handleMessage(_ json: [UInt8]) throws -> [UInt8] { @@ -62,7 +62,7 @@ struct WasmEnginePlugin: WasmPlugin { } try hostToPlugin.writeAll(json) - _ = try pumpFunction([]) + try pumpFunction() let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index c712a544d98e3..72d1340ee5bd6 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -18,20 +18,22 @@ import SystemPackage typealias DefaultWasmEngine = WasmKitEngine struct WasmKitEngine: WasmEngine { - private let module: Module - private let instance: ModuleInstance private let runtime: Runtime + private let functions: [String: (Function)] init(path: FilePath, imports: WASIBridgeToHost) throws { - module = try parseWasm(filePath: path) runtime = Runtime(hostModules: imports.hostModules) - instance = try runtime.instantiate(module: module) + + let module = try parseWasm(filePath: path) + let instance = try runtime.instantiate(module: module) + functions = instance.exports.compactMapValues { export in + guard case let .function(function) = export else { return nil } + return function + } } func function(named name: String) throws -> WasmFunction? { - guard case let .function(function) = instance.exportInstances.first(where: { $0.name == name })?.value else { - return nil - } - return { args in try function.invoke(args.map(Value.i32), runtime: runtime).map(\.i32) } + guard let (function) = functions[name] else { return nil } + return { _ = try function.invoke(runtime: runtime) } } } From f2f2ea2d6528a35ab37b5f5acba0261a4b1bb9d8 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 12 Jun 2024 16:47:17 +0530 Subject: [PATCH 060/105] nit --- .../Sources/swift-plugin-server/WasmKitEngine.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index 72d1340ee5bd6..b2caa56997152 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -19,7 +19,7 @@ typealias DefaultWasmEngine = WasmKitEngine struct WasmKitEngine: WasmEngine { private let runtime: Runtime - private let functions: [String: (Function)] + private let functions: [String: Function] init(path: FilePath, imports: WASIBridgeToHost) throws { runtime = Runtime(hostModules: imports.hostModules) @@ -33,7 +33,7 @@ struct WasmKitEngine: WasmEngine { } func function(named name: String) throws -> WasmFunction? { - guard let (function) = functions[name] else { return nil } + guard let function = functions[name] else { return nil } return { _ = try function.invoke(runtime: runtime) } } } From c0a75e1edf1acc662298df8f591129ea3fc0a55b Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 12 Jun 2024 17:01:17 +0530 Subject: [PATCH 061/105] record ServerPath --- lib/AST/ModuleDependencies.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 9c0271ff8ac73..173c5470f460e 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -668,7 +668,7 @@ void SwiftDependencyTracker::addCommonSearchPathDeps( case PluginSearchOption::Kind::LoadPlugin: { auto &val = entry.get(); FS->status(val.LibraryPath); - // TODO: what else do we need? + FS->status(val.ServerPath); break; } } From 011a0a56cb89e29caa143bfe3aa17c7d79c2f1f4 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 12 Jun 2024 20:02:17 +0530 Subject: [PATCH 062/105] Basic test --- test/Macros/Inputs/wasm_plugin.swift | 210 +++++++++++++++++++++++++++ test/Macros/macro_plugin_wasm.swift | 56 +++++++ 2 files changed, 266 insertions(+) create mode 100644 test/Macros/Inputs/wasm_plugin.swift create mode 100644 test/Macros/macro_plugin_wasm.swift diff --git a/test/Macros/Inputs/wasm_plugin.swift b/test/Macros/Inputs/wasm_plugin.swift new file mode 100644 index 0000000000000..484ee74b7f2bb --- /dev/null +++ b/test/Macros/Inputs/wasm_plugin.swift @@ -0,0 +1,210 @@ +import Foundation + +@main struct Plugin { + static func main() { + var spec = Spec(raw: mockPlugin) + readabilityHandler = { spec.handleNext() } + } +} + +struct Spec { + struct Item: Codable { + let expect: JSONValue + let response: JSONValue + } + + private let items: [Item] + private var handled: Set = [] + + init(raw: String) { + items = try! decoder.decode([Item].self, from: Data(raw.utf8)) + } + + mutating func handleNext() { + let actual: JSONValue = try! waitForNextMessage() + guard let index = items.firstIndex(where: { match(expect: $0.expect, val: actual) }) else { + fatalError("couldn't find matching item for request: \(actual)") + } + guard handled.insert(index).inserted else { + fatalError("request is already handled") + } + let response = substitute(value: items[index].response, req: actual) + try! outputMessage(response) + } + + private func match(expect: JSONValue, val: JSONValue) -> Bool { + switch (expect, val) { + case (.object(let lhs), .object(let rhs)): + return lhs.allSatisfy { key, value in + rhs[key].map { match(expect: value, val: $0) } ?? false + } + case (.array(let lhs), .array(let rhs)): + return zip(lhs, rhs).allSatisfy { match(expect: $0, val: $1) } + default: + return expect == val + } + } + + private func substitute(value: JSONValue, req: JSONValue) -> JSONValue { + var path: Substring + switch value { + case .object(let obj): + return .object(obj.mapValues { substitute(value: $0, req: req) }) + case .array(let arr): + return .array(arr.map { substitute(value: $0, req: req) }) + case .string(let str) where str.starts(with: "=req"): + path = str.dropFirst(4) + break + default: + return value + } + var value = req + while !path.isEmpty { + // '.' -> object key. + if path.starts(with: ".") { + guard case .object(let obj) = value else { + return .string("") + } + path = path.dropFirst() + let key = path.prefix(while: { $0.isNumber || $0.isLetter || $0 == "_" }) + path = path.dropFirst(key.count) + guard let next = obj[String(key)] else { + return .string("") + } + value = next + continue + } + // '[' + ']' -> array index. + if path.starts(with: "[") { + guard case .array(let arr) = value else { + return .string("") + } + path = path.dropFirst() + let index = path.prefix(while: { $0.isNumber }) + path = path.dropFirst(index.count) + guard path.starts(with: "]") else { + return .string("") + } + value = arr[idx] + continue + } + return .string("") + } + return value + } +} + +var readabilityHandler: (() -> Void)? + +@_expose(wasm, "swift_wasm_macro_v1_pump") +@_cdecl("swift_wasm_macro_v1_pump") +func wasmPump() { + guard let readabilityHandler else { fatalError("main not called") } + readabilityHandler() +} + +let decoder = JSONDecoder() +let encoder = JSONEncoder() + +private func outputMessage(_ message: TX) throws { + let encoded = try encoder.encode(message) + let count = encoded.count + var header = UInt64(count).littleEndian + withUnsafeBytes(of: &header) { _write(contentsOf: $0) } + + // Write the JSON payload. + encoded.withUnsafeBytes { _write(contentsOf: $0) } +} + +private func waitForNextMessage() throws -> RX { + // Read the header (a 64-bit length field in little endian byte order). + var header: UInt64 = 0 + withUnsafeMutableBytes(of: &header) { _read(into: $0) } + + // Read the JSON payload. + let count = Int(UInt64(littleEndian: header)) + let data = UnsafeMutableRawBufferPointer.allocate(byteCount: count, alignment: 1) + defer { data.deallocate() } + _read(into: data) + + // Decode and return the message. + return try decoder.decode(RX.self, from: Data(data)) +} + +private func _read(into buffer: UnsafeMutableRawBufferPointer) { + guard var ptr = buffer.baseAddress else { return } + let endPtr = ptr.advanced(by: buffer.count) + while ptr != endPtr { + switch read(fileno(stdin), ptr, numericCast(endPtr - ptr)) { + case -1: fatalError("Error: \(errno)") + case 0: fatalError("End of input") + case let n: ptr += Int(n) + } + } +} + +private func _write(contentsOf buffer: UnsafeRawBufferPointer) { + guard var ptr = buffer.baseAddress else { return } + let endPtr = ptr.advanced(by: buffer.count) + while ptr != endPtr { + switch write(fileno(stdout), ptr, numericCast(endPtr - ptr)) { + case -1: fatalError("Error: \(errno)") + case 0: fatalError("write returned 0") + case let n: ptr += Int(n) + } + } +} + +enum JSONValue: Hashable { + case object([String: JSONValue]) + case array([JSONValue]) + case string(String) + case number(Double) + case bool(Bool) + case null +} + +extension JSONValue: Codable { + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + + if let decoded = try? container.decode(String.self) { + self = .string(decoded) + } else if let decoded = try? container.decode(Double.self) { + self = .number(decoded) + } else if let decoded = try? container.decode(Bool.self) { + self = .bool(decoded) + } else if let decoded = try? container.decode([JSONValue].self) { + self = .array(decoded) + } else if let decoded = try? container.decode([String: JSONValue].self) { + self = .object(decoded) + } else if container.decodeNil() { + self = .null + } else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Could not decode JSONValue") + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + + switch self { + case .string(let string): + try container.encode(string) + case .bool(let bool): + try container.encode(bool) + case .number(let double): + try container.encode(double) + case .array(let array): + try container.encode(array) + case .object(let dictionary): + try container.encode(dictionary) + case .null: + try container.encodeNil() + } + } +} diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift new file mode 100644 index 0000000000000..32ebc2506e91a --- /dev/null +++ b/test/Macros/macro_plugin_wasm.swift @@ -0,0 +1,56 @@ +// REQUIRES: swift_swift_parser + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: /Library/Developer/Toolchains/swift-latest.xctoolchain/usr/bin/swiftc \ +// RUN: -swift-version 5 -static-stdlib -parse-as-library \ +// RUN: -o %t/Plugin.wasm \ +// RUN: -module-name MacroDefinition \ +// RUN: %t/MacroDefinition.swift \ +// RUN: %S/Inputs/wasm_plugin.swift \ +// RUN: -g -target wasm32-unknown-wasi \ +// RUN: -sdk ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/WASI.sdk \ +// RUN: -resource-dir ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static \ +// RUN: -Xcc --sysroot -Xcc ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/WASI.sdk \ +// RUN: -Xcc -resource-dir -Xcc ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static/clang \ +// RUN: -Xclang-linker -resource-dir -Xclang-linker ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static/clang + +// RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ +// RUN: -typecheck \ +// RUN: -swift-version 5 \ +// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -Rmacro-loading \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift \ +// RUN: > %t/macro-loading.txt 2>&1 + +// RUN: %FileCheck %s < %t/macro-loading.txt + +// CHECK: ->(plugin:[[#PID:]]) {"getCapability":{"capability":{"protocolVersion":[[#PROTOCOL_VERSION:]]}}} +// CHECK: <-(plugin:[[#PID]]) {"getCapabilityResult":{"capability":{"features":["load-plugin-library"],"protocolVersion":7}}} +// CHECK: ->(plugin:[[#PID]]) {"loadPluginLibrary":{"libraryPath":"{{.+}}/Plugin.wasm","moduleName":"MacroDefinition"}} +// CHECK: <-(plugin:[[#PID]]) {"loadPluginLibraryResult":{"diagnostics":[],"loaded":true}} +// CHECK: ->(plugin:[[#PID]]) {"expandFreestandingMacro":{"discriminator":"$s{{.+}}","lexicalContext":[{{.*}}],"macro":{"moduleName":"MacroDefinition","name":"constInt","typeName":"ConstMacro"},"macroRole":"expression","syntax":{"kind":"expression","location":{"column":16,"fileID":"MyApp/test.swift","fileName":"{{.+}}test.swift","line":4,"offset":143},"source":"#constInt"}}} +// CHECK: <-(plugin:[[#PID]]) {"expandMacroResult":{"diagnostics":[],"expandedSource":"1"}} + +//--- test.swift +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + let _: Int = #constInt +} + +//--- MacroDefinition.swift +let mockPlugin = #""" +[ + { + "expect": {"expandFreestandingMacro": { + "macro": {"moduleName": "MacroDefinition", "typeName": "ConstMacro"}, + "syntax": {"kind": "expression", "source": "#constInt"}}}, + "response": {"expandMacroResult": {"expandedSource": "1", "diagnostics": []}} + } +] +"""# From d2d3b949a5aa5ef09f2071ee1416bf89f789901b Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Wed, 12 Jun 2024 20:19:03 +0530 Subject: [PATCH 063/105] Parameterize --- test/Macros/macro_plugin_wasm.swift | 12 ++++++------ test/lit.cfg | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index 32ebc2506e91a..96bff69c71093 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -5,18 +5,18 @@ // RUN: split-file %s %t //#-- Prepare the Wasm macro plugin. -// RUN: /Library/Developer/Toolchains/swift-latest.xctoolchain/usr/bin/swiftc \ +// RUN: %swiftc_driver_plain \ // RUN: -swift-version 5 -static-stdlib -parse-as-library \ // RUN: -o %t/Plugin.wasm \ // RUN: -module-name MacroDefinition \ // RUN: %t/MacroDefinition.swift \ // RUN: %S/Inputs/wasm_plugin.swift \ // RUN: -g -target wasm32-unknown-wasi \ -// RUN: -sdk ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/WASI.sdk \ -// RUN: -resource-dir ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static \ -// RUN: -Xcc --sysroot -Xcc ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/WASI.sdk \ -// RUN: -Xcc -resource-dir -Xcc ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static/clang \ -// RUN: -Xclang-linker -resource-dir -Xclang-linker ~/Library/org.swift.swiftpm/swift-sdks/swift-wasm-DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi.artifactbundle/DEVELOPMENT-SNAPSHOT-2024-04-19-a-wasm32-unknown-wasi/wasm32-unknown-wasi/swift.xctoolchain/usr/lib/swift_static/clang +// RUN: -sdk %wasm-sdk-root/WASI.sdk \ +// RUN: -resource-dir %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static \ +// RUN: -Xcc --sysroot -Xcc %wasm-sdk-root/WASI.sdk \ +// RUN: -Xcc -resource-dir -Xcc %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static/clang \ +// RUN: -Xclang-linker -resource-dir -Xclang-linker %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static/clang // RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ // RUN: -typecheck \ diff --git a/test/lit.cfg b/test/lit.cfg index eaae9ec06721f..8fcd19bfcedfb 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -2970,3 +2970,7 @@ if kIsWindows and visual_studio_version: lit_config.note("Available features: " + ", ".join(sorted(config.available_features))) config.substitutions.append( ('%use_no_opaque_pointers', '-Xcc -Xclang -Xcc -no-opaque-pointers' ) ) + +wasm_sdk_root = lit_config.params.get('wasm_sdk_root', None) +if wasm_sdk_root is not None: + config.substitutions.append( ('%wasm-sdk-root', wasm_sdk_root) ) From b307eb4ae9546adc37908c86f3959b99b208dd13 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 29 Jun 2024 20:35:43 +0530 Subject: [PATCH 064/105] Fix deprecation warning --- .../Sources/SwiftInProcPluginServer/InProcPluginServer.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift index bf15260835e79..1a12efedd4819 100644 --- a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift +++ b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift @@ -67,11 +67,11 @@ public func handleMessage( /// Singleton "plugin server". struct InProcPluginServer { - private let handler: CompilerPluginMessageHandler + private let handler: PluginProviderMessageHandler @MainActor private init() { - self.handler = CompilerPluginMessageHandler( + self.handler = PluginProviderMessageHandler( provider: LibraryPluginProvider.shared ) } From 782956ac420c1d94a9bab48947351600759f57d6 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 30 Jun 2024 19:32:29 +0530 Subject: [PATCH 065/105] working test --- test/Macros/Inputs/wasi_shim.h | 60 ++++++++ test/Macros/Inputs/wasm_plugin.swift | 210 --------------------------- test/Macros/lit.local.cfg | 8 + test/Macros/macro_plugin_wasm.swift | 54 ++++--- 4 files changed, 99 insertions(+), 233 deletions(-) create mode 100644 test/Macros/Inputs/wasi_shim.h delete mode 100644 test/Macros/Inputs/wasm_plugin.swift diff --git a/test/Macros/Inputs/wasi_shim.h b/test/Macros/Inputs/wasi_shim.h new file mode 100644 index 0000000000000..83f03eb9d9e64 --- /dev/null +++ b/test/Macros/Inputs/wasi_shim.h @@ -0,0 +1,60 @@ +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +typedef uint8_t wasi_signal_t; +typedef uint16_t wasi_errno_t; +typedef uint32_t wasi_exitcode_t; +typedef int wasi_fd_t; +typedef __SIZE_TYPE__ wasi_size_t; + +typedef struct wasi_ciovec_t { + const uint8_t *buf; + wasi_size_t buf_len; +} wasi_ciovec_t; + +#define WASI_SIGNAL_ABRT (6) + +wasi_errno_t wasi_fd_write( + wasi_fd_t fd, + const wasi_ciovec_t *iovs, + wasi_size_t iovs_len, + wasi_size_t *nwritten +) __attribute__(( + __import_module__("wasi_snapshot_preview1"), + __import_name__("fd_write"), + __warn_unused_result__ +)); + +_Noreturn void wasi_proc_exit( + wasi_exitcode_t code +) __attribute__(( + __import_module__("wasi_snapshot_preview1"), + __import_name__("proc_exit") +)); + +wasi_errno_t wasi_proc_raise( + wasi_signal_t sig +) __attribute__(( + __import_module__("wasi_snapshot_preview1"), + __import_name__("proc_raise") +)); + +// libc shims + +static inline wasi_size_t swift_strlen(const char *buf) { + wasi_size_t len = 0; + while (buf[len]) len++; + return len; +} + +static inline wasi_errno_t swift_write(int fd, const void *buf, wasi_size_t len) { + struct wasi_ciovec_t vec = { .buf = (const uint8_t *)buf, .buf_len = len }; + wasi_size_t nwritten = 0; + return wasi_fd_write(fd, &vec, 1, &nwritten); +} + +static inline wasi_errno_t swift_puts(int fd, const char *buf) { + return swift_write(fd, buf, swift_strlen(buf)); +} diff --git a/test/Macros/Inputs/wasm_plugin.swift b/test/Macros/Inputs/wasm_plugin.swift deleted file mode 100644 index 484ee74b7f2bb..0000000000000 --- a/test/Macros/Inputs/wasm_plugin.swift +++ /dev/null @@ -1,210 +0,0 @@ -import Foundation - -@main struct Plugin { - static func main() { - var spec = Spec(raw: mockPlugin) - readabilityHandler = { spec.handleNext() } - } -} - -struct Spec { - struct Item: Codable { - let expect: JSONValue - let response: JSONValue - } - - private let items: [Item] - private var handled: Set = [] - - init(raw: String) { - items = try! decoder.decode([Item].self, from: Data(raw.utf8)) - } - - mutating func handleNext() { - let actual: JSONValue = try! waitForNextMessage() - guard let index = items.firstIndex(where: { match(expect: $0.expect, val: actual) }) else { - fatalError("couldn't find matching item for request: \(actual)") - } - guard handled.insert(index).inserted else { - fatalError("request is already handled") - } - let response = substitute(value: items[index].response, req: actual) - try! outputMessage(response) - } - - private func match(expect: JSONValue, val: JSONValue) -> Bool { - switch (expect, val) { - case (.object(let lhs), .object(let rhs)): - return lhs.allSatisfy { key, value in - rhs[key].map { match(expect: value, val: $0) } ?? false - } - case (.array(let lhs), .array(let rhs)): - return zip(lhs, rhs).allSatisfy { match(expect: $0, val: $1) } - default: - return expect == val - } - } - - private func substitute(value: JSONValue, req: JSONValue) -> JSONValue { - var path: Substring - switch value { - case .object(let obj): - return .object(obj.mapValues { substitute(value: $0, req: req) }) - case .array(let arr): - return .array(arr.map { substitute(value: $0, req: req) }) - case .string(let str) where str.starts(with: "=req"): - path = str.dropFirst(4) - break - default: - return value - } - var value = req - while !path.isEmpty { - // '.' -> object key. - if path.starts(with: ".") { - guard case .object(let obj) = value else { - return .string("") - } - path = path.dropFirst() - let key = path.prefix(while: { $0.isNumber || $0.isLetter || $0 == "_" }) - path = path.dropFirst(key.count) - guard let next = obj[String(key)] else { - return .string("") - } - value = next - continue - } - // '[' + ']' -> array index. - if path.starts(with: "[") { - guard case .array(let arr) = value else { - return .string("") - } - path = path.dropFirst() - let index = path.prefix(while: { $0.isNumber }) - path = path.dropFirst(index.count) - guard path.starts(with: "]") else { - return .string("") - } - value = arr[idx] - continue - } - return .string("") - } - return value - } -} - -var readabilityHandler: (() -> Void)? - -@_expose(wasm, "swift_wasm_macro_v1_pump") -@_cdecl("swift_wasm_macro_v1_pump") -func wasmPump() { - guard let readabilityHandler else { fatalError("main not called") } - readabilityHandler() -} - -let decoder = JSONDecoder() -let encoder = JSONEncoder() - -private func outputMessage(_ message: TX) throws { - let encoded = try encoder.encode(message) - let count = encoded.count - var header = UInt64(count).littleEndian - withUnsafeBytes(of: &header) { _write(contentsOf: $0) } - - // Write the JSON payload. - encoded.withUnsafeBytes { _write(contentsOf: $0) } -} - -private func waitForNextMessage() throws -> RX { - // Read the header (a 64-bit length field in little endian byte order). - var header: UInt64 = 0 - withUnsafeMutableBytes(of: &header) { _read(into: $0) } - - // Read the JSON payload. - let count = Int(UInt64(littleEndian: header)) - let data = UnsafeMutableRawBufferPointer.allocate(byteCount: count, alignment: 1) - defer { data.deallocate() } - _read(into: data) - - // Decode and return the message. - return try decoder.decode(RX.self, from: Data(data)) -} - -private func _read(into buffer: UnsafeMutableRawBufferPointer) { - guard var ptr = buffer.baseAddress else { return } - let endPtr = ptr.advanced(by: buffer.count) - while ptr != endPtr { - switch read(fileno(stdin), ptr, numericCast(endPtr - ptr)) { - case -1: fatalError("Error: \(errno)") - case 0: fatalError("End of input") - case let n: ptr += Int(n) - } - } -} - -private func _write(contentsOf buffer: UnsafeRawBufferPointer) { - guard var ptr = buffer.baseAddress else { return } - let endPtr = ptr.advanced(by: buffer.count) - while ptr != endPtr { - switch write(fileno(stdout), ptr, numericCast(endPtr - ptr)) { - case -1: fatalError("Error: \(errno)") - case 0: fatalError("write returned 0") - case let n: ptr += Int(n) - } - } -} - -enum JSONValue: Hashable { - case object([String: JSONValue]) - case array([JSONValue]) - case string(String) - case number(Double) - case bool(Bool) - case null -} - -extension JSONValue: Codable { - public init(from decoder: Decoder) throws { - let container = try decoder.singleValueContainer() - - if let decoded = try? container.decode(String.self) { - self = .string(decoded) - } else if let decoded = try? container.decode(Double.self) { - self = .number(decoded) - } else if let decoded = try? container.decode(Bool.self) { - self = .bool(decoded) - } else if let decoded = try? container.decode([JSONValue].self) { - self = .array(decoded) - } else if let decoded = try? container.decode([String: JSONValue].self) { - self = .object(decoded) - } else if container.decodeNil() { - self = .null - } else { - throw DecodingError.dataCorruptedError(in: container, debugDescription: "Could not decode JSONValue") - } - } - - public func encode(to encoder: Encoder) throws { - var container = encoder.singleValueContainer() - - switch self { - case .string(let string): - try container.encode(string) - case .bool(let bool): - try container.encode(bool) - case .number(let double): - try container.encode(double) - case .array(let array): - try container.encode(array) - case .object(let dictionary): - try container.encode(dictionary) - case .null: - try container.encodeNil() - } - } -} diff --git a/test/Macros/lit.local.cfg b/test/Macros/lit.local.cfg index fe1101a502530..22528cb47d7a6 100644 --- a/test/Macros/lit.local.cfg +++ b/test/Macros/lit.local.cfg @@ -31,3 +31,11 @@ else: ) config.substitutions.append(('%c-flags', config.c_flags)) config.substitutions.append(('%exe-linker-flags', config.exe_linker_flags)) + +config.substitutions.insert( + 0, + ( + '%swift-build-wasm-c-plugin', + f'{config.clang} %c-flags %exe-linker-flags -target wasm32 -nostdlib -iquote %S' + ) +) diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index 96bff69c71093..9cf1b297e0c18 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -5,18 +5,7 @@ // RUN: split-file %s %t //#-- Prepare the Wasm macro plugin. -// RUN: %swiftc_driver_plain \ -// RUN: -swift-version 5 -static-stdlib -parse-as-library \ -// RUN: -o %t/Plugin.wasm \ -// RUN: -module-name MacroDefinition \ -// RUN: %t/MacroDefinition.swift \ -// RUN: %S/Inputs/wasm_plugin.swift \ -// RUN: -g -target wasm32-unknown-wasi \ -// RUN: -sdk %wasm-sdk-root/WASI.sdk \ -// RUN: -resource-dir %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static \ -// RUN: -Xcc --sysroot -Xcc %wasm-sdk-root/WASI.sdk \ -// RUN: -Xcc -resource-dir -Xcc %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static/clang \ -// RUN: -Xclang-linker -resource-dir -Xclang-linker %wasm-sdk-root/swift.xctoolchain/usr/lib/swift_static/clang +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm // RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ // RUN: -typecheck \ @@ -43,14 +32,33 @@ func foo() { let _: Int = #constInt } -//--- MacroDefinition.swift -let mockPlugin = #""" -[ - { - "expect": {"expandFreestandingMacro": { - "macro": {"moduleName": "MacroDefinition", "typeName": "ConstMacro"}, - "syntax": {"kind": "expression", "source": "#constInt"}}}, - "response": {"expandMacroResult": {"expandedSource": "1", "diagnostics": []}} - } -] -"""# +//--- MacroDefinition.c +#include "Inputs/wasi_shim.h" + +_Noreturn static void swift_abort(const char *message) { + swift_puts(2, message); + wasi_proc_exit(1); +} + +static void write_json(const char *json) { + wasi_size_t len = swift_strlen(json); + uint64_t len64 = (uint64_t)len; + swift_write(1, &len64, sizeof(len64)); + swift_write(1, json, len); +} + +int was_start_called = 0; +int pump_calls = 0; + +__attribute__((export_name("_start"))) +void _start(void) { + if (was_start_called) swift_abort("_start called twice!"); + was_start_called = 1; +} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) { + if (!was_start_called) swift_abort("_start not called!"); + if (pump_calls++ != 0) swift_abort("expected pump to be called once"); + write_json("{\"expandMacroResult\": {\"expandedSource\": \"1\", \"diagnostics\": []}}"); +} From 8a88dc58b94cb83199b31f8ce4e338ef1b767bd4 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 30 Jun 2024 19:41:50 +0530 Subject: [PATCH 066/105] simplify --- test/Macros/Inputs/wasi_shim.h | 17 ++++++----------- test/Macros/macro_plugin_wasm.swift | 5 ----- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/test/Macros/Inputs/wasi_shim.h b/test/Macros/Inputs/wasi_shim.h index 83f03eb9d9e64..cc7ea544bf643 100644 --- a/test/Macros/Inputs/wasi_shim.h +++ b/test/Macros/Inputs/wasi_shim.h @@ -1,3 +1,6 @@ +// definitions from +// https://github.com/WebAssembly/wasi-libc/blob/320bbbcced68ce8e564b0dc4c8f80a5a5ad21a9c/libc-bottom-half/headers/public/wasi/api.h + typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; @@ -14,8 +17,6 @@ typedef struct wasi_ciovec_t { wasi_size_t buf_len; } wasi_ciovec_t; -#define WASI_SIGNAL_ABRT (6) - wasi_errno_t wasi_fd_write( wasi_fd_t fd, const wasi_ciovec_t *iovs, @@ -34,13 +35,6 @@ _Noreturn void wasi_proc_exit( __import_name__("proc_exit") )); -wasi_errno_t wasi_proc_raise( - wasi_signal_t sig -) __attribute__(( - __import_module__("wasi_snapshot_preview1"), - __import_name__("proc_raise") -)); - // libc shims static inline wasi_size_t swift_strlen(const char *buf) { @@ -55,6 +49,7 @@ static inline wasi_errno_t swift_write(int fd, const void *buf, wasi_size_t len) return wasi_fd_write(fd, &vec, 1, &nwritten); } -static inline wasi_errno_t swift_puts(int fd, const char *buf) { - return swift_write(fd, buf, swift_strlen(buf)); +_Noreturn static inline void swift_abort(const char *message) { + swift_write(2, message, swift_strlen(message)); + wasi_proc_exit(1); } diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index 9cf1b297e0c18..d01e9487a4e5c 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -35,11 +35,6 @@ func foo() { //--- MacroDefinition.c #include "Inputs/wasi_shim.h" -_Noreturn static void swift_abort(const char *message) { - swift_puts(2, message); - wasi_proc_exit(1); -} - static void write_json(const char *json) { wasi_size_t len = swift_strlen(json); uint64_t len64 = (uint64_t)len; From fcb838d932adf2d6683d9769af06aa71a31435cd Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 30 Jun 2024 19:44:45 +0530 Subject: [PATCH 067/105] remove wasm_sdk_root --- test/lit.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/lit.cfg b/test/lit.cfg index 3779c67006669..09c8974c6c03f 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -2957,7 +2957,3 @@ if kIsWindows and visual_studio_version: lit_config.note("Available features: " + ", ".join(sorted(config.available_features))) config.substitutions.append( ('%use_no_opaque_pointers', '-Xcc -Xclang -Xcc -no-opaque-pointers' ) ) - -wasm_sdk_root = lit_config.params.get('wasm_sdk_root', None) -if wasm_sdk_root is not None: - config.substitutions.append( ('%wasm-sdk-root', wasm_sdk_root) ) From 469d84a688b425ea6ee1181a8892d9d621beee86 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 30 Jun 2024 20:27:44 +0530 Subject: [PATCH 068/105] error tests --- .../Macros/macro_plugin_wasm_badversion.swift | 32 +++++++++++++++ .../macro_plugin_wasm_guest_error.swift | 39 +++++++++++++++++++ test/Macros/macro_plugin_wasm_nostart.swift | 32 +++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 test/Macros/macro_plugin_wasm_badversion.swift create mode 100644 test/Macros/macro_plugin_wasm_guest_error.swift create mode 100644 test/Macros/macro_plugin_wasm_nostart.swift diff --git a/test/Macros/macro_plugin_wasm_badversion.swift b/test/Macros/macro_plugin_wasm_badversion.swift new file mode 100644 index 0000000000000..ed061a491c219 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_badversion.swift @@ -0,0 +1,32 @@ +// REQUIRES: swift_swift_parser + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift + +//--- test.swift +// expected-warning @+2 {{Wasm plugin has an unknown ABI (could not find 'swift_wasm_macro_v1_pump')}} +// expected-note @+1 {{declared here}} +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{Wasm plugin has an unknown ABI (could not find 'swift_wasm_macro_v1_pump')}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +__attribute__((export_name("_start"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v100_pump"))) +void pump(void) {} diff --git a/test/Macros/macro_plugin_wasm_guest_error.swift b/test/Macros/macro_plugin_wasm_guest_error.swift new file mode 100644 index 0000000000000..5bbdc66787d52 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_guest_error.swift @@ -0,0 +1,39 @@ +// REQUIRES: swift_swift_parser + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift \ +// RUN: 2>%t/macro-loading.txt + +// RUN: %FileCheck %s < %t/macro-loading.txt + +// CHECK: guest error! + +//--- test.swift +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{failed to communicate with external macro}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +#include "Inputs/wasi_shim.h" + +__attribute__((export_name("_start"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) { + swift_abort("guest error!\n"); +} diff --git a/test/Macros/macro_plugin_wasm_nostart.swift b/test/Macros/macro_plugin_wasm_nostart.swift new file mode 100644 index 0000000000000..1ae8b5123ea17 --- /dev/null +++ b/test/Macros/macro_plugin_wasm_nostart.swift @@ -0,0 +1,32 @@ +// REQUIRES: swift_swift_parser + +// RUN: %empty-directory(%t) + +// RUN: split-file %s %t + +//#-- Prepare the Wasm macro plugin. +// RUN: %swift-build-wasm-c-plugin %t/MacroDefinition.c -o %t/Plugin.wasm + +// RUN: %target-swift-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -module-name MyApp \ +// RUN: %t/test.swift + +//--- test.swift +// expected-warning @+2 {{Wasm plugin does not have a '_start' entrypoint}} +// expected-note @+1 {{declared here}} +@freestanding(expression) macro constInt() -> Int = #externalMacro(module: "MacroDefinition", type: "ConstMacro") + +func foo() { + // expected-error @+1 {{Wasm plugin does not have a '_start' entrypoint}} + let _: Int = #constInt +} + +//--- MacroDefinition.c +__attribute__((export_name("_tart"))) +void _start(void) {} + +__attribute__((export_name("swift_wasm_macro_v1_pump"))) +void pump(void) {} From cc41cf13488c1b57be8b854810023f7d4a2acbb4 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 1 Jul 2024 00:14:55 +0000 Subject: [PATCH 069/105] Update WasmKit to 0.0.5 --- utils/update_checkout/update-checkout-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index a937efb72f7ad..5b3f95ebbb72e 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -159,7 +159,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.3", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -306,7 +306,7 @@ "swift-nio-ssl": "2.15.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.3", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -682,7 +682,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.3", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", From e89fd6a85d30d57be9bae577e611b5cead938020 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 14 Jul 2024 17:31:21 +0530 Subject: [PATCH 070/105] post-merge fixup --- lib/Frontend/ModuleInterfaceLoader.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 9e6143fb42b41..8bc8fe9c0b39f 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1715,6 +1715,15 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( } break; } + case PluginSearchOption::Kind::LoadPlugin: { + auto &val = entry.get(); + for (auto &moduleName : val.ModuleNames) { + GenericArgs.push_back("-load-plugin"); + GenericArgs.push_back( + ArgSaver.save(val.LibraryPath + ":" + val.ServerPath + "#" + moduleName)); + } + break; + } case PluginSearchOption::Kind::PluginPath: { auto &val = entry.get(); GenericArgs.push_back("-plugin-path"); From 2b477241dd90b65342ad8794d17d5472daa09388 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 14 Jul 2024 19:42:20 +0530 Subject: [PATCH 071/105] no opaque types --- .../Sources/swift-plugin-server/WasmMessageHandler.swift | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index 1ec17d9638cc5..f6cd57cf8c6dc 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -30,7 +30,7 @@ final class WasmInterceptingMessageHandler: PluginMe guard libraryPath.hasSuffix(".wasm") else { break } let libraryFilePath = FilePath(libraryPath) do { - loadedWasmPlugins[moduleName] = try defaultWasmPlugin.init(path: libraryFilePath) + loadedWasmPlugins[moduleName] = try DefaultWasmPlugin(path: libraryFilePath) } catch { return .loadPluginLibraryResult( loaded: false, @@ -110,5 +110,3 @@ protocol WasmPlugin { func handleMessage(_ json: [UInt8]) throws -> [UInt8] } - -private var defaultWasmPlugin: (some WasmPlugin).Type { DefaultWasmPlugin.self } From 4285cf1a5188a4b3cb7cc6c0b1cd7a3aeb2bdc10 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sun, 14 Jul 2024 20:07:08 +0530 Subject: [PATCH 072/105] tentative fix for lldb build --- tools/swift-plugin-server/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 791597e1644d9..a0e88108ebd00 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,9 +1,11 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) # override the remote SwiftSystem and ArgParser dependencies in WasmKit - FetchContent_Declare(SwiftSystem SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-system") - FetchContent_Declare(ArgumentParser SOURCE_DIR "${PROJECT_SOURCE_DIR}/../swift-argument-parser") + # TODO: create new variables instead of piggybacking off of SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE + file(TO_CMAKE_PATH "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}" swift_syntax_path) + FetchContent_Declare(SwiftSystem SOURCE_DIR "${swift_syntax_path}/../swift-system") + FetchContent_Declare(ArgumentParser SOURCE_DIR "${swift_syntax_path}/../swift-argument-parser") - FetchContent_Declare(WasmKit SOURCE_DIR "${PROJECT_SOURCE_DIR}/../wasmkit") + FetchContent_Declare(WasmKit SOURCE_DIR "${swift_syntax_path}/../wasmkit") FetchContent_MakeAvailable(WasmKit) add_pure_swift_host_tool(swift-plugin-server From 56b6cae5c959c0946a2a5e97453a524a9a8513d4 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Jul 2024 00:48:23 +0530 Subject: [PATCH 073/105] =?UTF-8?q?don=E2=80=99t=20depend=20on=20ArgumentP?= =?UTF-8?q?arser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/swift-plugin-server/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index a0e88108ebd00..16cd34a890f62 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,10 +1,10 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) - # override the remote SwiftSystem and ArgParser dependencies in WasmKit + # override the remote SwiftSystem dependency in WasmKit # TODO: create new variables instead of piggybacking off of SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE file(TO_CMAKE_PATH "${SWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE}" swift_syntax_path) FetchContent_Declare(SwiftSystem SOURCE_DIR "${swift_syntax_path}/../swift-system") - FetchContent_Declare(ArgumentParser SOURCE_DIR "${swift_syntax_path}/../swift-argument-parser") + set(WASMKIT_BUILD_CLI OFF) FetchContent_Declare(WasmKit SOURCE_DIR "${swift_syntax_path}/../wasmkit") FetchContent_MakeAvailable(WasmKit) From 96363e12646102e3c15e61d5a348b9705e0ddae8 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Mon, 15 Jul 2024 01:09:28 +0530 Subject: [PATCH 074/105] temporarily update wasmkit checkout --- utils/update_checkout/update-checkout-config.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index e8ba30461207c..07b1e783379ce 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -98,7 +98,7 @@ "wasi-libc": { "remote": { "id": "WebAssembly/wasi-libc" } }, "wasmkit": { - "remote": { "id": "swiftwasm/WasmKit" } + "remote": { "id": "kabiroberai/WasmKit" } }, "curl": { "remote": { "id": "curl/curl" } @@ -162,7 +162,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "develop", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -310,7 +310,7 @@ "swift-nio-ssl": "2.15.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "develop", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -686,7 +686,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "develop", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", From 913041b2d38d3d0ca13216a7d8a6b6270dff5b2c Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jul 2024 04:02:10 +0000 Subject: [PATCH 075/105] Revert "temporarily update wasmkit checkout" This reverts commit 96363e12646102e3c15e61d5a348b9705e0ddae8. --- utils/update_checkout/update-checkout-config.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 07b1e783379ce..e8ba30461207c 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -98,7 +98,7 @@ "wasi-libc": { "remote": { "id": "WebAssembly/wasi-libc" } }, "wasmkit": { - "remote": { "id": "kabiroberai/WasmKit" } + "remote": { "id": "swiftwasm/WasmKit" } }, "curl": { "remote": { "id": "curl/curl" } @@ -162,7 +162,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "develop", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -310,7 +310,7 @@ "swift-nio-ssl": "2.15.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "develop", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -686,7 +686,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "develop", + "wasmkit": "0.0.5", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", From 6cd389e093048dfeef22706b5076612165092833 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jul 2024 04:03:27 +0000 Subject: [PATCH 076/105] Update WasmKit to 0.0.6 To include a CMake fix https://github.com/swiftwasm/WasmKit/pull/101 --- utils/update_checkout/update-checkout-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index e8ba30461207c..0f32dd1657701 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -162,7 +162,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "0.0.6", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -310,7 +310,7 @@ "swift-nio-ssl": "2.15.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "0.0.6", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -686,7 +686,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.5", + "wasmkit": "0.0.6", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", From c3f4276482904b5dcc785725c32ebf382b55fcb5 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 15 Jul 2024 07:58:33 +0000 Subject: [PATCH 077/105] [test] Annotate macro tests that require the WebAssembly code generator --- test/Macros/macro_plugin_wasm.swift | 1 + test/Macros/macro_plugin_wasm_badversion.swift | 1 + test/Macros/macro_plugin_wasm_guest_error.swift | 1 + test/Macros/macro_plugin_wasm_nostart.swift | 1 + 4 files changed, 4 insertions(+) diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index d01e9487a4e5c..3763d1be86ee3 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -1,4 +1,5 @@ // REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly // RUN: %empty-directory(%t) diff --git a/test/Macros/macro_plugin_wasm_badversion.swift b/test/Macros/macro_plugin_wasm_badversion.swift index ed061a491c219..d82bd47e5310c 100644 --- a/test/Macros/macro_plugin_wasm_badversion.swift +++ b/test/Macros/macro_plugin_wasm_badversion.swift @@ -1,4 +1,5 @@ // REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly // RUN: %empty-directory(%t) diff --git a/test/Macros/macro_plugin_wasm_guest_error.swift b/test/Macros/macro_plugin_wasm_guest_error.swift index 5bbdc66787d52..7b2d0e042372a 100644 --- a/test/Macros/macro_plugin_wasm_guest_error.swift +++ b/test/Macros/macro_plugin_wasm_guest_error.swift @@ -1,4 +1,5 @@ // REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly // RUN: %empty-directory(%t) diff --git a/test/Macros/macro_plugin_wasm_nostart.swift b/test/Macros/macro_plugin_wasm_nostart.swift index 1ae8b5123ea17..d692802eae3ec 100644 --- a/test/Macros/macro_plugin_wasm_nostart.swift +++ b/test/Macros/macro_plugin_wasm_nostart.swift @@ -1,4 +1,5 @@ // REQUIRES: swift_swift_parser +// REQUIRES: CODEGENERATOR=WebAssembly // RUN: %empty-directory(%t) From 4341058da46b5e865f04d7ff44b548427d9ac8c7 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 21 Jul 2024 23:28:09 +0000 Subject: [PATCH 078/105] Update WasmKit to 0.0.7 To include Windows support of the WASI implementation. --- utils/update_checkout/update-checkout-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 0f32dd1657701..c6825ac0ed054 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -162,7 +162,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.6", + "wasmkit": "0.0.7", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -310,7 +310,7 @@ "swift-nio-ssl": "2.15.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.6", + "wasmkit": "0.0.7", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", @@ -686,7 +686,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-20", - "wasmkit": "0.0.6", + "wasmkit": "0.0.7", "curl": "curl-8_5_0", "icu": "maint/maint-69", "libxml2": "v2.11.5", From 393933a492a93817e1b3bccb963ea0649e236d5f Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 22 Jul 2024 08:25:49 +0000 Subject: [PATCH 079/105] Add temporary workaround for Windows pipe support of swift-system The support for `pipe` on Windows was added by https://github.com/apple/swift-system/commit/cfd3f9da906cf62efafd162d524df6334cc85da6 but it is not included in the latest swift-system release 1.3.1. This commit adds a temporary inlined implementation of `pipe` API for Windows until we update to the next release of swift-system. --- .../swift-plugin-server/WasmEngine.swift | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index a57064ca3b82f..310e9be7adb07 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -89,3 +89,30 @@ struct WasmEngineError: Error, CustomStringConvertible { self.description = message } } + +// `pipe` support on Windows is not included in the latest swift-system release 1.3.1 +// but it is available in the latest main branch. Remove the following code when we +// update to the next release of swift-system. +#if os(Windows) +import ucrt + +internal var system_errno: CInt { + var value: CInt = 0 + _ = ucrt._get_errno(&value) + return value +} + +extension FileDescriptor { + static func pipe() throws -> (readEnd: FileDescriptor, writeEnd: FileDescriptor) { + var fds: (Int32, Int32) = (-1, -1) + return try withUnsafeMutablePointer(to: &fds) { pointer in + return try pointer.withMemoryRebound(to: Int32.self, capacity: 2) { fds in + guard _pipe(fds, 4096, _O_BINARY | _O_NOINHERIT) == 0 else { + throw Errno(rawValue: system_errno) + } + return (FileDescriptor(rawValue: fds[0]), FileDescriptor(rawValue: fds[1])) + } + } + } +} +#endif From 9c5e7fcce052fb4366606b9694df2a114a36072f Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 22 Jul 2024 11:59:23 +0000 Subject: [PATCH 080/105] [test] Remove usage of unused lit variables for building wasm plugins `%c-flags` and `%exe-linker-flags` are derived from `CMAKE_C_FLAGS` and `CMAKE_EXE_LINKER_FLAGS` respectively, which are not configured for Wasm targets. And also those variables are not set in Windows build. So, this patch removes the usage of those variables. --- test/Macros/lit.local.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Macros/lit.local.cfg b/test/Macros/lit.local.cfg index 22528cb47d7a6..53d28816c7ded 100644 --- a/test/Macros/lit.local.cfg +++ b/test/Macros/lit.local.cfg @@ -36,6 +36,6 @@ config.substitutions.insert( 0, ( '%swift-build-wasm-c-plugin', - f'{config.clang} %c-flags %exe-linker-flags -target wasm32 -nostdlib -iquote %S' + f'{config.clang} -target wasm32 -nostdlib -iquote %S' ) ) From 5890c3e3f229742c563ce782cdc443a5b203ad81 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 23 Jul 2024 19:47:58 +0900 Subject: [PATCH 081/105] Change the first `-load-plugin` path separator from `:` to `#` The ':' character can cause conflicts in Windows paths (e.g., C:\path\to\file). The '#' character is less likely to appear in file paths and we already use it for the second path separator in `-load-plugin` option --- include/swift/AST/DiagnosticsFrontend.def | 2 +- include/swift/Option/Options.td | 2 +- lib/Frontend/CompilerInvocation.cpp | 4 ++-- lib/Frontend/ModuleInterfaceLoader.cpp | 2 +- lib/Serialization/Serialization.cpp | 2 +- test/Macros/macro_plugin_wasm.swift | 2 +- test/Macros/macro_plugin_wasm_badversion.swift | 2 +- test/Macros/macro_plugin_wasm_guest_error.swift | 2 +- test/Macros/macro_plugin_wasm_nostart.swift | 2 +- tools/swift-ide-test/swift-ide-test.cpp | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 23b1140ad4949..d5b2ce2c519ab 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -134,7 +134,7 @@ ERROR(error_load_plugin_executable,none, "make sure to use format '#'", (StringRef)) ERROR(error_load_plugin,none, "invalid value '%0' in '-load-plugin'; " - "make sure to use format ':#'", (StringRef)) + "make sure to use format '##'", (StringRef)) NOTE(note_valid_swift_versions, none, "valid arguments to '-swift-version' are %0", (StringRef)) diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 75452192b8721..4f3bf9b5168d1 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -2037,7 +2037,7 @@ def load_plugin: Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>, HelpText<"Path to a plugin library, a server to load it in, and a comma-separated " "list of module names where the macro types are declared">, - MetaVarName<":#">; + MetaVarName<"##">; def disable_sandbox: Flag<["-"], "disable-sandbox">, diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 2d9a3692c8e05..d2ef2bd799117 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2073,14 +2073,14 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, break; } case OPT_load_plugin: { - // ':#' where the module names are + // '##' where the module names are // comma separated. StringRef pathAndServer; StringRef modulesStr; std::tie(pathAndServer, modulesStr) = StringRef(A->getValue()).rsplit('#'); StringRef path; StringRef server; - std::tie(path, server) = pathAndServer.rsplit(':'); + std::tie(path, server) = pathAndServer.rsplit('#'); std::vector moduleNames; for (auto name : llvm::split(modulesStr, ',')) { moduleNames.emplace_back(name); diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 8bc8fe9c0b39f..f3d7aaf52ecac 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1720,7 +1720,7 @@ void InterfaceSubContextDelegateImpl::inheritOptionsForBuildingInterface( for (auto &moduleName : val.ModuleNames) { GenericArgs.push_back("-load-plugin"); GenericArgs.push_back( - ArgSaver.save(val.LibraryPath + ":" + val.ServerPath + "#" + moduleName)); + ArgSaver.save(val.LibraryPath + "#" + val.ServerPath + "#" + moduleName)); } break; } diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp index d8f3e16c8342e..7c3391c05444f 100644 --- a/lib/Serialization/Serialization.cpp +++ b/lib/Serialization/Serialization.cpp @@ -1216,7 +1216,7 @@ void Serializer::writeHeader() { } case PluginSearchOption::Kind::LoadPlugin: { auto &opt = elem.get(); - std::string optStr = opt.LibraryPath + ":" + opt.ServerPath + "#"; + std::string optStr = opt.LibraryPath + "#" + opt.ServerPath + "#"; llvm::interleave( opt.ModuleNames, [&](auto &name) { optStr += name; }, [&]() { optStr += ","; }); diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index 3763d1be86ee3..e047596070e8f 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -11,7 +11,7 @@ // RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ // RUN: -typecheck \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -Rmacro-loading \ // RUN: -module-name MyApp \ // RUN: %t/test.swift \ diff --git a/test/Macros/macro_plugin_wasm_badversion.swift b/test/Macros/macro_plugin_wasm_badversion.swift index d82bd47e5310c..5f72fe1f4c6dc 100644 --- a/test/Macros/macro_plugin_wasm_badversion.swift +++ b/test/Macros/macro_plugin_wasm_badversion.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift diff --git a/test/Macros/macro_plugin_wasm_guest_error.swift b/test/Macros/macro_plugin_wasm_guest_error.swift index 7b2d0e042372a..984a8b426def7 100644 --- a/test/Macros/macro_plugin_wasm_guest_error.swift +++ b/test/Macros/macro_plugin_wasm_guest_error.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift \ // RUN: 2>%t/macro-loading.txt diff --git a/test/Macros/macro_plugin_wasm_nostart.swift b/test/Macros/macro_plugin_wasm_nostart.swift index d692802eae3ec..c1789cacf8dd9 100644 --- a/test/Macros/macro_plugin_wasm_nostart.swift +++ b/test/Macros/macro_plugin_wasm_nostart.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm:%swift-plugin-server#MacroDefinition \ +// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index 48f4ab58df7e4..406c5e9b7381e 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -4583,7 +4583,7 @@ int main(int argc, char *argv[]) { std::tie(pathAndServer, modulesStr) = StringRef(arg).rsplit('#'); StringRef path; StringRef server; - std::tie(path, server) = pathAndServer.rsplit(':'); + std::tie(path, server) = pathAndServer.rsplit('#'); std::vector moduleNames; for (auto name : llvm::split(modulesStr, ',')) { moduleNames.emplace_back(name); From 0f39ec840467dcd2f798d2481064048569810b36 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 23 Jul 2024 20:05:40 +0900 Subject: [PATCH 082/105] [test] Fix path segment separator of plugin path for Windows --- test/Macros/macro_plugin_wasm.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index e047596070e8f..f93e8ff2f00ea 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -21,7 +21,7 @@ // CHECK: ->(plugin:[[#PID:]]) {"getCapability":{"capability":{"protocolVersion":[[#PROTOCOL_VERSION:]]}}} // CHECK: <-(plugin:[[#PID]]) {"getCapabilityResult":{"capability":{"features":["load-plugin-library"],"protocolVersion":7}}} -// CHECK: ->(plugin:[[#PID]]) {"loadPluginLibrary":{"libraryPath":"{{.+}}/Plugin.wasm","moduleName":"MacroDefinition"}} +// CHECK: ->(plugin:[[#PID]]) {"loadPluginLibrary":{"libraryPath":"{{.*[\\/]}}Plugin.wasm","moduleName":"MacroDefinition"}} // CHECK: <-(plugin:[[#PID]]) {"loadPluginLibraryResult":{"diagnostics":[],"loaded":true}} // CHECK: ->(plugin:[[#PID]]) {"expandFreestandingMacro":{"discriminator":"$s{{.+}}","lexicalContext":[{{.*}}],"macro":{"moduleName":"MacroDefinition","name":"constInt","typeName":"ConstMacro"},"macroRole":"expression","syntax":{"kind":"expression","location":{"column":16,"fileID":"MyApp/test.swift","fileName":"{{.+}}test.swift","line":4,"offset":143},"source":"#constInt"}}} // CHECK: <-(plugin:[[#PID]]) {"expandMacroResult":{"diagnostics":[],"expandedSource":"1"}} From 6e56934aaf7dd30e763272e3bc9766d75379c548 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 27 Aug 2024 12:07:36 +0100 Subject: [PATCH 083/105] Attempt to use `main` branch of WasmKit --- utils/update_checkout/update-checkout-config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index d657d40030d95..615c82547ed67 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -13,7 +13,7 @@ "swift-async-algorithms": { "remote": { "id": "apple/swift-async-algorithms" } }, "swift-atomics": { - "remote": { "id": "apple/swift-atomics" } }, + "remote": { "id": "apple/swift-atomics" } }, "swift-collections": { "remote": { "id": "apple/swift-collections" } }, "swift-crypto": { @@ -27,7 +27,7 @@ "swift-log": { "remote": { "id": "apple/swift-log" } }, "swift-numerics": { - "remote": { "id": "apple/swift-numerics" } }, + "remote": { "id": "apple/swift-numerics" } }, "swift-toolchain-sqlite": { "remote": { "id": "swiftlang/swift-toolchain-sqlite" } }, "swift-tools-support-core": { @@ -45,9 +45,9 @@ "swift-corelibs-xctest": { "remote": { "id": "apple/swift-corelibs-xctest" } }, "swift-corelibs-foundation": { - "remote": { "id": "apple/swift-corelibs-foundation" } }, + "remote": { "id": "apple/swift-corelibs-foundation" } }, "swift-foundation-icu": { - "remote": { "id": "apple/swift-foundation-icu" } }, + "remote": { "id": "apple/swift-foundation-icu" } }, "swift-foundation": { "remote": { "id": "apple/swift-foundation" } }, "swift-corelibs-libdispatch": { @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.0.8", + "wasmkit": "main", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" From 577055348fbfa31d27218b5f722fae3d8867d933 Mon Sep 17 00:00:00 2001 From: Kabir Oberai Date: Sat, 21 Sep 2024 17:20:46 -0400 Subject: [PATCH 084/105] Revert wasmkit bump --- utils/update_checkout/update-checkout-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 75af71026e831..681523754c0e0 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "main", + "wasmkit": "0.0.8", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" From ed544002e9ab6feded5b29eb62d49335f1eba260 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 10 Oct 2024 10:29:22 +0100 Subject: [PATCH 085/105] Bump WasmKit to 0.1.0 in `update-checkout-config.json` --- utils/update_checkout/update-checkout-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 681523754c0e0..910b5c04e9cc4 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.0.8", + "wasmkit": "0.1.0", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" From d568d670282eca62776c3c5b5042646af3fe7881 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 10 Oct 2024 16:37:00 +0100 Subject: [PATCH 086/105] Update for WasmKit 0.1.0 API --- .../swift-plugin-server/WasmKitEngine.swift | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index b2caa56997152..8b7f379d2fb59 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -18,22 +18,28 @@ import SystemPackage typealias DefaultWasmEngine = WasmKitEngine struct WasmKitEngine: WasmEngine { - private let runtime: Runtime + private let engine: Engine private let functions: [String: Function] init(path: FilePath, imports: WASIBridgeToHost) throws { - runtime = Runtime(hostModules: imports.hostModules) + self.engine = Engine() + let store = Store(engine: engine) let module = try parseWasm(filePath: path) - let instance = try runtime.instantiate(module: module) - functions = instance.exports.compactMapValues { export in - guard case let .function(function) = export else { return nil } - return function + var moduleImports = Imports() + imports.link(to: &moduleImports, store: store) + let instance = try module.instantiate(store: store) + var functions = [String: Function]() + for (name, export) in instance.exports { + guard case let .function(function) = export else { continue } + functions[name] = function } + + self.functions = functions } func function(named name: String) throws -> WasmFunction? { guard let function = functions[name] else { return nil } - return { _ = try function.invoke(runtime: runtime) } + return { _ = try function.invoke() } } } From 75bd7ca812091b2c10688cfbf3c89f85d5ee18c5 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 11 Oct 2024 12:15:07 +0100 Subject: [PATCH 087/105] Fix missing imports in `WasmKitEngine.swift` --- .../Sources/swift-plugin-server/WasmKitEngine.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index 8b7f379d2fb59..adac2d35e58d7 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -28,7 +28,7 @@ struct WasmKitEngine: WasmEngine { let module = try parseWasm(filePath: path) var moduleImports = Imports() imports.link(to: &moduleImports, store: store) - let instance = try module.instantiate(store: store) + let instance = try module.instantiate(store: store, imports: moduleImports) var functions = [String: Function]() for (name, export) in instance.exports { guard case let .function(function) = export else { continue } From c1e928d4a48c21e3a14f6a2dd655f1b08b0cede0 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sun, 13 Oct 2024 17:29:48 +0900 Subject: [PATCH 088/105] Bump wasmkit to 0.1.1 to enable WMO This release includes a fix to the build system that enables WMO for release builds. https://github.com/swiftwasm/WasmKit/pull/146 --- utils/update_checkout/update-checkout-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 910b5c04e9cc4..423ad92f0b782 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.1.0", + "wasmkit": "0.1.1", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" From 7683e83f18657052a1270e7c958669a6568de213 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 21 Oct 2024 14:51:15 +0100 Subject: [PATCH 089/105] Bump WasmKit to 0.1.2 in `update-checkout-config.json` --- utils/update_checkout/update-checkout-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 423ad92f0b782..5925bd445f41d 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.1.1", + "wasmkit": "0.1.2", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -422,7 +422,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.0.8", + "wasmkit": "0.1.2", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -799,7 +799,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.0.8", + "wasmkit": "0.1.2", "curl": "curl-8_9_1", "icu": "maint/maint-69", "libxml2": "v2.11.5", From e0427b29645f7cbc7e9b8036a71b210d9fb375af Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 21 Oct 2024 15:41:20 +0100 Subject: [PATCH 090/105] Bump stack size in `WasmKitEngine.swift` to avoid OOB --- .../Sources/swift-plugin-server/WasmKitEngine.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index adac2d35e58d7..db9f3a749dfb6 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -22,7 +22,9 @@ struct WasmKitEngine: WasmEngine { private let functions: [String: Function] init(path: FilePath, imports: WASIBridgeToHost) throws { - self.engine = Engine() + var configuration = EngineConfiguration() + configuration.stackSize = 1 << 20 + self.engine = Engine(configuration: configuration) let store = Store(engine: engine) let module = try parseWasm(filePath: path) From 1852df9bc5a4c61f723da8b7770acc2bf86e3bd2 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 4 Dec 2024 15:57:21 +0000 Subject: [PATCH 091/105] Remove outdated use of `PluginSearchOption::LoadPlugin` This is no longer relevant after recent conflict resolution. --- lib/AST/PluginLoader.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index ce8fee482566c..59646d7721192 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -122,14 +122,6 @@ PluginLoader::getPluginMap() { continue; } - case PluginSearchOption::Kind::LoadPlugin: { - auto &val = entry.get(); - for (auto &moduleName : val.ModuleNames) { - try_emplace(moduleName, val.LibraryPath, val.ServerPath); - } - continue; - } - // '-plugin-path '. case PluginSearchOption::Kind::PluginPath: { auto &val = entry.get(); From e49c980fdcad847f9467cabfc5abad5aa4bcac2f Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 4 Dec 2024 16:01:36 +0000 Subject: [PATCH 092/105] Bump WasmKit to 0.1.3 in `update-checkout-config.json` --- utils/update_checkout/update-checkout-config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 24c5685d64b91..92499b0fe156c 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -165,7 +165,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -372,7 +372,7 @@ "swift-experimental-string-processing": "swift/main", "swift-sdk-generator": "main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "libxml2": "v2.11.5", "zlib": "v1.3.1" @@ -420,7 +420,7 @@ "swift-nio": "2.65.0", "swift-experimental-string-processing": "swift/main", "wasi-libc": "wasi-sdk-22", - "wasmkit": "0.1.2", + "wasmkit": "0.1.3", "curl": "curl-8_9_1", "icu": "maint/maint-69", "libxml2": "v2.11.5", From 052058a102841bf5ea5e4885cfeaafa9b9f9dbf7 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 4 Dec 2024 16:11:01 +0000 Subject: [PATCH 093/105] Remove outdated `case OPT_load_plugin` in `CompilerInvocation.cpp` This case no longer exists after recent conflict resolution with `main`. --- lib/Frontend/CompilerInvocation.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index e4de05d1860b7..3057d6cf458c3 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2195,30 +2195,6 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, } break; } - case OPT_load_plugin: { - // '##' where the module names are - // comma separated. - StringRef pathAndServer; - StringRef modulesStr; - std::tie(pathAndServer, modulesStr) = StringRef(A->getValue()).rsplit('#'); - StringRef path; - StringRef server; - std::tie(path, server) = pathAndServer.rsplit('#'); - std::vector moduleNames; - for (auto name : llvm::split(modulesStr, ',')) { - moduleNames.emplace_back(name); - } - if (path.empty() || server.empty() || moduleNames.empty()) { - Diags.diagnose(SourceLoc(), diag::error_load_plugin, - A->getValue()); - } else { - Opts.PluginSearchOpts.emplace_back( - PluginSearchOption::LoadPlugin{resolveSearchPath(path), - resolveSearchPath(server), - std::move(moduleNames)}); - } - break; - } case OPT_plugin_path: { Opts.PluginSearchOpts.emplace_back( PluginSearchOption::PluginPath{resolveSearchPath(A->getValue())}); From f86052879ade7948cde149e52b4e72d766ac80df Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 4 Dec 2024 16:36:07 +0000 Subject: [PATCH 094/105] Remove outdated use of `LoadPlugin` from `swift-ide-test.cpp` This symbol no longer exists after recent conflict resolution with `main`. --- tools/swift-ide-test/swift-ide-test.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp index 3c887a08728da..6fd9ddff92fd6 100644 --- a/tools/swift-ide-test/swift-ide-test.cpp +++ b/tools/swift-ide-test/swift-ide-test.cpp @@ -356,12 +356,6 @@ LoadPluginExecutable("load-plugin-executable", llvm::cl::desc("load plugin executable"), llvm::cl::cat(Category)); -static llvm::cl::list -LoadPlugin("load-plugin", - llvm::cl::desc("load plugin"), - llvm::cl::cat(Category)); - - static llvm::cl::opt EnableSourceImport("enable-source-import", llvm::cl::Hidden, llvm::cl::cat(Category), llvm::cl::init(false)); @@ -4589,24 +4583,6 @@ int main(int argc, char *argv[]) { std::move(moduleNames)}); } } - if (!options::LoadPlugin.empty()) { - for (auto arg: options::LoadPlugin) { - StringRef pathAndServer; - StringRef modulesStr; - std::tie(pathAndServer, modulesStr) = StringRef(arg).rsplit('#'); - StringRef path; - StringRef server; - std::tie(path, server) = pathAndServer.rsplit('#'); - std::vector moduleNames; - for (auto name : llvm::split(modulesStr, ',')) { - moduleNames.emplace_back(name); - } - InitInvok.getSearchPathOptions().PluginSearchOpts.emplace_back( - PluginSearchOption::LoadPlugin{std::string(path), - std::string(server), - std::move(moduleNames)}); - } - } for (auto path : options::PluginPath) { InitInvok.getSearchPathOptions().PluginSearchOpts.emplace_back( PluginSearchOption::PluginPath{path}); From 28a9efa20583822bdc01c842db3c1ffe637c56b0 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 6 Dec 2024 12:37:53 +0000 Subject: [PATCH 095/105] Update lit tests to use `-load-resolved-plugin` --- include/swift/Option/Options.td | 15 ++++----------- test/Macros/macro_plugin_wasm.swift | 2 +- test/Macros/macro_plugin_wasm_badversion.swift | 2 +- test/Macros/macro_plugin_wasm_guest_error.swift | 2 +- test/Macros/macro_plugin_wasm_nostart.swift | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index aae47c121e691..5072c88d509fb 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -173,12 +173,12 @@ def verify_incremental_dependencies : Flag<["-"], "verify-incremental-dependencies">, Flags<[FrontendOption, HelpHidden]>, HelpText<"Enable the dependency verifier for each frontend job">; - + def strict_implicit_module_context : Flag<["-"], "strict-implicit-module-context">, Flags<[FrontendOption, HelpHidden]>, HelpText<"Enable the strict forwarding of compilation context to downstream implicit module dependencies">; - + def no_strict_implicit_module_context : Flag<["-"], "no-strict-implicit-module-context">, Flags<[FrontendOption, HelpHidden]>, @@ -315,7 +315,7 @@ def tools_directory : Separate<["-"], "tools-directory">, def D : JoinedOrSeparate<["-"], "D">, Flags<[FrontendOption]>, HelpText<"Marks a conditional compilation flag as true">; - + def e : Separate<["-"], "e">, Flags<[NewDriverOnlyOption]>, HelpText<"Executes a line of code provided on the command line">; @@ -2102,7 +2102,7 @@ def external_plugin_path : Separate<["-"], "external-plugin-path">, Group, HelpText<"Add directory to the plugin search path with a plugin server executable">, MetaVarName<"#">; - + def cas_backend: Flag<["-"], "cas-backend">, Flags<[FrontendOption, NoDriverOption]>, HelpText<"Enable using CASBackend for object file output">; @@ -2142,13 +2142,6 @@ def in_process_plugin_server_path : Separate<["-"], "in-process-plugin-server-pa Flags<[FrontendOption, ArgumentIsPath]>, HelpText<"Path to dynamic library plugin server">; -def load_plugin: - Separate<["-"], "load-plugin">, Group, - Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>, - HelpText<"Path to a plugin library, a server to load it in, and a comma-separated " - "list of module names where the macro types are declared">, - MetaVarName<"##">; - def disable_sandbox: Flag<["-"], "disable-sandbox">, Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>, diff --git a/test/Macros/macro_plugin_wasm.swift b/test/Macros/macro_plugin_wasm.swift index f93e8ff2f00ea..871a17414ea10 100644 --- a/test/Macros/macro_plugin_wasm.swift +++ b/test/Macros/macro_plugin_wasm.swift @@ -11,7 +11,7 @@ // RUN: env SWIFT_DUMP_PLUGIN_MESSAGING=1 %target-swift-frontend \ // RUN: -typecheck \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -Rmacro-loading \ // RUN: -module-name MyApp \ // RUN: %t/test.swift \ diff --git a/test/Macros/macro_plugin_wasm_badversion.swift b/test/Macros/macro_plugin_wasm_badversion.swift index 5f72fe1f4c6dc..7d63493732c30 100644 --- a/test/Macros/macro_plugin_wasm_badversion.swift +++ b/test/Macros/macro_plugin_wasm_badversion.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift diff --git a/test/Macros/macro_plugin_wasm_guest_error.swift b/test/Macros/macro_plugin_wasm_guest_error.swift index 984a8b426def7..e0ff5a9a4b112 100644 --- a/test/Macros/macro_plugin_wasm_guest_error.swift +++ b/test/Macros/macro_plugin_wasm_guest_error.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift \ // RUN: 2>%t/macro-loading.txt diff --git a/test/Macros/macro_plugin_wasm_nostart.swift b/test/Macros/macro_plugin_wasm_nostart.swift index c1789cacf8dd9..18929f911e7b6 100644 --- a/test/Macros/macro_plugin_wasm_nostart.swift +++ b/test/Macros/macro_plugin_wasm_nostart.swift @@ -11,7 +11,7 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify \ // RUN: -swift-version 5 \ -// RUN: -load-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ +// RUN: -load-resolved-plugin %t/Plugin.wasm#%swift-plugin-server#MacroDefinition \ // RUN: -module-name MyApp \ // RUN: %t/test.swift From 54eb48307c8cc86adc341f3918ca10a5c5da227e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 12 Dec 2024 13:52:38 +0000 Subject: [PATCH 096/105] Adopt new `shutDown` API introduced in `swift-syntax` --- .../Sources/swift-plugin-server/WasmEngine.swift | 6 ++++++ .../swift-plugin-server/WasmKitEngine.swift | 4 ++++ .../swift-plugin-server/WasmMessageHandler.swift | 16 +++++++++++++--- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 310e9be7adb07..138e566093cd3 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -20,6 +20,8 @@ protocol WasmEngine { init(path: FilePath, imports: WASIBridgeToHost) throws func function(named name: String) throws -> WasmFunction? + + func shutDown() throws } typealias DefaultWasmPlugin = WasmEnginePlugin @@ -80,6 +82,10 @@ struct WasmEnginePlugin: WasmPlugin { size = received } } + + func shutDown() throws { + try self.engine.shutDown() + } } struct WasmEngineError: Error, CustomStringConvertible { diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index db9f3a749dfb6..8d83a7e54f13b 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -44,4 +44,8 @@ struct WasmKitEngine: WasmEngine { guard let function = functions[name] else { return nil } return { _ = try function.invoke() } } + + func shutDown() throws { + // No resources requiring explicit shut down in `WasmKitEngine`. + } } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index f6cd57cf8c6dc..e5855a5f0c082 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -30,7 +30,7 @@ final class WasmInterceptingMessageHandler: PluginMe guard libraryPath.hasSuffix(".wasm") else { break } let libraryFilePath = FilePath(libraryPath) do { - loadedWasmPlugins[moduleName] = try DefaultWasmPlugin(path: libraryFilePath) + self.loadedWasmPlugins[moduleName] = try DefaultWasmPlugin(path: libraryFilePath) } catch { return .loadPluginLibraryResult( loaded: false, @@ -40,7 +40,7 @@ final class WasmInterceptingMessageHandler: PluginMe return .loadPluginLibraryResult(loaded: true, diagnostics: []) case .expandAttachedMacro(let macro, _, _, let syntax, _, _, _, _, _), .expandFreestandingMacro(let macro, _, _, let syntax, _): - if let response = expandMacro(macro, message: message, location: syntax.location) { + if let response = self.expandMacro(macro, message: message, location: syntax.location) { return response } // else break case .getCapability: @@ -53,12 +53,20 @@ final class WasmInterceptingMessageHandler: PluginMe return base.handleMessage(message) } + func shutDown() throws { + for plugin in self.loadedWasmPlugins.values { + try plugin.shutDown() + } + + self.loadedWasmPlugins = [:] + } + private func expandMacro( _ macro: PluginMessage.MacroReference, message: HostToPluginMessage, location: PluginMessage.SourceLocation? ) -> PluginToHostMessage? { - guard let plugin = loadedWasmPlugins[macro.moduleName] else { return nil } + guard let plugin = self.loadedWasmPlugins[macro.moduleName] else { return nil } do { let request = try JSON.encode(message) let responseRaw = try plugin.handleMessage(request) @@ -109,4 +117,6 @@ protocol WasmPlugin { init(path: FilePath) throws func handleMessage(_ json: [UInt8]) throws -> [UInt8] + + func shutDown() throws } From 224ac091f75aef8b8b5709fef6d979a4dcd1d13d Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 13 Dec 2024 19:03:47 +0000 Subject: [PATCH 097/105] Add a trivial echo macro test to the package --- tools/swift-plugin-server/Package.swift | 8 ++ .../PluginServerTests/WasmEngineTests.swift | 101 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift index ca3a8057e0e3a..bbea463dcd651 100644 --- a/tools/swift-plugin-server/Package.swift +++ b/tools/swift-plugin-server/Package.swift @@ -32,6 +32,14 @@ let package = Package( .product(name: "_SwiftLibraryPluginProvider", package: "swift-syntax"), ] ), + .testTarget( + name: "PluginServerTests", + dependencies: [ + .product(name: "WAT", package: "WasmKit"), + .product(name: "WasmKit", package: "WasmKit"), + "swift-plugin-server", + ] + ), ], cxxLanguageStandard: .cxx17 ) diff --git a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift new file mode 100644 index 0000000000000..017171fcdfde2 --- /dev/null +++ b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift @@ -0,0 +1,101 @@ +import SystemPackage +import Testing +import WasmKit +import WasmKitWASI +import WAT + +private let echo = #""" +(module + (import "wasi_snapshot_preview1" "fd_read" + (func $fd_read + (param (; fd ;) i32 (; ioVecOffset ;) i32 (; ioVecCount ;) i32 (; readCountPointer ;) i32) + (result (; errno ;) i32) + ) + ) + + (import "wasi_snapshot_preview1" "fd_write" + (func $fd_write + (param (; fd ;) i32 (; ioVecOffset ;) i32 (; ioVecCount ;) i32 (; writtenCountPointer ;) i32) + (result (; errno ;) i32) + ) + ) + + (export "_start" (func $start)) + (export "swift_wasm_macro_v1_pump" (func $pump)) + (memory $memory 1) + (export "memory" (memory $memory)) + + (func $start) + + (func $pump + (memory.fill + (i32.const 0) + (i32.const 0) + (i32.const 1024) + ) + + (i32.store + (i32.const 0) (; first byte is the buffer pointer ;) + (i32.const 8) (; 4 bytes for the pointer + 4 bytes for the count ;) + ) + (i32.store + (i32.const 4 ) (; 4 bytes already used for the pointer ;) + (i32.const 1024) (; 1024 bytes should be enough for everyone (probably) ;) + ) + + (call $fd_read + (i32.const 0) (; stdin FD ;) + (i32.const 0) (; ioVecOffset ;) + (i32.const 1) (; ioVecCount ;) + (i32.const 1028) (; skip buffer and its count and store read counter after that ;) + ) + (drop) (; ignore the result ;) + + (call $fd_write + (i32.const 1) (; stdout FD ;) + (i32.const 0) (; ioVecOffset ;) + (i32.const 1) (; ioVecCount ;) + (i32.const 1028) (; skip buffer and its count and store write counter after that ;) + ) + (drop) (; ignore the result ;) + ) +) +"""# + +@Suite +struct WasmEngineTests { + @Test + func basic() throws { + let binary = try wat2wasm(echo) + let module = try parseWasm(bytes: binary) + + let engine = Engine() + let store = Store(engine: engine) + + var imports = Imports() + + let hostToPluginPipe = try FileDescriptor.pipe() + let pluginToHostPipe = try FileDescriptor.pipe() + + let bridge = try WASIBridgeToHost( + stdin: hostToPluginPipe.readEnd, + stdout: pluginToHostPipe.writeEnd, + stderr: .standardError + ) + + bridge.link(to: &imports, store: store) + let instance = try module.instantiate(store: store, imports: imports) + #expect(try instance.exports[function: "_start"]!() == []) + + let str = "Hello World!\n" + try hostToPluginPipe.writeEnd.writeAll(str.utf8) + + #expect(try instance.exports[function: "swift_wasm_macro_v1_pump"]!() == []) + + var buffer = [UInt8](repeating: 0, count: 13) + _ = try buffer.withUnsafeMutableBytes { + try pluginToHostPipe.readEnd.read(into: $0) + } + #expect(String(decoding: buffer, as: UTF8.self) == str) + } +} From dc8cd3cc4232d109ee25d602e3b2d8d85555edf3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 16 Dec 2024 15:06:51 +0000 Subject: [PATCH 098/105] Test `WasmEnginePlugin` APIs in `WasmEngineTests` --- .../PluginServerTests/WasmEngineTests.swift | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift index 017171fcdfde2..35ac0f6be271d 100644 --- a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift +++ b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift @@ -1,3 +1,5 @@ +@testable import swift_plugin_server +import Foundation import SystemPackage import Testing import WasmKit @@ -67,35 +69,11 @@ struct WasmEngineTests { @Test func basic() throws { let binary = try wat2wasm(echo) - let module = try parseWasm(bytes: binary) + let echoURL = FileManager.default.temporaryDirectory.appendingPathExtension("echo.wasm") + try Data(binary).write(to: echoURL) - let engine = Engine() - let store = Store(engine: engine) - - var imports = Imports() - - let hostToPluginPipe = try FileDescriptor.pipe() - let pluginToHostPipe = try FileDescriptor.pipe() - - let bridge = try WASIBridgeToHost( - stdin: hostToPluginPipe.readEnd, - stdout: pluginToHostPipe.writeEnd, - stderr: .standardError - ) - - bridge.link(to: &imports, store: store) - let instance = try module.instantiate(store: store, imports: imports) - #expect(try instance.exports[function: "_start"]!() == []) - - let str = "Hello World!\n" - try hostToPluginPipe.writeEnd.writeAll(str.utf8) - - #expect(try instance.exports[function: "swift_wasm_macro_v1_pump"]!() == []) - - var buffer = [UInt8](repeating: 0, count: 13) - _ = try buffer.withUnsafeMutableBytes { - try pluginToHostPipe.readEnd.read(into: $0) - } - #expect(String(decoding: buffer, as: UTF8.self) == str) + let engine = try WasmEnginePlugin(path: FilePath(echoURL.path)) + let input: [UInt8] = [1,2,3,4,5] + #expect(try engine.handleMessage(input) == input) } } From f56ec6a16c2f687afe71dcc2531758d2c56b23aa Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 16 Dec 2024 15:24:08 +0000 Subject: [PATCH 099/105] Make formatting consistent in `swift-plugin-server` --- tools/swift-plugin-server/.editorconfig | 9 ++++ tools/swift-plugin-server/.swiftformat | 3 ++ .../InProcPluginServer.swift | 15 +++---- .../swift-plugin-server/WasmEngine.swift | 43 ++++++++++--------- .../swift-plugin-server/WasmKitEngine.swift | 2 +- .../WasmMessageHandler.swift | 26 +++++------ .../swift-plugin-server.swift | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) create mode 100644 tools/swift-plugin-server/.editorconfig create mode 100644 tools/swift-plugin-server/.swiftformat diff --git a/tools/swift-plugin-server/.editorconfig b/tools/swift-plugin-server/.editorconfig new file mode 100644 index 0000000000000..410ff6c9ed018 --- /dev/null +++ b/tools/swift-plugin-server/.editorconfig @@ -0,0 +1,9 @@ +# editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/tools/swift-plugin-server/.swiftformat b/tools/swift-plugin-server/.swiftformat new file mode 100644 index 0000000000000..57b8fe01a6a08 --- /dev/null +++ b/tools/swift-plugin-server/.swiftformat @@ -0,0 +1,3 @@ +--indent 2 +--self insert + diff --git a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift index 2902f8c200eba..fa61d6acab2fb 100644 --- a/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift +++ b/tools/swift-plugin-server/Sources/SwiftInProcPluginServer/InProcPluginServer.swift @@ -14,17 +14,17 @@ @_spi(PluginMessage) import SwiftLibraryPluginProvider #if canImport(Darwin) -import Darwin + import Darwin #elseif canImport(Glibc) -import Glibc + import Glibc #elseif canImport(Bionic) -import Bionic + import Bionic #elseif canImport(Musl) -import Musl + import Musl #elseif canImport(ucrt) -import ucrt + import ucrt #else -#error("'malloc' not found") + #error("'malloc' not found") #endif /// Entry point. @@ -80,11 +80,10 @@ struct InProcPluginServer { func handleMessage(_ input: UnsafeBufferPointer) throws -> [UInt8] { let request = try JSON.decode(HostToPluginMessage.self, from: input) - let response = handler.handleMessage(request) + let response = self.handler.handleMessage(request) return try JSON.encode(response) } @MainActor static let shared = Self() } - diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 138e566093cd3..bdc41e7cde2d3 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -10,9 +10,9 @@ // //===----------------------------------------------------------------------===// +import SystemPackage import WASI import WasmTypes -import SystemPackage typealias WasmFunction = () throws -> Void @@ -20,7 +20,7 @@ protocol WasmEngine { init(path: FilePath, imports: WASIBridgeToHost) throws func function(named name: String) throws -> WasmFunction? - + func shutDown() throws } @@ -44,7 +44,8 @@ struct WasmEnginePlugin: WasmPlugin { stdout: pluginToHostPipes.writeEnd, stderr: .standardError ) - engine = try Engine(path: path, imports: bridge) + + self.engine = try Engine(path: path, imports: bridge) let exportName = "swift_wasm_macro_v1_pump" guard let pump = try engine.function(named: exportName) else { @@ -60,11 +61,11 @@ struct WasmEnginePlugin: WasmPlugin { func handleMessage(_ json: [UInt8]) throws -> [UInt8] { try withUnsafeBytes(of: UInt64(json.count).littleEndian) { - _ = try hostToPlugin.writeAll($0) + _ = try self.hostToPlugin.writeAll($0) } - try hostToPlugin.writeAll(json) + try self.hostToPlugin.writeAll(json) - try pumpFunction() + try self.pumpFunction() let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) @@ -100,25 +101,25 @@ struct WasmEngineError: Error, CustomStringConvertible { // but it is available in the latest main branch. Remove the following code when we // update to the next release of swift-system. #if os(Windows) -import ucrt + import ucrt -internal var system_errno: CInt { - var value: CInt = 0 - _ = ucrt._get_errno(&value) - return value -} + var system_errno: CInt { + var value: CInt = 0 + _ = ucrt._get_errno(&value) + return value + } -extension FileDescriptor { - static func pipe() throws -> (readEnd: FileDescriptor, writeEnd: FileDescriptor) { - var fds: (Int32, Int32) = (-1, -1) - return try withUnsafeMutablePointer(to: &fds) { pointer in - return try pointer.withMemoryRebound(to: Int32.self, capacity: 2) { fds in - guard _pipe(fds, 4096, _O_BINARY | _O_NOINHERIT) == 0 else { - throw Errno(rawValue: system_errno) + extension FileDescriptor { + static func pipe() throws -> (readEnd: FileDescriptor, writeEnd: FileDescriptor) { + var fds: (Int32, Int32) = (-1, -1) + return try withUnsafeMutablePointer(to: &fds) { pointer in + try pointer.withMemoryRebound(to: Int32.self, capacity: 2) { fds in + guard _pipe(fds, 4096, _O_BINARY | _O_NOINHERIT) == 0 else { + throw Errno(rawValue: system_errno) + } + return (FileDescriptor(rawValue: fds[0]), FileDescriptor(rawValue: fds[1])) } - return (FileDescriptor(rawValue: fds[0]), FileDescriptor(rawValue: fds[1])) } } } -} #endif diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index 8d83a7e54f13b..e4c7529e17b33 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -10,10 +10,10 @@ // //===----------------------------------------------------------------------===// +import SystemPackage import WASI import WasmKit import WasmKitWASI -import SystemPackage typealias DefaultWasmEngine = WasmKitEngine diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift index e5855a5f0c082..e397c06c1335d 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmMessageHandler.swift @@ -15,7 +15,7 @@ import SystemPackage /// A `PluginMessageHandler` that intercepts messages intended for Wasm plugins. final class WasmInterceptingMessageHandler: PluginMessageHandler { - private var loadedWasmPlugins: [String: WasmPlugin] = [:] + private var loadedWasmPlugins: [String: any WasmPlugin] = [:] let base: Base init(base: Base) { @@ -26,7 +26,7 @@ final class WasmInterceptingMessageHandler: PluginMe /// Otherwise, forward it to `base`. func handleMessage(_ message: HostToPluginMessage) -> PluginToHostMessage { switch message { - case .loadPluginLibrary(let libraryPath, let moduleName): + case let .loadPluginLibrary(libraryPath, moduleName): guard libraryPath.hasSuffix(".wasm") else { break } let libraryFilePath = FilePath(libraryPath) do { @@ -38,19 +38,19 @@ final class WasmInterceptingMessageHandler: PluginMe ) } return .loadPluginLibraryResult(loaded: true, diagnostics: []) - case .expandAttachedMacro(let macro, _, _, let syntax, _, _, _, _, _), - .expandFreestandingMacro(let macro, _, _, let syntax, _): + case let .expandAttachedMacro(macro, _, _, syntax, _, _, _, _, _), + let .expandFreestandingMacro(macro, _, _, syntax, _): if let response = self.expandMacro(macro, message: message, location: syntax.location) { return response } // else break case .getCapability: break -#if !SWIFT_PACKAGE - @unknown default: - break -#endif + #if !SWIFT_PACKAGE + @unknown default: + break + #endif } - return base.handleMessage(message) + return self.base.handleMessage(message) } func shutDown() throws { @@ -91,8 +91,8 @@ final class WasmInterceptingMessageHandler: PluginMe } } -extension PluginMessage.Diagnostic { - fileprivate init( +fileprivate extension PluginMessage.Diagnostic { + init( errorMessage: String, position: PluginMessage.Diagnostic.Position = .invalid ) { @@ -107,8 +107,8 @@ extension PluginMessage.Diagnostic { } } -extension PluginMessage.SourceLocation { - fileprivate var position: PluginMessage.Diagnostic.Position { +fileprivate extension PluginMessage.SourceLocation { + var position: PluginMessage.Diagnostic.Position { .init(fileName: fileName, offset: offset) } } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 62c05ab2a9e96..31ebc78353eb3 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -14,7 +14,7 @@ @_spi(PluginMessage) import SwiftLibraryPluginProvider @main -final class SwiftPluginServer { +enum SwiftPluginServer { static func main() throws { let connection = try StandardIOMessageConnection() let listener = CompilerPluginMessageListener( From 1626ad83a373538a2fc0fdf5092bbeb812228891 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 16 Dec 2024 16:07:51 +0000 Subject: [PATCH 100/105] Create stdin/stdout pipes for plugins with `mkfifo` This makes the API more portable, allowing file paths instead of file descriptors to be passed to Wasm engines. --- .../swift-plugin-server/FilePathTemp.swift | 111 +++++++++++ .../FilePathTempPOSIX.swift | 176 ++++++++++++++++++ .../swift-plugin-server/FilePathWindows.swift | 116 ++++++++++++ ...n-server.swift => SwiftPluginServer.swift} | 0 .../swift-plugin-server/WasmEngine.swift | 64 +++++-- .../swift-plugin-server/WasmKitEngine.swift | 24 ++- 6 files changed, 472 insertions(+), 19 deletions(-) create mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift create mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift create mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift rename tools/swift-plugin-server/Sources/swift-plugin-server/{swift-plugin-server.swift => SwiftPluginServer.swift} (100%) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift new file mode 100644 index 0000000000000..324bd603020be --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTemp.swift @@ -0,0 +1,111 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unsupported platform") +#endif + +import SystemPackage + +// MARK: - API + +/// Create a temporary path for the duration of the closure. +/// +/// - Parameters: +/// - basename: The base name for the temporary path. +/// - body: The closure to execute. +/// +/// Creates a temporary directory with a name based on the given `basename`, +/// executes `body`, passing in the path of the created directory, then +/// deletes the directory and all of its contents before returning. +func withTemporaryFilePath( + basename: FilePath.Component, + _ body: (FilePath) throws -> R +) throws -> R { + let temporaryDir = try createUniqueTemporaryDirectory(basename: basename) + defer { + try? _recursiveRemove(at: temporaryDir) + } + + return try body(temporaryDir) +} + +// MARK: - Internals + +fileprivate let base64 = Array( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".utf8 +) + +/// Create a directory that is only accessible to the current user. +/// +/// - Parameters: +/// - path: The path of the directory to create. +/// - Returns: `true` if a new directory was created. +/// +/// This function will throw if there is an error, except if the error +/// is that the directory exists, in which case it returns `false`. +fileprivate func makeLockedDownDirectory(at path: FilePath) throws -> Bool { + return try path.withPlatformString { + if mkdir($0, 0o700) == 0 { + return true + } + let err = errno + if err == Errno.fileExists.rawValue { + return false + } else { + throw Errno(rawValue: err) + } + } +} + +/// Generate a random string of base64 filename safe characters. +/// +/// - Parameters: +/// - length: The number of characters in the returned string. +/// - Returns: A random string of length `length`. +fileprivate func createRandomString(length: Int) -> String { + return String( + decoding: (0.. FilePath { + var tempDir = try _getTemporaryDirectory() + tempDir.append(basename) + + while true { + tempDir.extension = createRandomString(length: 16) + + if try makeLockedDownDirectory(at: tempDir) { + return tempDir + } + } +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift new file mode 100644 index 0000000000000..be1a7f797aaef --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathTempPOSIX.swift @@ -0,0 +1,176 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information + */ + +#if !os(Windows) + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unsupported platform") +#endif + +import SystemPackage + +/// Get the path to the system temporary directory. +internal func _getTemporaryDirectory() throws -> FilePath { + guard let tmp = getenv("TMPDIR") else { + return "/tmp" + } + + return FilePath(stringLiteral: .init(cString: tmp)) +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - path: The directory to be deleted. +/// +/// Removes a directory completely, including all of its contents. +internal func _recursiveRemove( + at path: FilePath +) throws { + let dirfd = try FileDescriptor.open(path, .readOnly, options: .directory) + defer { + try? dirfd.close() + } + + let dot: (CInterop.PlatformChar, CInterop.PlatformChar) = (46, 0) + try withUnsafeBytes(of: dot) { + try recursiveRemove( + in: dirfd.rawValue, + name: $0.assumingMemoryBound(to: CInterop.PlatformChar.self).baseAddress! + ) + } + + try path.withPlatformString { + let error = rmdir($0) + if error != 0 { + throw Errno(rawValue: error) + } + } +} + +internal let SYSTEM_AT_REMOVE_DIR = AT_REMOVEDIR +internal let SYSTEM_DT_DIR = DT_DIR +internal typealias system_dirent = dirent +#if os(Linux) || os(Android) +internal typealias system_DIRPtr = OpaquePointer +#else +internal typealias system_DIRPtr = UnsafeMutablePointer +#endif + + +/// Open a directory by reference to its parent and name. +/// +/// - Parameters: +/// - dirfd: An open file descriptor for the parent directory. +/// - name: The name of the directory to open. +/// - Returns: A pointer to a `DIR` structure. +/// +/// This is like `opendir()`, but instead of taking a path, it uses a +/// file descriptor pointing at the parent, thus avoiding path length +/// limits. +fileprivate func impl_opendirat( + _ dirfd: CInt, + _ name: UnsafePointer +) -> system_DIRPtr? { + let fd = openat(dirfd, name, + FileDescriptor.AccessMode.readOnly.rawValue + | FileDescriptor.OpenOptions.directory.rawValue) + if fd < 0 { + return nil + } + return fdopendir(fd) +} + +/// Invoke a closure for each file within a particular directory. +/// +/// - Parameters: +/// - dirfd: The parent of the directory to be enumerated. +/// - subdir: The subdirectory to be enumerated. +/// - body: The closure that will be invoked. +/// +/// We skip the `.` and `..` pseudo-entries. +fileprivate func forEachFile( + in dirfd: CInt, + subdir: UnsafePointer, + _ body: (system_dirent) throws -> () +) throws { + guard let dir = impl_opendirat(dirfd, subdir) else { + throw Errno(rawValue: errno) + } + defer { + _ = closedir(dir) + } + + while let dirent = readdir(dir) { + // Skip . and .. + if dirent.pointee.d_name.0 == 46 + && (dirent.pointee.d_name.1 == 0 + || (dirent.pointee.d_name.1 == 46 + && dirent.pointee.d_name.2 == 0)) { + continue + } + + try body(dirent.pointee) + } +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - dirfd: The parent of the directory to be removed. +/// - name: The name of the directory to be removed. +/// +/// Removes a directory completely, including all of its contents. +fileprivate func recursiveRemove( + in dirfd: CInt, + name: UnsafePointer +) throws { + // First, deal with subdirectories + try forEachFile(in: dirfd, subdir: name) { dirent in + if dirent.d_type == SYSTEM_DT_DIR { + try withUnsafeBytes(of: dirent.d_name) { + try recursiveRemove( + in: dirfd, + name: $0.assumingMemoryBound(to: CInterop.PlatformChar.self) + .baseAddress! + ) + } + } + } + + // Now delete the contents of this directory + try forEachFile(in: dirfd, subdir: name) { dirent in + let flag: CInt + + if dirent.d_type == SYSTEM_DT_DIR { + flag = SYSTEM_AT_REMOVE_DIR + } else { + flag = 0 + } + + let result = withUnsafeBytes(of: dirent.d_name) { + unlinkat(dirfd, $0.assumingMemoryBound(to: CInterop.PlatformChar.self).baseAddress!, flag) + } + + if result != 0 { + throw Errno(rawValue: errno) + } + } +} + +#endif // !os(Windows) + diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift new file mode 100644 index 0000000000000..b19f0c7603c34 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/FilePathWindows.swift @@ -0,0 +1,116 @@ +/* + This source file is part of the Swift System open source project + + Copyright (c) 2024 Apple Inc. and the Swift System project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See https://swift.org/LICENSE.txt for license information +*/ + +#if os(Windows) + +import SystemPackage + +import WinSDK + +/// Get the path to the system temporary directory. +internal func _getTemporaryDirectory() throws -> FilePath { + return try withUnsafeTemporaryAllocation(of: CInterop.PlatformChar.self, + capacity: Int(MAX_PATH) + 1) { + buffer in + + guard GetTempPath2W(DWORD(buffer.count), buffer.baseAddress) != 0 else { + throw Errno(windowsError: GetLastError()) + } + + return FilePath(SystemString(platformString: buffer.baseAddress!)) + } +} + +/// Invoke a closure for each file within a particular directory. +/// +/// - Parameters: +/// - path: The path at which we should enumerate items. +/// - body: The closure that will be invoked. +/// +/// We skip the `.` and `..` pseudo-entries. +fileprivate func forEachFile( + at path: FilePath, + _ body: (WIN32_FIND_DATAW) throws -> () +) rethrows { + let searchPath = path.appending("\\*") + + try searchPath.withPlatformString { szPath in + var findData = WIN32_FIND_DATAW() + let hFind = FindFirstFileW(szPath, &findData) + if hFind == INVALID_HANDLE_VALUE { + throw Errno(windowsError: GetLastError()) + } + defer { + FindClose(hFind) + } + + repeat { + // Skip . and .. + if findData.cFileName.0 == 46 + && (findData.cFileName.1 == 0 + || (findData.cFileName.1 == 46 + && findData.cFileName.2 == 0)) { + continue + } + + try body(findData) + } while FindNextFileW(hFind, &findData) + } +} + +/// Delete the entire contents of a directory, including its subdirectories. +/// +/// - Parameters: +/// - path: The directory to be deleted. +/// +/// Removes a directory completely, including all of its contents. +internal func _recursiveRemove( + at path: FilePath +) throws { + // First, deal with subdirectories + try forEachFile(at: path) { findData in + if (findData.dwFileAttributes & DWORD(FILE_ATTRIBUTE_DIRECTORY)) != 0 { + let name = withUnsafeBytes(of: findData.cFileName) { + return SystemString(platformString: $0.assumingMemoryBound( + to: CInterop.PlatformChar.self).baseAddress!) + } + let component = FilePath.Component(name)! + let subpath = path.appending(component) + + try _recursiveRemove(at: subpath) + } + } + + // Now delete everything else + try forEachFile(at: path) { findData in + let name = withUnsafeBytes(of: findData.cFileName) { + return SystemString(platformString: $0.assumingMemoryBound( + to: CInterop.PlatformChar.self).baseAddress!) + } + let component = FilePath.Component(name)! + let subpath = path.appending(component) + + if (findData.dwFileAttributes & DWORD(FILE_ATTRIBUTE_DIRECTORY)) == 0 { + try subpath.withPlatformString { + if !DeleteFileW($0) { + throw Errno(windowsError: GetLastError()) + } + } + } + } + + // Finally, delete the parent + try path.withPlatformString { + if !RemoveDirectoryW($0) { + throw Errno(windowsError: GetLastError()) + } + } +} + +#endif // os(Windows) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/SwiftPluginServer.swift similarity index 100% rename from tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift rename to tools/swift-plugin-server/Sources/swift-plugin-server/SwiftPluginServer.swift diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index bdc41e7cde2d3..dc92ad255ed88 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -10,6 +10,18 @@ // //===----------------------------------------------------------------------===// +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unsupported platform") +#endif + import SystemPackage import WASI import WasmTypes @@ -17,10 +29,13 @@ import WasmTypes typealias WasmFunction = () throws -> Void protocol WasmEngine { - init(path: FilePath, imports: WASIBridgeToHost) throws + init(path: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws func function(named name: String) throws -> WasmFunction? + func writeToPlugin(_ storage: some Sequence) throws + func readFromPlugin(into storage: UnsafeMutableRawBufferPointer) throws -> Int + func shutDown() throws } @@ -28,24 +43,38 @@ typealias DefaultWasmPlugin = WasmEnginePlugin // a WasmPlugin implementation that delegates to a WasmEngine struct WasmEnginePlugin: WasmPlugin { - private let hostToPlugin: FileDescriptor - private let pluginToHost: FileDescriptor + enum Error: Swift.Error { + case failedToCreateNamedPipe(FilePath) + } + private let pumpFunction: WasmFunction + private let tempDirectory: FilePath + private let stdinPath: FilePath + private let stdoutPath: FilePath let engine: Engine init(path: FilePath) throws { - let hostToPluginPipes = try FileDescriptor.pipe() - let pluginToHostPipes = try FileDescriptor.pipe() - self.hostToPlugin = hostToPluginPipes.writeEnd - self.pluginToHost = pluginToHostPipes.readEnd + self.tempDirectory = try createUniqueTemporaryDirectory(basename: "swift_wasm_macros") + self.stdinPath = self.tempDirectory.appending("stdin") + self.stdoutPath = self.tempDirectory.appending("stdout") + + let stdinResult = self.stdinPath.withCString { + mkfifo($0, S_IRUSR | S_IWUSR) + } + + guard stdinResult == 0 else { + throw Error.failedToCreateNamedPipe(self.stdinPath) + } - let bridge = try WASIBridgeToHost( - stdin: hostToPluginPipes.readEnd, - stdout: pluginToHostPipes.writeEnd, - stderr: .standardError - ) + let stdoutResult = self.stdoutPath.withCString { + mkfifo($0, S_IRUSR | S_IWUSR) + } + + guard stdoutResult == 0 else { + throw Error.failedToCreateNamedPipe(self.stdoutPath) + } - self.engine = try Engine(path: path, imports: bridge) + self.engine = try Engine(path: path, stdinPath: self.stdinPath, stdoutPath: self.stdoutPath) let exportName = "swift_wasm_macro_v1_pump" guard let pump = try engine.function(named: exportName) else { @@ -61,14 +90,14 @@ struct WasmEnginePlugin: WasmPlugin { func handleMessage(_ json: [UInt8]) throws -> [UInt8] { try withUnsafeBytes(of: UInt64(json.count).littleEndian) { - _ = try self.hostToPlugin.writeAll($0) + _ = try engine.writeToPlugin($0) } - try self.hostToPlugin.writeAll(json) + try engine.writeToPlugin(json) try self.pumpFunction() let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in - let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) + let lengthCount = try engine.readFromPlugin(into: UnsafeMutableRawBufferPointer(buffer)) guard lengthCount == 8 else { throw WasmEngineError(message: "Wasm plugin sent invalid response") } @@ -76,7 +105,7 @@ struct WasmEnginePlugin: WasmPlugin { } let length = Int(UInt64(littleEndian: lengthRaw)) return try [UInt8](unsafeUninitializedCapacity: length) { buffer, size in - let received = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer)) + let received = try engine.readFromPlugin(into: UnsafeMutableRawBufferPointer(buffer)) guard received == length else { throw WasmEngineError(message: "Wasm plugin sent truncated response") } @@ -86,6 +115,7 @@ struct WasmEnginePlugin: WasmPlugin { func shutDown() throws { try self.engine.shutDown() + try _recursiveRemove(at: self.tempDirectory) } } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index e4c7529e17b33..aa07c46095000 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -20,8 +20,13 @@ typealias DefaultWasmEngine = WasmKitEngine struct WasmKitEngine: WasmEngine { private let engine: Engine private let functions: [String: Function] + private let hostToPlugin: FileDescriptor + private let pluginToHost: FileDescriptor + + init(path: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws { + self.hostToPlugin = try FileDescriptor.open(stdinPath, .readWrite) + self.pluginToHost = try FileDescriptor.open(stdoutPath, .readWrite) - init(path: FilePath, imports: WASIBridgeToHost) throws { var configuration = EngineConfiguration() configuration.stackSize = 1 << 20 self.engine = Engine(configuration: configuration) @@ -29,6 +34,12 @@ struct WasmKitEngine: WasmEngine { let module = try parseWasm(filePath: path) var moduleImports = Imports() + + let imports = try WASIBridgeToHost( + stdin: self.hostToPlugin, + stdout: self.pluginToHost, + stderr: .standardError + ) imports.link(to: &moduleImports, store: store) let instance = try module.instantiate(store: store, imports: moduleImports) var functions = [String: Function]() @@ -45,7 +56,16 @@ struct WasmKitEngine: WasmEngine { return { _ = try function.invoke() } } + func writeToPlugin(_ storage: some Sequence) throws { + try self.hostToPlugin.writeAll(storage) + } + + func readFromPlugin(into storage: UnsafeMutableRawBufferPointer) throws -> Int { + try self.pluginToHost.read(into: storage) + } + func shutDown() throws { - // No resources requiring explicit shut down in `WasmKitEngine`. + try self.hostToPlugin.close() + try self.pluginToHost.close() } } From 4204eef086a5fb3b66949f17a85d6eecb86ac672 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 16 Dec 2024 16:10:36 +0000 Subject: [PATCH 101/105] Clean up temp Wasm fixture file in `WasmEngineTests` --- .../Tests/PluginServerTests/WasmEngineTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift index 35ac0f6be271d..528d1d87bee48 100644 --- a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift +++ b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift @@ -71,6 +71,7 @@ struct WasmEngineTests { let binary = try wat2wasm(echo) let echoURL = FileManager.default.temporaryDirectory.appendingPathExtension("echo.wasm") try Data(binary).write(to: echoURL) + defer { try! FileManager.default.removeItem(at: echoURL) } let engine = try WasmEnginePlugin(path: FilePath(echoURL.path)) let input: [UInt8] = [1,2,3,4,5] From 1cf1e203feb0b086ad5f7ea88b7f7bf75e7eee2e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 16 Dec 2024 16:22:56 +0000 Subject: [PATCH 102/105] Make `init` parameter names more explicit in `WasmEngine` --- .../Sources/swift-plugin-server/WasmEngine.swift | 4 ++-- .../Sources/swift-plugin-server/WasmKitEngine.swift | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index dc92ad255ed88..599fe8e989d3c 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -29,7 +29,7 @@ import WasmTypes typealias WasmFunction = () throws -> Void protocol WasmEngine { - init(path: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws + init(pluginPath: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws func function(named name: String) throws -> WasmFunction? @@ -74,7 +74,7 @@ struct WasmEnginePlugin: WasmPlugin { throw Error.failedToCreateNamedPipe(self.stdoutPath) } - self.engine = try Engine(path: path, stdinPath: self.stdinPath, stdoutPath: self.stdoutPath) + self.engine = try Engine(pluginPath: path, stdinPath: self.stdinPath, stdoutPath: self.stdoutPath) let exportName = "swift_wasm_macro_v1_pump" guard let pump = try engine.function(named: exportName) else { diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index aa07c46095000..52ed462960b74 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -23,7 +23,7 @@ struct WasmKitEngine: WasmEngine { private let hostToPlugin: FileDescriptor private let pluginToHost: FileDescriptor - init(path: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws { + init(pluginPath: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws { self.hostToPlugin = try FileDescriptor.open(stdinPath, .readWrite) self.pluginToHost = try FileDescriptor.open(stdoutPath, .readWrite) @@ -32,7 +32,7 @@ struct WasmKitEngine: WasmEngine { self.engine = Engine(configuration: configuration) let store = Store(engine: engine) - let module = try parseWasm(filePath: path) + let module = try parseWasm(filePath: pluginPath) var moduleImports = Imports() let imports = try WASIBridgeToHost( From 0da4f22081333a028eef181e70e1496c6ae89916 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 17 Dec 2024 13:10:04 +0000 Subject: [PATCH 103/105] Remove use of named pipes as engine-specific --- .../swift-plugin-server/WasmEngine.swift | 32 ++----------------- .../swift-plugin-server/WasmKitEngine.swift | 12 ++++--- .../PluginServerTests/WasmEngineTests.swift | 4 +-- 3 files changed, 11 insertions(+), 37 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 599fe8e989d3c..2193a677f70c9 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -29,7 +29,7 @@ import WasmTypes typealias WasmFunction = () throws -> Void protocol WasmEngine { - init(pluginPath: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws + init(pluginPath: FilePath) throws func function(named name: String) throws -> WasmFunction? @@ -43,38 +43,11 @@ typealias DefaultWasmPlugin = WasmEnginePlugin // a WasmPlugin implementation that delegates to a WasmEngine struct WasmEnginePlugin: WasmPlugin { - enum Error: Swift.Error { - case failedToCreateNamedPipe(FilePath) - } - private let pumpFunction: WasmFunction - private let tempDirectory: FilePath - private let stdinPath: FilePath - private let stdoutPath: FilePath let engine: Engine init(path: FilePath) throws { - self.tempDirectory = try createUniqueTemporaryDirectory(basename: "swift_wasm_macros") - self.stdinPath = self.tempDirectory.appending("stdin") - self.stdoutPath = self.tempDirectory.appending("stdout") - - let stdinResult = self.stdinPath.withCString { - mkfifo($0, S_IRUSR | S_IWUSR) - } - - guard stdinResult == 0 else { - throw Error.failedToCreateNamedPipe(self.stdinPath) - } - - let stdoutResult = self.stdoutPath.withCString { - mkfifo($0, S_IRUSR | S_IWUSR) - } - - guard stdoutResult == 0 else { - throw Error.failedToCreateNamedPipe(self.stdoutPath) - } - - self.engine = try Engine(pluginPath: path, stdinPath: self.stdinPath, stdoutPath: self.stdoutPath) + self.engine = try Engine(pluginPath: path) let exportName = "swift_wasm_macro_v1_pump" guard let pump = try engine.function(named: exportName) else { @@ -115,7 +88,6 @@ struct WasmEnginePlugin: WasmPlugin { func shutDown() throws { try self.engine.shutDown() - try _recursiveRemove(at: self.tempDirectory) } } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift index 52ed462960b74..5e6139d33cdbd 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmKitEngine.swift @@ -23,9 +23,11 @@ struct WasmKitEngine: WasmEngine { private let hostToPlugin: FileDescriptor private let pluginToHost: FileDescriptor - init(pluginPath: FilePath, stdinPath: FilePath, stdoutPath: FilePath) throws { - self.hostToPlugin = try FileDescriptor.open(stdinPath, .readWrite) - self.pluginToHost = try FileDescriptor.open(stdoutPath, .readWrite) + init(pluginPath: FilePath) throws { + let hostToPluginPipe = try FileDescriptor.pipe() + let pluginToHostPipe = try FileDescriptor.pipe() + self.hostToPlugin = hostToPluginPipe.writeEnd + self.pluginToHost = pluginToHostPipe.readEnd var configuration = EngineConfiguration() configuration.stackSize = 1 << 20 @@ -36,8 +38,8 @@ struct WasmKitEngine: WasmEngine { var moduleImports = Imports() let imports = try WASIBridgeToHost( - stdin: self.hostToPlugin, - stdout: self.pluginToHost, + stdin: hostToPluginPipe.readEnd, + stdout: pluginToHostPipe.writeEnd, stderr: .standardError ) imports.link(to: &moduleImports, store: store) diff --git a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift index 528d1d87bee48..73ce5451a6949 100644 --- a/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift +++ b/tools/swift-plugin-server/Tests/PluginServerTests/WasmEngineTests.swift @@ -73,8 +73,8 @@ struct WasmEngineTests { try Data(binary).write(to: echoURL) defer { try! FileManager.default.removeItem(at: echoURL) } - let engine = try WasmEnginePlugin(path: FilePath(echoURL.path)) + let wasmKit = try WasmEnginePlugin(path: FilePath(echoURL.path)) let input: [UInt8] = [1,2,3,4,5] - #expect(try engine.handleMessage(input) == input) + #expect(try wasmKit.handleMessage(input) == input) } } From 3991b60873bb800584ac9df8daab658b92d03d9e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 17 Dec 2024 13:17:42 +0000 Subject: [PATCH 104/105] Fix renaming in `tools/swift-plugin-server/CMakeLists.txt` --- tools/swift-plugin-server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index aa68cf32f8124..26430ea6adb69 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -9,7 +9,7 @@ if (SWIFT_BUILD_SWIFT_SYNTAX) FetchContent_MakeAvailable(WasmKit) add_pure_swift_host_tool(swift-plugin-server - Sources/swift-plugin-server/swift-plugin-server.swift + Sources/swift-plugin-server/SwiftPluginServer.swift Sources/swift-plugin-server/WasmEngine.swift Sources/swift-plugin-server/WasmKitEngine.swift Sources/swift-plugin-server/WasmMessageHandler.swift From 636fa0e6b272fd148c11dcee8d92ceab4fb83310 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 17 Dec 2024 15:17:46 +0000 Subject: [PATCH 105/105] Fix Windows compatibility --- .../Sources/swift-plugin-server/WasmEngine.swift | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift index 2193a677f70c9..d5f04c1bc350d 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/WasmEngine.swift @@ -10,18 +10,6 @@ // //===----------------------------------------------------------------------===// -#if canImport(Darwin) -import Darwin -#elseif canImport(Glibc) -import Glibc -#elseif canImport(Musl) -import Musl -#elseif canImport(WASILibc) -import WASILibc -#else -#error("Unsupported platform") -#endif - import SystemPackage import WASI import WasmTypes