Add package-benchmark benchmark suite

kateinoigakukun · kateinoigakukun · commit 66fa6c5f9bd2 · 2024-10-25T05:56:26.000+09:00
diff --git a/Benchmarks/Benchmarks/MacroPlugin/MacroPlugin.swift b/Benchmarks/Benchmarks/MacroPlugin/MacroPlugin.swift
@@ -0,0 +1,198 @@
+import Benchmark
+import WasmKit
+import SystemPackage
+import Foundation
+import WasmKitWASI
+
+let benchmarks = {
+    let macrosDir = FilePath(#filePath)
+        .removingLastComponent()
+        .removingLastComponent()
+        .removingLastComponent()
+        .removingLastComponent()
+        .pushing("Vendor/swift-stringify-macro.wasm/Sources")
+
+    let handshakeMessage = """
+    {
+        "getCapability":{
+            "capability":{
+                "protocolVersion":7
+            }
+        }
+    }
+    """
+    let expandMessages = [
+        "StringifyMacros.wasm": """
+        {
+           "expandFreestandingMacro":{
+              "discriminator":"$s7Example0015mainswift_tzEGbfMX2_6_33_B384B672EB89465DCC67528E23350CF9Ll9stringifyfMf_",
+              "lexicalContext":[
+                 
+              ],
+              "macro":{
+                 "moduleName":"StringifyMacros",
+                 "name":"stringify",
+                 "typeName":"StringifyMacro"
+              },
+              "macroRole":"expression",
+              "syntax":{
+                 "kind":"expression",
+                 "location":{
+                    "column":7,
+                    "fileID":"Example/main.swift",
+                    "fileName":"",
+                    "line":3,
+                    "offset":24
+                 },
+                 "source":"#stringify(1 + 1)"
+              }
+           }
+        }
+        """,
+        "FoundationMacros.wasm": """
+        {
+           "expandFreestandingMacro":{
+              "discriminator":"$s7Example0015mainswift_tzEGbfMX2_6_33_B384B672EB89465DCC67528E23350CF9Ll9stringifyfMf_",
+              "lexicalContext":[
+                 
+              ],
+              "macro":{
+                 "moduleName":"FoundationMacros",
+                 "name":"Expression",
+                 "typeName":"ExpressionMacro"
+              },
+              "macroRole":"expression",
+              "syntax":{
+                 "kind":"expression",
+                 "location":{
+                    "column":7,
+                    "fileID":"Example/main.swift",
+                    "fileName":"",
+                    "line":3,
+                    "offset":24
+                 },
+                 "source":"#Expression<Int, Int> { $0 + 1 }"
+              }
+           }
+        }
+        """,
+        "TestingMacros.wasm": """
+        {
+           "expandFreestandingMacro":{
+              "discriminator":"$s7Example0015mainswift_tzEGbfMX2_6_33_B384B672EB89465DCC67528E23350CF9Ll9stringifyfMf_",
+              "lexicalContext":[
+                 
+              ],
+              "macro":{
+                 "moduleName":"TestingMacros",
+                 "name":"expect",
+                 "typeName":"ExpectMacro"
+              },
+              "macroRole":"expression",
+              "syntax":{
+                 "kind":"expression",
+                 "location":{
+                    "column":7,
+                    "fileID":"Example/main.swift",
+                    "fileName":"",
+                    "line":3,
+                    "offset":24
+                 },
+                 "source":"#expect(1 == 2)"
+              }
+           }
+        }
+        """
+    ]
+    
+    for file in try! FileManager.default.contentsOfDirectory(
+        atPath: macrosDir.string
+    ) {
+        guard file.hasSuffix(".wasm") else { continue }
+
+        struct Setup {
+            let hostToPlugin: FileDescriptor
+            let pluginToHost: FileDescriptor
+            let pump: Function
+            let expandMessage: String
+
+            init(filePath: FilePath, expandMessage: String) throws {
+                let engine = Engine()
+                let store = Store(engine: engine)
+                let module = try parseWasm(filePath: filePath)
+
+                let hostToPluginPipes = try FileDescriptor.pipe()
+                let pluginToHostPipes = try FileDescriptor.pipe()
+                let bridge = try WASIBridgeToHost(
+                  stdin: hostToPluginPipes.readEnd,
+                  stdout: pluginToHostPipes.writeEnd,
+                  stderr: .standardError
+                )
+                var imports = Imports()
+                bridge.link(to: &imports, store: store)
+                let instance = try module.instantiate(store: store, imports: imports)
+                try instance.exports[function: "_start"]!()
+                let pump = instance.exports[function: "swift_wasm_macro_v1_pump"]!
+
+                self.hostToPlugin = hostToPluginPipes.writeEnd
+                self.pluginToHost = pluginToHostPipes.readEnd
+                self.pump = pump
+                self.expandMessage = expandMessage
+            }
+
+            func writeMessage(_ message: String) throws {
+                let bytes = message.utf8
+                try withUnsafeBytes(of: UInt64(bytes.count).littleEndian) {
+                  _ = try hostToPlugin.writeAll($0)
+                }
+                try hostToPlugin.writeAll(bytes)
+            }
+            func readMessage() throws -> [UInt8] {
+                let lengthRaw = try withUnsafeTemporaryAllocation(of: UInt8.self, capacity: 8) { buffer in
+                    let lengthCount = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer))
+                    guard lengthCount == 8 else { fatalError() }
+                    return buffer.withMemoryRebound(to: UInt64.self, \.baseAddress!.pointee)
+                }
+                let length = Int(UInt64(littleEndian: lengthRaw))
+                return try [UInt8](unsafeUninitializedCapacity: length) { buffer, size in
+                    let received = try pluginToHost.read(into: UnsafeMutableRawBufferPointer(buffer))
+                    guard received == length else {
+                        fatalError()
+                    }
+                    size = received
+                }
+            }
+
+            func tick() throws {
+                try writeMessage(expandMessage)
+                try pump()
+                _ = try readMessage()
+            }
+        }
+
+        guard let expandMessage = expandMessages[file] else {
+            fatalError("Expand message definition not found for \(file)")
+        }
+
+        Benchmark("Startup \(file)") { benchmark in
+            let setup = try Setup(filePath: macrosDir.appending(file), expandMessage: expandMessage)
+            try setup.writeMessage(handshakeMessage)
+            try setup.tick()
+        }
+
+        Benchmark("Expand \(file)") { benchmark, setup in
+            try setup.tick()
+        } setup: { () -> Setup in
+            let setup = try Setup(
+                filePath: macrosDir.appending(file),
+                expandMessage: expandMessage
+            )
+            try setup.writeMessage(handshakeMessage)
+
+            // Warmup
+            try setup.tick()
+
+            return setup
+        }
+    }
+}
diff --git a/Benchmarks/Benchmarks/MicroBench/MicroBench.swift b/Benchmarks/Benchmarks/MicroBench/MicroBench.swift
@@ -0,0 +1,18 @@
+import Benchmark
+import WasmKit
+import WAT
+
+let benchmarks = {
+    Benchmark("empty instantiation") { benchmark in
+        for _ in benchmark.scaledIterations {
+            let engine = Engine()
+            let store = Store(engine: engine)
+            let module = try parseWasm(bytes: wat2wasm("""
+            (module
+                (func (export "_start"))
+            )
+            """))
+            _ = try module.instantiate(store: store)
+        }
+    }
+}
diff --git a/Benchmarks/Benchmarks/WishYouWereFast/WishYouWereFast.swift b/Benchmarks/Benchmarks/WishYouWereFast/WishYouWereFast.swift
@@ -0,0 +1,38 @@
+import Benchmark
+import WasmKit
+import WasmKitWASI
+import Foundation
+import SystemPackage
+
+let benchmarks = {
+    let wishYouWereFast = URL(fileURLWithPath: #filePath)
+        .deletingLastPathComponent()
+        .deletingLastPathComponent()
+        .deletingLastPathComponent()
+        .deletingLastPathComponent()
+        .appendingPathComponent("Vendor")
+        .appendingPathComponent("wish-you-were-fast")
+        .appendingPathComponent("wasm")
+        .appendingPathComponent("suites")
+        .appendingPathComponent("libsodium")
+
+    let devNull = try! FileDescriptor.open("/dev/null", .readWrite)
+
+    for file in try! FileManager.default.contentsOfDirectory(
+        atPath: wishYouWereFast.path
+    ) {
+        guard file.hasSuffix(".wasm") else { continue }
+        Benchmark("\(file)") { benchmark in
+            let engine = Engine()
+            let store = Store(engine: engine)
+            let module = try parseWasm(
+                filePath: FilePath(wishYouWereFast.appendingPathComponent(file).path)
+            )
+            let wasi = try WASIBridgeToHost(stdout: devNull, stderr: devNull)
+            var imports = Imports()
+            wasi.link(to: &imports, store: store)
+            let instance = try module.instantiate(store: store, imports: imports)
+            _ = try wasi.start(instance)
+        }
+    }
+}
diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift
@@ -0,0 +1,61 @@
+// swift-tools-version: 5.8
+
+import PackageDescription
+
+let package = Package(
+    name: "Benchmarks",
+    platforms: [.macOS(.v13)],
+    dependencies: [
+        .package(path: "../"),
+        .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")),
+    ]
+)
+
+// Benchmark of WishYouWereFast
+package.targets += [
+    .executableTarget(
+        name: "WishYouWereFast",
+        dependencies: [
+            .product(name: "WasmKit", package: "WasmKit"),
+            .product(name: "WasmKitWASI", package: "WasmKit"),
+            .product(name: "Benchmark", package: "package-benchmark"),
+        ],
+        path: "Benchmarks/WishYouWereFast",
+        plugins: [
+            .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+        ]
+    ),
+]
+
+// Benchmark of MicroBench
+package.targets += [
+    .executableTarget(
+        name: "MicroBench",
+        dependencies: [
+            .product(name: "WAT", package: "WasmKit"),
+            .product(name: "WasmKit", package: "WasmKit"),
+            .product(name: "WasmKitWASI", package: "WasmKit"),
+            .product(name: "Benchmark", package: "package-benchmark"),
+        ],
+        path: "Benchmarks/MicroBench",
+        plugins: [
+            .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+        ]
+    ),
+]
+
+// Benchmark of MacroPlugin
+package.targets += [
+    .executableTarget(
+        name: "MacroPlugin",
+        dependencies: [
+            .product(name: "WasmKit", package: "WasmKit"),
+            .product(name: "WasmKitWASI", package: "WasmKit"),
+            .product(name: "Benchmark", package: "package-benchmark"),
+        ],
+        path: "Benchmarks/MacroPlugin",
+        plugins: [
+            .plugin(name: "BenchmarkPlugin", package: "package-benchmark")
+        ]
+    ),
+]
diff --git a/Vendor/checkout-dependency b/Vendor/checkout-dependency
@@ -34,10 +34,16 @@ class CheckoutDependency:
         print(f"Checking out '{dependency_name}' to {dependency['revision']}")
         subprocess.run(["git", "-C", dependency_path, "checkout", dependency["revision"]], check=True)
 
+    def checkout_category(self, category):
+        for dependency_name, dependency in self.dependencies.items():
+            if category in dependency.get("categories", []):
+                self.checkout(dependency_name)
+
     def checkout_all(self):
         for dependency_name in self.dependencies.keys():
             self.checkout(dependency_name)
 
+
 def main():
     import argparse
     dependencies_file = os.path.join(os.path.dirname(__file__), "dependencies.json")
@@ -46,13 +52,22 @@ def main():
     parser = argparse.ArgumentParser(description="Checkout dependency repositories")
     available_dependencies = ", ".join(checkout_dependency.dependencies.keys())
     parser.add_argument("names", nargs="*", help=f"Available dependencies: {available_dependencies}")
+    parser.add_argument("--all", action="store_true", help="Checkout all dependencies")
+    parser.add_argument("--category", action="append", dest="categories",
+                        default=["default"],
+                        help="Checkout dependencies by category")
+
     args = parser.parse_args()
 
     if args.names:
         for dependency_name in args.names:
             checkout_dependency.checkout(dependency_name)
-    else:
+    elif args.all:
         checkout_dependency.checkout_all()
+    elif args.categories:
+        for category in args.categories:
+            checkout_dependency.checkout_category(category)
+
 
 if __name__ == "__main__":
     main()
diff --git a/Vendor/dependencies.json b/Vendor/dependencies.json
