Benchmark Rust against pinned C# reference (#105)

benchmarks/csharp-reference/MetaCrate.ReferenceBenchmarks.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <ReferenceRoot Condition="'$(ReferenceRoot)' == ''">$(MSBuildThisFileDirectory)../../../libremetaverse</ReferenceRoot>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(ReferenceRoot)/LibreMetaverse/LibreMetaverse.csproj" />
+    <ProjectReference Include="$(ReferenceRoot)/LibreMetaverse.Rendering.Simple/LibreMetaverse.Rendering.Simple.csproj" />
+    <ProjectReference Include="$(ReferenceRoot)/LibreMetaverse.Rendering.MeshFoundry/LibreMetaverse.Rendering.MeshFoundry.csproj" />
+  </ItemGroup>
+</Project>

benchmarks/csharp-reference/Program.cs

@@ -0,0 +1,346 @@
+using System.Diagnostics;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Security.Cryptography;
+using System.Text.Json;
+using LibreMetaverse;
+using LibreMetaverse.Assets;
+using LibreMetaverse.Imaging;
+using LibreMetaverse.Messages.Linden;
+using LibreMetaverse.Packets;
+using LibreMetaverse.Rendering;
+using LibreMetaverse.StructuredData;
+using IOPath = System.IO.Path;
+
+const int Schema = 1;
+const int Runs = 7;
+const string ExpectedReference = "2aa70bb68513b39795da5d13c88f31b86e85a3ba";
+
+if (args.Length == 0 || args[0] != "run")
+    throw new ArgumentException("usage: dotnet run ... -- run --fixture-root PATH --output FILE");
+string root = Value("--fixture-root") ?? "benchmarks/fixtures";
+string output = Value("--output") ?? "benchmarks/results/csharp-linux-x86_64.json";
+string referenceRoot = Value("--reference-root") ?? throw new ArgumentException("missing --reference-root");
+VerifyReferenceCommit(referenceRoot);
+Fixture fixture = JsonSerializer.Deserialize<Fixture>(File.ReadAllBytes(IOPath.Combine(root, "workloads.json")), Options())
+    ?? throw new InvalidDataException("empty fixture");
+if (fixture.Schema != 1 || fixture.ReferenceCommit != ExpectedReference || fixture.WarmupDivisor == 0)
+    throw new InvalidDataException("fixture schema or reference commit does not match policy");
+Dictionary<string, string> hashes = VerifyManifest(root);
+var context = new Context(fixture, root);
+var results = new List<BenchmarkResult>();
+foreach (Workload workload in Workloads())
+{
+    int iterations = workload.Iterations(fixture);
+    Metrics cold = Measure(context, workload.Run, 1);
+    GC.KeepAlive(workload.Run(context, Math.Max(1, iterations / fixture.WarmupDivisor)));
+    var samples = new List<Metrics>();
+    for (int index = 0; index < Runs; index++) samples.Add(Measure(context, workload.Run, iterations));
+    Metrics median = samples.OrderBy(value => value.ElapsedNs).ElementAt(Runs / 2);
+    results.Add(new BenchmarkResult(workload.Name, workload.Category, "operation", workload.Fixture, cold, samples, median));
+}
+var report = new Report(
+    Schema,
+    "dotnet-reference",
+    ExpectedReference,
+    hashes,
+    new EnvironmentInfo(
+        OperatingSystem.IsLinux() ? "linux" : OperatingSystem.IsWindows() ? "windows" : OperatingSystem.IsMacOS() ? "macos" : RuntimeInformation.OSDescription,
+        RuntimeInformation.ProcessArchitecture switch { Architecture.X64 => "x86_64", Architecture.Arm64 => "aarch64", Architecture.X86 => "x86", Architecture.Arm => "arm", _ => RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant() },
+        Environment.ProcessorCount,
+        RuntimeInformation.FrameworkDescription,
+        "Release net8.0"),
+    results);
+Directory.CreateDirectory(IOPath.GetDirectoryName(IOPath.GetFullPath(output))!);
+File.WriteAllText(output, JsonSerializer.Serialize(report, Options()) + "\n");
+
+string? Value(string key)
+{
+    int index = Array.IndexOf(args, key);
+    return index >= 0 && index + 1 < args.Length ? args[index + 1] : null;
+}
+
+static JsonSerializerOptions Options() => new()
+{
+    PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+    WriteIndented = true
+};
+
+static Dictionary<string, string> VerifyManifest(string root)
+{
+    var expected = JsonSerializer.Deserialize<Dictionary<string, string>>(
+        File.ReadAllBytes(IOPath.Combine(root, "manifest.json")), Options())
+        ?? throw new InvalidDataException("empty manifest");
+    foreach ((string name, string hash) in expected)
+    {
+        string actual = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(IOPath.Combine(root, name)))).ToLowerInvariant();
+        if (actual != hash) throw new InvalidDataException($"fixture hash mismatch for {name}");
+    }
+    return expected;
+}
+
+void VerifyReferenceCommit(string referenceRoot)
+{
+    var start = new ProcessStartInfo("git")
+    {
+        RedirectStandardOutput = true,
+        UseShellExecute = false
+    };
+    start.ArgumentList.Add("-C");
+    start.ArgumentList.Add(referenceRoot);
+    start.ArgumentList.Add("rev-parse");
+    start.ArgumentList.Add("HEAD");
+    var process = Process.Start(start) ?? throw new InvalidOperationException("could not start git");
+    string commit = process.StandardOutput.ReadToEnd().Trim();
+    process.WaitForExit();
+    if (process.ExitCode != 0 || commit != ExpectedReference)
+        throw new InvalidDataException($"reference checkout must be pinned to {ExpectedReference}, got {commit}");
+}
+
+static Metrics Measure(Context context, Func<Context, int, ulong> function, int iterations)
+{
+    long allocatedBefore = GC.GetAllocatedBytesForCurrentThread();
+    long heapBefore = GC.GetTotalMemory(false);
+    long started = Stopwatch.GetTimestamp();
+    ulong checksum = function(context, iterations);
+    long elapsed = Stopwatch.GetTimestamp() - started;
+    long heapAfter = GC.GetTotalMemory(false);
+    long allocated = GC.GetAllocatedBytesForCurrentThread() - allocatedBefore;
+    GC.KeepAlive(checksum);
+    long elapsedNs = checked((long)(elapsed * (1_000_000_000.0 / Stopwatch.Frequency)));
+    double seconds = elapsedNs / 1_000_000_000.0;
+    return new Metrics(iterations, elapsedNs, (double)elapsedNs / iterations, iterations / seconds, null, allocated, heapAfter - heapBefore);
+}
+
+static Workload[] Workloads() =>
+[
+    new("uuid_math", "uuid/math", "workloads.json", f => f.UuidMathIterations, UuidMath),
+    new("llsd_xml", "LLSD", "workloads.json", f => f.LlsdIterations, LlsdXml),
+    new("llsd_json", "LLSD", "workloads.json", f => f.LlsdIterations, LlsdJson),
+    new("llsd_binary", "LLSD", "workloads.json", f => f.LlsdIterations, LlsdBinary),
+    new("llsd_notation", "LLSD", "workloads.json", f => f.LlsdIterations, LlsdNotation),
+    new("llsd_protobuf", "LLSD", "workloads.json", f => f.LlsdIterations, LlsdProtobuf),
+    new("packet_codec", "packet codec", "agent_pause.packet", f => f.PacketIterations, PacketCodec),
+    new("asset_decode", "asset decode", "notecard.txt", f => f.AssetIterations, AssetDecode),
+    new("image_decode", "image decode", "image.tga", f => f.ImageIterations, ImageDecode),
+    new("mesh_decode", "mesh decode", "mesh_asset.bin", f => f.MeshIterations, MeshDecode),
+    new("inventory_update", "inventory update", "workloads.json", f => f.InventoryIterations, InventoryUpdate),
+    new("object_update", "object update", "workloads.json", f => f.ObjectIterations, ObjectUpdate),
+    new("rendering", "rendering", "workloads.json", f => f.RenderingIterations, Rendering),
+    new("client_throughput", "client throughput", "workloads.json", f => f.ClientIterations, ClientThroughput)
+];
+
+static ulong UuidMath(Context context, int iterations)
+{
+    Quaternion rotation = Quaternion.CreateFromEulers(0.25f, 0.5f, 0.75f);
+    Matrix4 matrix = Matrix4.CreateTranslation(new Vector3(10, 20, 30));
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        var parsed = new UUID(context.Fixture.Uuid);
+        matrix = Matrix4.Transform(matrix, rotation);
+        checksum ^= parsed.GetULong() ^ BitConverter.SingleToUInt32Bits(matrix.M41) ^ (ulong)index;
+    }
+    return checksum;
+}
+
+static ulong LlsdXml(Context context, int iterations) => LlsdBytes(context, iterations, OSDParser.SerializeLLSDXmlBytes, OSDParser.DeserializeLLSDXml);
+static ulong LlsdBinary(Context context, int iterations) => LlsdBytes(context, iterations, OSDParser.SerializeLLSDBinary, OSDParser.DeserializeLLSDBinary);
+static ulong LlsdProtobuf(Context context, int iterations) => LlsdBytes(context, iterations, value => OSDParser.SerializeLLSDProtobuf(value, true), OSDParser.DeserializeLLSDProtobuf);
+
+static ulong LlsdBytes(Context context, int iterations, Func<OSD, byte[]> serialize, Func<byte[], OSD> deserialize)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        byte[] encoded = serialize(context.Llsd);
+        OSD decoded = deserialize(encoded);
+        checksum ^= (ulong)encoded.Length ^ (decoded.AsBoolean() ? 1UL : 0UL);
+    }
+    return checksum;
+}
+
+static ulong LlsdJson(Context context, int iterations)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        string encoded = OSDParser.SerializeJsonString(context.Llsd);
+        OSD decoded = OSDParser.DeserializeJson(encoded);
+        checksum ^= (ulong)encoded.Length ^ (decoded.AsBoolean() ? 1UL : 0UL);
+    }
+    return checksum;
+}
+
+static ulong LlsdNotation(Context context, int iterations)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        string encoded = OSDParser.SerializeLLSDNotation(context.Llsd);
+        OSD decoded = OSDParser.DeserializeLLSDNotation(encoded);
+        checksum ^= (ulong)encoded.Length ^ (decoded.AsBoolean() ? 1UL : 0UL);
+    }
+    return checksum;
+}
+
+static ulong PacketCodec(Context context, int iterations)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        int position = 0;
+        var packet = new AgentPausePacket(context.Packet, ref position);
+        byte[] encoded = packet.ToBytes();
+        checksum ^= (ulong)encoded.Length ^ packet.AgentData.SerialNum;
+    }
+    return checksum;
+}
+
+static ulong AssetDecode(Context context, int iterations)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        var asset = new AssetNotecard(context.Uuid, context.Notecard);
+        if (!asset.Decode()) throw new InvalidDataException("notecard decode returned false");
+        checksum ^= (ulong)asset.BodyText.Length;
+    }
+    return checksum;
+}
+
+static ulong ImageDecode(Context context, int iterations)
+{
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        using var stream = new MemoryStream(context.Image, false);
+        ManagedImage image = Targa.DecodeToManagedImage(stream);
+        checksum ^= (ulong)image.Red.Length;
+    }
+    return checksum;
+}
+
+static ulong MeshDecode(Context context, int iterations)
+{
+    var renderer = new MeshFoundry();
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        FacetedMesh mesh = renderer.GenerateFacetedMeshMesh(new Primitive(), context.Mesh, DetailLevel.Highest)
+            ?? throw new InvalidDataException("mesh fixture returned no geometry");
+        checksum ^= (ulong)mesh.Faces.Sum(face => face.Vertices.Count + face.Indices.Count);
+    }
+    return checksum;
+}
+
+static ulong InventoryUpdate(Context context, int iterations)
+{
+    var store = new Inventory(new GridClient(), context.Uuid);
+    for (int index = 0; index < iterations; index++)
+    {
+        var item = new InventoryItem(new UUID((ulong)(index % context.Fixture.InventoryItems + 1))) { Name = $"item-{index}" };
+        store.UpdateNodeFor(item);
+    }
+    return (ulong)store.Count;
+}
+
+static ulong ObjectUpdate(Context context, int iterations)
+{
+    Primitive[] objects = Enumerable.Range(0, context.Fixture.ObjectCount).Select(index => new Primitive { LocalID = (uint)index }).ToArray();
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        Primitive value = objects[index % objects.Length];
+        value.Position = new Vector3(index, index & 255, 21);
+        value.Rotation = Quaternion.CreateFromAxisAngle(Vector3.UnitZ, (index & 255) / 255.0f);
+        checksum ^= value.LocalID ^ BitConverter.SingleToUInt32Bits(value.Position.X);
+    }
+    return checksum;
+}
+
+static ulong Rendering(Context context, int iterations)
+{
+    var renderer = new SimpleRenderer();
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        FacetedMesh mesh = renderer.GenerateFacetedMesh(BoxPrimitive(), DetailLevel.High);
+        checksum ^= (ulong)mesh.Faces.Sum(face => face.Indices.Count);
+    }
+    return checksum;
+}
+
+static Primitive BoxPrimitive() => new()
+{
+    PrimData = new Primitive.ConstructionData
+    {
+        PCode = PCode.Prim, PathCurve = PathCurve.Line, ProfileCurve = ProfileCurve.Square,
+        PathScaleX = 1, PathScaleY = 1, PathBegin = 0, PathEnd = 1,
+        ProfileBegin = 0, ProfileEnd = 1, ProfileHollow = 0,
+        ProfileHole = HoleType.Same, PathRevolutions = 1
+    }
+};
+
+static ulong ClientThroughput(Context context, int iterations)
+{
+    var client = new GridClient();
+    ulong checksum = 0;
+    for (int index = 0; index < iterations; index++)
+    {
+        for (int batch = 0; batch < context.Fixture.ClientBatch; batch++)
+        {
+            var source = new RemoteParcelRequestReply { ParcelID = context.Uuid };
+            OSDMap map = source.Serialize();
+            var target = new RemoteParcelRequestReply();
+            target.Deserialize(map);
+            checksum ^= target.ParcelID.GetULong() ^ (ulong)index;
+        }
+    }
+    GC.KeepAlive(client);
+    return checksum;
+}
+
+sealed class Context
+{
+    public Context(Fixture fixture, string root)
+    {
+        Fixture = fixture;
+        Uuid = new UUID(fixture.Uuid);
+        Llsd = new OSDMap
+        {
+            ["agent_id"] = OSD.FromUUID(Uuid), ["name"] = OSD.FromString("MetaCrate benchmark"),
+            ["active"] = OSD.FromBoolean(true), ["score"] = OSD.FromReal(42.25),
+            ["items"] = IntegerArray(fixture.LlsdArrayLength)
+        };
+        Packet = File.ReadAllBytes(IOPath.Combine(root, "agent_pause.packet"));
+        Notecard = File.ReadAllBytes(IOPath.Combine(root, "notecard.txt"));
+        Image = File.ReadAllBytes(IOPath.Combine(root, "image.tga"));
+        Mesh = File.ReadAllBytes(IOPath.Combine(root, "mesh_asset.bin"));
+    }
+    private static OSDArray IntegerArray(int count)
+    {
+        var values = new OSDArray(count);
+        for (int index = 0; index < count; index++) values.Add(OSD.FromInteger(index));
+        return values;
+    }
+    public Fixture Fixture { get; }
+    public UUID Uuid { get; }
+    public OSD Llsd { get; }
+    public byte[] Packet { get; }
+    public byte[] Notecard { get; }
+    public byte[] Image { get; }
+    public byte[] Mesh { get; }
+}
+
+sealed record Fixture(
+    int Schema, string ReferenceCommit, string Uuid, int UuidMathIterations, int LlsdIterations,
+    int PacketIterations, int AssetIterations, int ImageIterations, int MeshIterations,
+    int InventoryIterations, int ObjectIterations, int RenderingIterations, int ClientIterations,
+    int WarmupDivisor, int LlsdArrayLength, int PacketBlocks, int ImageWidth, int ImageHeight,
+    int MeshSide, int InventoryItems, int ObjectCount, int ClientBatch);
+sealed record Metrics(int Iterations, long ElapsedNs, double LatencyNs, double ThroughputPerSecond, long? Allocations, long BytesAllocated, long RetainedBytes);
+sealed record BenchmarkResult(string Name, string Category, string Unit, string Fixture, Metrics Cold, List<Metrics> WarmSamples, Metrics WarmMedian);
+sealed record EnvironmentInfo(string Os, string Arch, int LogicalCpus, string Toolchain, string Profile);
+sealed record Report(int Schema, string Runtime, string ReferenceCommit, Dictionary<string, string> FixtureHashes, EnvironmentInfo Environment, List<BenchmarkResult> Results);
+sealed record Workload(string Name, string Category, string Fixture, Func<Fixture, int> Iterations, Func<Context, int, ulong> Run);