From 3a429462e6c9b134fbf821c7043062ae9221d1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6llner?= Date: Sat, 8 Feb 2020 18:36:21 +0100 Subject: [PATCH] added benchmarking suite --- Benchmarking/Benchmark/Benchmark.csproj | 24 ++++++++++ Benchmarking/Benchmark/CapnpBenchmark.cs | 46 +++++++++++++++++++ Benchmarking/Benchmark/GrpcBenchmark.cs | 42 +++++++++++++++++ Benchmarking/Benchmark/Program.cs | 18 ++++++++ Benchmarking/Benchmark/Protos/Echo.capnp | 5 ++ Benchmarking/Benchmark/Protos/Echo.proto | 13 ++++++ Benchmarking/CapnpBench.sln | 37 +++++++++++++++ .../EchoServiceCapnp/EchoServiceCapnp.csproj | 13 ++++++ Benchmarking/EchoServiceCapnp/Program.cs | 20 ++++++++ .../EchoServiceCapnp/Protos/Echo.capnp | 5 ++ .../Services/CapnpEchoService.cs | 20 ++++++++ .../EchoServiceGrpc/EchoServiceGrpc.csproj | 15 ++++++ Benchmarking/EchoServiceGrpc/Program.cs | 27 +++++++++++ .../EchoServiceGrpc/Protos/Echo.proto | 13 ++++++ .../Services/GrpcEchoService.cs | 26 +++++++++++ Benchmarking/EchoServiceGrpc/Startup.cs | 43 +++++++++++++++++ .../appsettings.Development.json | 10 ++++ Benchmarking/EchoServiceGrpc/appsettings.json | 15 ++++++ 18 files changed, 392 insertions(+) create mode 100644 Benchmarking/Benchmark/Benchmark.csproj create mode 100644 Benchmarking/Benchmark/CapnpBenchmark.cs create mode 100644 Benchmarking/Benchmark/GrpcBenchmark.cs create mode 100644 Benchmarking/Benchmark/Program.cs create mode 100644 Benchmarking/Benchmark/Protos/Echo.capnp create mode 100644 Benchmarking/Benchmark/Protos/Echo.proto create mode 100644 Benchmarking/CapnpBench.sln create mode 100644 Benchmarking/EchoServiceCapnp/EchoServiceCapnp.csproj create mode 100644 Benchmarking/EchoServiceCapnp/Program.cs create mode 100644 Benchmarking/EchoServiceCapnp/Protos/Echo.capnp create mode 100644 Benchmarking/EchoServiceCapnp/Services/CapnpEchoService.cs create mode 100644 Benchmarking/EchoServiceGrpc/EchoServiceGrpc.csproj create mode 100644 Benchmarking/EchoServiceGrpc/Program.cs create mode 100644 Benchmarking/EchoServiceGrpc/Protos/Echo.proto create mode 100644 Benchmarking/EchoServiceGrpc/Services/GrpcEchoService.cs create mode 100644 Benchmarking/EchoServiceGrpc/Startup.cs create mode 100644 Benchmarking/EchoServiceGrpc/appsettings.Development.json create mode 100644 Benchmarking/EchoServiceGrpc/appsettings.json diff --git a/Benchmarking/Benchmark/Benchmark.csproj b/Benchmarking/Benchmark/Benchmark.csproj new file mode 100644 index 0000000..805886e --- /dev/null +++ b/Benchmarking/Benchmark/Benchmark.csproj @@ -0,0 +1,24 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/Benchmarking/Benchmark/CapnpBenchmark.cs b/Benchmarking/Benchmark/CapnpBenchmark.cs new file mode 100644 index 0000000..63f3f1d --- /dev/null +++ b/Benchmarking/Benchmark/CapnpBenchmark.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Capnp.Rpc; +using CapnpGen; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Benchmark +{ + public class CapnpBenchmark + { + [Params(20, 200, 2000, 20000, 200000, 2000000)] + public int PayloadBytes; + + TcpRpcClient _client; + IEchoer _echoer; + byte[] _payload; + + [GlobalSetup] + public void Setup() + { + _client = new TcpRpcClient("localhost", 5002); + _client.WhenConnected.Wait(); + _echoer = _client.GetMain(); + _payload = new byte[PayloadBytes]; + new Random().NextBytes(_payload); + } + + [GlobalCleanup] + public void Cleanup() + { + _echoer.Dispose(); + _client.Dispose(); + } + + [Benchmark] + public void Echo() + { + var t = _echoer.Echo(_payload); + t.Wait(); + if (t.Result?.Count != _payload.Length) + throw new InvalidOperationException("Echo server malfunction"); + } + } +} diff --git a/Benchmarking/Benchmark/GrpcBenchmark.cs b/Benchmarking/Benchmark/GrpcBenchmark.cs new file mode 100644 index 0000000..573d7c2 --- /dev/null +++ b/Benchmarking/Benchmark/GrpcBenchmark.cs @@ -0,0 +1,42 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Jobs; +using Grpc.Net.Client; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Benchmark +{ + public class GrpcBenchmark + { + [Params(20, 200, 2000, 20000, 200000, 2000000)] + public int PayloadBytes; + + GrpcChannel _channel; + Echoer.EchoerClient _echoer; + byte[] _payload; + + [GlobalSetup] + public void Setup() + { + _channel = GrpcChannel.ForAddress("https://localhost:5001"); + _echoer = new Echoer.EchoerClient(_channel); + _payload = new byte[PayloadBytes]; + new Random().NextBytes(_payload); + } + + [GlobalCleanup] + public void Teardown() + { + _channel.Dispose(); + } + + [Benchmark] + public void Echo() + { + var reply = _echoer.Echo(new EchoRequest { Payload = Google.Protobuf.ByteString.CopyFrom(_payload) }); + if (reply?.Payload?.Length != _payload.Length) + throw new InvalidOperationException("Echo server malfunction"); + } + } +} diff --git a/Benchmarking/Benchmark/Program.cs b/Benchmarking/Benchmark/Program.cs new file mode 100644 index 0000000..16a0924 --- /dev/null +++ b/Benchmarking/Benchmark/Program.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Running; +using Capnp.Rpc; +using Grpc.Net.Client; +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace Benchmark +{ + class Program + { + static void Main(string[] args) + { + BenchmarkRunner.Run(); + BenchmarkRunner.Run(); + } + } +} diff --git a/Benchmarking/Benchmark/Protos/Echo.capnp b/Benchmarking/Benchmark/Protos/Echo.capnp new file mode 100644 index 0000000..2af55d5 --- /dev/null +++ b/Benchmarking/Benchmark/Protos/Echo.capnp @@ -0,0 +1,5 @@ +@0x8c309c720de8cf7c; + +interface Echoer { + echo @0 (input : Data) -> (output : Data); +} diff --git a/Benchmarking/Benchmark/Protos/Echo.proto b/Benchmarking/Benchmark/Protos/Echo.proto new file mode 100644 index 0000000..ddbbaea --- /dev/null +++ b/Benchmarking/Benchmark/Protos/Echo.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +service Echoer { + rpc Echo (EchoRequest) returns (EchoReply); +} + +message EchoRequest { + bytes payload = 1; +} + +message EchoReply { + bytes payload = 1; +} \ No newline at end of file diff --git a/Benchmarking/CapnpBench.sln b/Benchmarking/CapnpBench.sln new file mode 100644 index 0000000..061559d --- /dev/null +++ b/Benchmarking/CapnpBench.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29728.190 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoServiceGrpc", "EchoServiceGrpc\EchoServiceGrpc.csproj", "{D59C7B71-3887-426B-A636-2DBDA0549817}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "Benchmark\Benchmark.csproj", "{7F7580CA-CCF0-4650-87BF-502D51A8F435}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoServiceCapnp", "EchoServiceCapnp\EchoServiceCapnp.csproj", "{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D59C7B71-3887-426B-A636-2DBDA0549817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D59C7B71-3887-426B-A636-2DBDA0549817}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D59C7B71-3887-426B-A636-2DBDA0549817}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D59C7B71-3887-426B-A636-2DBDA0549817}.Release|Any CPU.Build.0 = Release|Any CPU + {7F7580CA-CCF0-4650-87BF-502D51A8F435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7F7580CA-CCF0-4650-87BF-502D51A8F435}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7F7580CA-CCF0-4650-87BF-502D51A8F435}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7F7580CA-CCF0-4650-87BF-502D51A8F435}.Release|Any CPU.Build.0 = Release|Any CPU + {309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Debug|Any CPU.Build.0 = Debug|Any CPU + {309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Release|Any CPU.ActiveCfg = Release|Any CPU + {309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A6702BC8-8CC0-4BDA-8BD5-D5D268291E93} + EndGlobalSection +EndGlobal diff --git a/Benchmarking/EchoServiceCapnp/EchoServiceCapnp.csproj b/Benchmarking/EchoServiceCapnp/EchoServiceCapnp.csproj new file mode 100644 index 0000000..f50d2fa --- /dev/null +++ b/Benchmarking/EchoServiceCapnp/EchoServiceCapnp.csproj @@ -0,0 +1,13 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + diff --git a/Benchmarking/EchoServiceCapnp/Program.cs b/Benchmarking/EchoServiceCapnp/Program.cs new file mode 100644 index 0000000..fb58c04 --- /dev/null +++ b/Benchmarking/EchoServiceCapnp/Program.cs @@ -0,0 +1,20 @@ +using Capnp.Rpc; +using EchoServiceCapnp.Services; +using System; +using System.Net; + +namespace EchoServiceCapnp +{ + class Program + { + static void Main(string[] args) + { + using (var server = new TcpRpcServer(IPAddress.Any, 5002)) + { + server.Main = new CapnpEchoService(); + Console.WriteLine("Press RETURN to stop listening"); + Console.ReadLine(); + } + } + } +} diff --git a/Benchmarking/EchoServiceCapnp/Protos/Echo.capnp b/Benchmarking/EchoServiceCapnp/Protos/Echo.capnp new file mode 100644 index 0000000..2af55d5 --- /dev/null +++ b/Benchmarking/EchoServiceCapnp/Protos/Echo.capnp @@ -0,0 +1,5 @@ +@0x8c309c720de8cf7c; + +interface Echoer { + echo @0 (input : Data) -> (output : Data); +} diff --git a/Benchmarking/EchoServiceCapnp/Services/CapnpEchoService.cs b/Benchmarking/EchoServiceCapnp/Services/CapnpEchoService.cs new file mode 100644 index 0000000..60e7afb --- /dev/null +++ b/Benchmarking/EchoServiceCapnp/Services/CapnpEchoService.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace EchoServiceCapnp.Services +{ + public class CapnpEchoService : CapnpGen.IEchoer + { + public void Dispose() + { + } + + public Task> Echo(IReadOnlyList input, CancellationToken cancellationToken_ = default) + { + return Task.FromResult(input); + } + } +} diff --git a/Benchmarking/EchoServiceGrpc/EchoServiceGrpc.csproj b/Benchmarking/EchoServiceGrpc/EchoServiceGrpc.csproj new file mode 100644 index 0000000..861d83c --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/EchoServiceGrpc.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp3.1 + + + + + + + + + + + diff --git a/Benchmarking/EchoServiceGrpc/Program.cs b/Benchmarking/EchoServiceGrpc/Program.cs new file mode 100644 index 0000000..f24aee5 --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/Program.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; + +namespace EchoService +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + // Additional configuration is required to successfully run gRPC on macOS. + // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682 + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/Benchmarking/EchoServiceGrpc/Protos/Echo.proto b/Benchmarking/EchoServiceGrpc/Protos/Echo.proto new file mode 100644 index 0000000..b1b0c0d --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/Protos/Echo.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +service Echoer { + rpc Echo (EchoRequest) returns (EchoReply); +} + +message EchoRequest { + bytes payload = 1; +} + +message EchoReply { + bytes payload = 1; +} \ No newline at end of file diff --git a/Benchmarking/EchoServiceGrpc/Services/GrpcEchoService.cs b/Benchmarking/EchoServiceGrpc/Services/GrpcEchoService.cs new file mode 100644 index 0000000..71a7ef9 --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/Services/GrpcEchoService.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Grpc.Core; +using Microsoft.Extensions.Logging; + +namespace EchoService +{ + public class GrpcEchoService : Echoer.EchoerBase + { + private readonly ILogger _logger; + public GrpcEchoService(ILogger logger) + { + _logger = logger; + } + + public override Task Echo(EchoRequest request, ServerCallContext context) + { + return Task.FromResult(new EchoReply + { + Payload = request.Payload + }); + } + } +} diff --git a/Benchmarking/EchoServiceGrpc/Startup.cs b/Benchmarking/EchoServiceGrpc/Startup.cs new file mode 100644 index 0000000..b93a112 --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/Startup.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace EchoService +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + services.AddGrpc(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGrpcService(); + + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909"); + }); + }); + } + } +} diff --git a/Benchmarking/EchoServiceGrpc/appsettings.Development.json b/Benchmarking/EchoServiceGrpc/appsettings.Development.json new file mode 100644 index 0000000..1ca5366 --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/appsettings.Development.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Warning", + "System": "Warning", + "Grpc": "Warning", + "Microsoft": "Warning" + } + } +} diff --git a/Benchmarking/EchoServiceGrpc/appsettings.json b/Benchmarking/EchoServiceGrpc/appsettings.json new file mode 100644 index 0000000..3110458 --- /dev/null +++ b/Benchmarking/EchoServiceGrpc/appsettings.json @@ -0,0 +1,15 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Warning", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Warning" + } + }, + "AllowedHosts": "*", + "Kestrel": { + "EndpointDefaults": { + "Protocols": "Http2" + } + } +}