forward integration

Benchmarking/Benchmark/Benchmark.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
+    <PackageReference Include="Capnp.Net.Runtime" Version="1.2.189" />
+    <PackageReference Include="CapnpC.CSharp.MsBuild.Generation" Version="1.2.138" />
+    <PackageReference Include="Google.Protobuf" Version="3.11.3" />
+    <PackageReference Include="Grpc.Net.Client" Version="2.27.0" />
+    <PackageReference Include="Grpc.Tools" Version="2.27.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Protobuf Include="Protos\Echo.proto" GrpcServices="Client" />
+  </ItemGroup>
+
+</Project>

Benchmarking/Benchmark/CapnpBenchmark.cs

@@ -0,0 +1,46 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Jobs;
+using Capnp.Rpc;
+using CapnpGen;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Benchmark
+{
+    public class CapnpBenchmark
+    {
+        [Params(20, 200, 2000, 20000, 200000, 2000000)]
+        public int PayloadBytes;
+
+        TcpRpcClient _client;
+        IEchoer _echoer;
+        byte[] _payload;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _client = new TcpRpcClient("localhost", 5002);
+            _client.WhenConnected.Wait();
+            _echoer = _client.GetMain<IEchoer>();
+            _payload = new byte[PayloadBytes];
+            new Random().NextBytes(_payload);
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            _echoer.Dispose();
+            _client.Dispose();
+        }
+
+        [Benchmark]
+        public void Echo()
+        {
+            var t = _echoer.Echo(_payload);
+            t.Wait();
+            if (t.Result?.Count != _payload.Length)
+                throw new InvalidOperationException("Echo server malfunction");
+        }
+    }
+}

Benchmarking/Benchmark/GrpcBenchmark.cs

@@ -0,0 +1,42 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Jobs;
+using Grpc.Net.Client;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Benchmark
+{
+    public class GrpcBenchmark
+    {
+        [Params(20, 200, 2000, 20000, 200000, 2000000)]
+        public int PayloadBytes;
+
+        GrpcChannel _channel;
+        Echoer.EchoerClient _echoer;
+        byte[] _payload;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _channel = GrpcChannel.ForAddress("https://localhost:5001");
+            _echoer = new Echoer.EchoerClient(_channel);
+            _payload = new byte[PayloadBytes];
+            new Random().NextBytes(_payload);
+        }
+
+        [GlobalCleanup]
+        public void Teardown()
+        {
+            _channel.Dispose();
+        }
+
+        [Benchmark]
+        public void Echo()
+        {
+            var reply = _echoer.Echo(new EchoRequest { Payload = Google.Protobuf.ByteString.CopyFrom(_payload) });
+            if (reply?.Payload?.Length != _payload.Length)
+                throw new InvalidOperationException("Echo server malfunction");
+        }
+    }
+}

Benchmarking/Benchmark/Program.cs

@@ -0,0 +1,18 @@
+using BenchmarkDotNet.Running;
+using Capnp.Rpc;
+using Grpc.Net.Client;
+using System;
+using System.Diagnostics;
+using System.Threading.Tasks;
+
+namespace Benchmark
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            BenchmarkRunner.Run<GrpcBenchmark>();
+            BenchmarkRunner.Run<CapnpBenchmark>();
+        }
+    }
+}

Benchmarking/Benchmark/Protos/Echo.capnp

@@ -0,0 +1,5 @@
+@0x8c309c720de8cf7c;
+
+interface Echoer {
+  echo @0 (input : Data) -> (output : Data);
+}

Benchmarking/Benchmark/Protos/Echo.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+service Echoer {
+  rpc Echo (EchoRequest) returns (EchoReply);
+}
+
+message EchoRequest {
+  bytes payload = 1;
+}
+
+message EchoReply {
+  bytes payload = 1;
+}

Benchmarking/CapnpBench.sln

@@ -0,0 +1,37 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.29728.190
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoServiceGrpc", "EchoServiceGrpc\EchoServiceGrpc.csproj", "{D59C7B71-3887-426B-A636-2DBDA0549817}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmark", "Benchmark\Benchmark.csproj", "{7F7580CA-CCF0-4650-87BF-502D51A8F435}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EchoServiceCapnp", "EchoServiceCapnp\EchoServiceCapnp.csproj", "{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{D59C7B71-3887-426B-A636-2DBDA0549817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D59C7B71-3887-426B-A636-2DBDA0549817}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D59C7B71-3887-426B-A636-2DBDA0549817}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D59C7B71-3887-426B-A636-2DBDA0549817}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7F7580CA-CCF0-4650-87BF-502D51A8F435}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7F7580CA-CCF0-4650-87BF-502D51A8F435}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7F7580CA-CCF0-4650-87BF-502D51A8F435}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7F7580CA-CCF0-4650-87BF-502D51A8F435}.Release|Any CPU.Build.0 = Release|Any CPU
+		{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{309A4A26-F29E-4F49-AB49-76BAE0FD7D62}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {A6702BC8-8CC0-4BDA-8BD5-D5D268291E93}
+	EndGlobalSection
+EndGlobal

Benchmarking/EchoServiceCapnp/EchoServiceCapnp.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Capnp.Net.Runtime" Version="1.2.189" />
+    <PackageReference Include="CapnpC.CSharp.MsBuild.Generation" Version="1.2.138" />
+  </ItemGroup>
+
+</Project>

Benchmarking/EchoServiceCapnp/Program.cs

@@ -0,0 +1,20 @@
+using Capnp.Rpc;
+using EchoServiceCapnp.Services;
+using System;
+using System.Net;
+
+namespace EchoServiceCapnp
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            using (var server = new TcpRpcServer(IPAddress.Any, 5002))
+            {
+                server.Main = new CapnpEchoService();
+                Console.WriteLine("Press RETURN to stop listening");
+                Console.ReadLine();
+            }
+        }
+    }
+}

Benchmarking/EchoServiceCapnp/Protos/Echo.capnp

@@ -0,0 +1,5 @@
+@0x8c309c720de8cf7c;
+
+interface Echoer {
+  echo @0 (input : Data) -> (output : Data);
+}

Benchmarking/EchoServiceCapnp/Services/CapnpEchoService.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace EchoServiceCapnp.Services
+{
+    public class CapnpEchoService : CapnpGen.IEchoer
+    {
+        public void Dispose()
+        {
+        }
+
+        public Task<IReadOnlyList<byte>> Echo(IReadOnlyList<byte> input, CancellationToken cancellationToken_ = default)
+        {
+            return Task.FromResult(input);
+        }
+    }
+}

Benchmarking/EchoServiceGrpc/EchoServiceGrpc.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Protobuf Include="Protos\Echo.proto" GrpcServices="Server" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Grpc.AspNetCore" Version="2.24.0" />
+  </ItemGroup>
+
+</Project>

Benchmarking/EchoServiceGrpc/Program.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Hosting;
+
+namespace EchoService
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+
+        // Additional configuration is required to successfully run gRPC on macOS.
+        // For instructions on how to configure Kestrel and gRPC clients on macOS, visit https://go.microsoft.com/fwlink/?linkid=2099682
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureWebHostDefaults(webBuilder =>
+                {
+                    webBuilder.UseStartup<Startup>();
+                });
+    }
+}

Benchmarking/EchoServiceGrpc/Protos/Echo.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+service Echoer {
+  rpc Echo (EchoRequest) returns (EchoReply);
+}
+
+message EchoRequest {
+  bytes payload = 1;
+}
+
+message EchoReply {
+  bytes payload = 1;
+}

Benchmarking/EchoServiceGrpc/Services/GrpcEchoService.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Grpc.Core;
+using Microsoft.Extensions.Logging;
+
+namespace EchoService
+{
+    public class GrpcEchoService : Echoer.EchoerBase
+    {
+        private readonly ILogger<GrpcEchoService> _logger;
+        public GrpcEchoService(ILogger<GrpcEchoService> logger)
+        {
+            _logger = logger;
+        }
+
+        public override Task<EchoReply> Echo(EchoRequest request, ServerCallContext context)
+        {
+            return Task.FromResult(new EchoReply
+            {
+                Payload = request.Payload
+            });
+        }
+    }
+}

Benchmarking/EchoServiceGrpc/Startup.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+
+namespace EchoService
+{
+    public class Startup
+    {
+        // This method gets called by the runtime. Use this method to add services to the container.
+        // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
+        public void ConfigureServices(IServiceCollection services)
+        {
+            services.AddGrpc();
+        }
+
+        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
+        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+        {
+            if (env.IsDevelopment())
+            {
+                app.UseDeveloperExceptionPage();
+            }
+
+            app.UseRouting();
+
+            app.UseEndpoints(endpoints =>
+            {
+                endpoints.MapGrpcService<GrpcEchoService>();
+
+                endpoints.MapGet("/", async context =>
+                {
+                    await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909");
+                });
+            });
+        }
+    }
+}

Benchmarking/EchoServiceGrpc/appsettings.Development.json

@@ -0,0 +1,10 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "System": "Warning",
+      "Grpc": "Warning",
+      "Microsoft": "Warning"
+    }
+  }
+}

Benchmarking/EchoServiceGrpc/appsettings.json

@@ -0,0 +1,15 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Warning"
+    }
+  },
+  "AllowedHosts": "*",
+  "Kestrel": {
+    "EndpointDefaults": {
+      "Protocols": "Http2"
+    }
+  }
+}

Capnp.Net.Runtime.Tests.Core21/Capnp.Net.Runtime.Tests.Core21.csproj

@@ -22,6 +22,7 @@
     <Compile Include="..\Capnp.Net.Runtime.Tests\ProvidedCapabilityMock.cs" Link="ProvidedCapabilityMock.cs" />
     <Compile Include="..\Capnp.Net.Runtime.Tests\ProvidedCapabilityMultiCallMock.cs" Link="ProvidedCapabilityMultiCallMock.cs" />
     <Compile Include="..\Capnp.Net.Runtime.Tests\RpcSchemaTests.cs" Link="RpcSchemaTests.cs" />
+    <Compile Include="..\Capnp.Net.Runtime.Tests\ScatteringStream.cs" Link="ScatteringStream.cs" />
     <Compile Include="..\Capnp.Net.Runtime.Tests\SegmentAllocatorTests.cs" Link="SegmentAllocatorTests.cs" />
     <Compile Include="..\Capnp.Net.Runtime.Tests\TcpRpc.cs" Link="TcpRpc.cs" />
     <Compile Include="..\Capnp.Net.Runtime.Tests\TcpRpcAdvancedStuff.cs" Link="TcpRpcAdvancedStuff.cs" />

Capnp.Net.Runtime.Tests/ScatteringStream.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Capnp.Net.Runtime.Tests
+{
+    /// <summary>
+    /// Imitates the behavior of a TCP connection by real hardware, which splits data transfer into multiple packets.
+    /// </summary>
+    class ScatteringStream : Stream
+    {
+        readonly Stream _baseStream;
+        readonly int _mtu;
+
+        public ScatteringStream(Stream baseStream, int mtu)
+        {
+            _baseStream = baseStream;
+            _mtu = mtu;
+        }
+
+        public override bool CanRead => _baseStream.CanRead;
+
+        public override bool CanSeek => false;
+
+        public override bool CanWrite => _baseStream.CanWrite;
+
+        public override long Length => _baseStream.Length;
+
+        public override long Position 
+        { 
+            get => _baseStream.Position; 
+            set => throw new NotImplementedException(); 
+        }
+
+        public override void Flush() => _baseStream.Flush();
+
+        public override int Read(byte[] buffer, int offset, int count) => _baseStream.Read(buffer, offset, count);
+
+        public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException();
+
+        public override void SetLength(long value) => throw new NotImplementedException();
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            while (count > 0)
+            {
+                int amount = Math.Min(count, _mtu);
+                _baseStream.Write(buffer, offset, amount);
+                _baseStream.Flush();
+                offset += amount;
+                count -= amount;
+            }
+        }
+    }
+}

Capnp.Net.Runtime.Tests/TcpRpcPorted.cs

@@ -484,7 +484,7 @@ namespace Capnp.Net.Runtime.Tests
                             }
                             catch (AggregateException exception) when (exception.InnerException is RpcException rpcException && rpcException.Message == "Cannot access a disposed object.")
                             {
-                                Logger.Log(LogLevel.Information, $"Oops, object disposed. Counter = {cap.Count}, tx count = {client.SendCount}, rx count = {client.RecvCount}");
+                                Console.WriteLine($"Oops, object disposed. Counter = {cap.Count}, tx count = {client.SendCount}, rx count = {client.RecvCount}");
                                 throw;
                             }
 

Capnp.Net.Runtime.Tests/TcpRpcStress.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Text;
 using System.Threading;
 
@@ -89,5 +90,48 @@ namespace Capnp.Net.Runtime.Tests
             var t = new TcpRpc();
             Repeat(100, t.PipelineAfterReturn);
         }
+
+        [TestMethod]
+        public void ScatteredTransfer()
+        {
+
+            using (var server = new TcpRpcServer(IPAddress.Any, TcpPort))
+            using (var client = new TcpRpcClient())
+            {
+                server.OnConnectionChanged += (_, e) =>
+                {
+                    if (e.Connection.State == ConnectionState.Initializing)
+                    {
+                        e.Connection.InjectMidlayer(s => new ScatteringStream(s, 7));
+                    }
+                };
+
+                client.InjectMidlayer(s => new ScatteringStream(s, 10));
+                client.Connect("localhost", TcpPort);
+                client.WhenConnected.Wait();
+
+                var counters = new Counters();
+                server.Main = new TestInterfaceImpl(counters);
+                using (var main = client.GetMain<ITestInterface>())
+                {
+                    for (int i = 0; i < 100; i++)
+                    {
+                        var request1 = main.Foo(123, true, default);
+                        var request3 = Assert.ThrowsExceptionAsync<RpcException>(() => main.Bar(default));
+                        var s = new TestAllTypes();
+                        Common.InitTestMessage(s);
+                        var request2 = main.Baz(s, default);
+
+                        Assert.IsTrue(request1.Wait(MediumNonDbgTimeout));
+                        Assert.IsTrue(request2.Wait(MediumNonDbgTimeout));
+                        Assert.IsTrue(request3.Wait(MediumNonDbgTimeout));
+
+                        Assert.AreEqual("foo", request1.Result);
+                        Assert.AreEqual(2, counters.CallCount);
+                        counters.CallCount = 0;
+                    }
+                }
+            }
+        }
     }
 }

Capnp.Net.Runtime/Capnp.Net.Runtime.csproj

@@ -20,7 +20,6 @@
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <RepositoryType>Git</RepositoryType>
     <PackageTags>capnp "Cap'n Proto" RPC serialization cerealization</PackageTags>
-    <Version>1.3-local$([System.DateTime]::UtcNow.ToString(yyMMddHHmm))</Version>
     <Configurations>Debug;Release</Configurations>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
   </PropertyGroup>

Capnp.Net.Runtime/Framing.cs

@@ -80,6 +80,9 @@ namespace Capnp
             return new WireFrame(buffers);
         }
 
+        static InvalidDataException StreamClosed() 
+            => new InvalidDataException("Prematurely reached end of stream. Expected more bytes according to framing header.");
+
         static void FillBuffersFromFrames(Memory<ulong>[] buffers, uint segmentCount, BinaryReader reader)
         {
             for (uint i = 0; i < segmentCount; i++)
@@ -90,7 +93,12 @@ namespace Capnp
 
                 if (tmpBuffer.Length != buffer.Length)
                 {
-                    throw new InvalidDataException("Expected more bytes according to framing header");
+                    // Note w.r.t. issue #37: If there are temporarily less bytes available, 
+                    // this will NOT cause ReadBytes to return a shorter buffer. 
+                    // Only if the end of the stream is reached will we enter this branch. And this will be an error condition,
+                    // since it would mean that the connection was closed in the middle of a frame transfer.
+
+                    throw StreamClosed();
                 }
                 
                 // Fastest way to do this without /unsafe
@@ -101,7 +109,15 @@ namespace Capnp
                 }
 #else
                 var buffer = MemoryMarshal.Cast<ulong, byte>(buffers[i].Span);
-                reader.Read(buffer);
+
+                do
+                {
+                    int obtained = reader.Read(buffer);
+                    if (obtained == 0)
+                        throw StreamClosed();
+                    buffer = buffer.Slice(obtained);
+                }
+                while (buffer.Length > 0);
 #endif
             }
         }

Capnp.Net.Runtime/Rpc/IConnection.cs

@@ -1,5 +1,6 @@
 using Capnp.FrameTracing;
 using System;
+using System.IO;
 
 namespace Capnp.Rpc
 {
@@ -51,6 +52,15 @@ namespace Capnp.Rpc
         /// <exception cref="InvalidOperationException">Connection is not in state 'Initializing'</exception>
         void AttachTracer(IFrameTracer tracer);
 
+        /// <summary>
+        /// Installs a midlayer. A midlayer is a protocal layer that resides somewhere between capnp serialization and the raw TCP stream.
+        /// Thus, we have a hook mechanism for transforming data before it is sent to the TCP connection or after it was received
+        /// by the TCP connection, respectively. This mechanism may be used for integrating various (de-)compression algorithms.
+        /// </summary>
+        /// <param name="createFunc">Callback for wrapping the midlayer around its underlying stream</param>
+        /// <exception cref="ArgumentNullException"><paramref name="createFunc"/> is null</exception>
+        void InjectMidlayer(Func<Stream, Stream> createFunc);
+
         /// <summary>
         /// Prematurely closes this connection. Note that there is usually no need to close a connection manually. The typical use case
         /// of this method is to refuse an incoming connection in the <code>TcpRpcServer.OnConnectionChanged</code> callback.

Capnp.Net.Runtime/Rpc/TcpRpcClient.cs

@@ -40,6 +40,7 @@ namespace Capnp.Rpc
 
         readonly RpcEngine _rpcEngine;
         readonly TcpClient _client;
+        Func<Stream, Stream> _createLayers = _ => _;
         RpcEngine.RpcEndpoint? _inboundEndpoint;
         OutboundTcpEndpoint? _outboundEndpoint;
         FramePump? _pump;
@@ -78,7 +79,9 @@ namespace Capnp.Rpc
             await ConnectAsync(host, port);
 
             State = ConnectionState.Active;
-            _pump = new FramePump(_client.GetStream());
+
+            var stream = _createLayers(_client.GetStream());
+            _pump = new FramePump(stream);
             _attachTracerAction?.Invoke();
             _outboundEndpoint = new OutboundTcpEndpoint(this, _pump);
             _inboundEndpoint = _rpcEngine.AddEndpoint(_outboundEndpoint);
@@ -218,6 +221,25 @@ namespace Capnp.Rpc
             };
         }
 
+        /// <summary>
+        /// Installs a midlayer. A midlayer is a protocal layer that resides somewhere between capnp serialization and the raw TCP stream.
+        /// Thus, we have a hook mechanism for transforming data before it is sent to the TCP connection or after it was received
+        /// by the TCP connection, respectively. This mechanism may be used for integrating various (de-)compression algorithms.
+        /// </summary>
+        /// <param name="createFunc">Callback for wrapping the midlayer around its underlying stream</param>
+        /// <exception cref="ArgumentNullException"><paramref name="createFunc"/> is null</exception>
+        public void InjectMidlayer(Func<Stream, Stream> createFunc)
+        {
+            if (createFunc == null)
+                throw new ArgumentNullException(nameof(createFunc));
+
+            if (State != ConnectionState.Initializing)
+                throw new InvalidOperationException("Connection is not in state 'Initializing'");
+
+            var last = _createLayers;
+            _createLayers = _ => createFunc(last(_));
+        }
+
         /// <summary>
         /// Prematurely closes this connection. Note that there is usually no need to close a connection manually.
         /// </summary>

Capnp.Net.Runtime/Rpc/TcpRpcServer.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Net;
 using System.Net.Sockets;
 using System.Threading;
@@ -60,24 +61,29 @@ namespace Capnp.Rpc
         class Connection: IConnection
         {
             readonly TcpRpcServer _server;
+            Stream _stream;
 
-            public Connection(TcpRpcServer server, TcpClient client, FramePump pump, OutboundTcpEndpoint outboundEp, RpcEngine.RpcEndpoint inboundEp)
+            public Connection(TcpRpcServer server, TcpClient client)
             {
                 _server = server;
                 Client = client;
-                Pump = pump;
+                _stream = client.GetStream();
-                OutboundEp = outboundEp;
-                InboundEp = inboundEp;
             }
 
             public void Start()
             {
+                Pump = new FramePump(_stream);
+                OutboundEp = new OutboundTcpEndpoint(_server, Pump);
+                InboundEp = _server._rpcEngine.AddEndpoint(OutboundEp);
+                Pump.FrameReceived += InboundEp.Forward;
+
+                State = ConnectionState.Active;
+
                 PumpRunner = new Thread(o =>
                 {
                     try
                     {
                         Thread.CurrentThread.Name = $"TCP RPC Server Thread {Thread.CurrentThread.ManagedThreadId}";
-                        State = ConnectionState.Active;
 
                         Pump.Run();
                     }
@@ -122,6 +128,24 @@ namespace Capnp.Rpc
                 Pump.AttachTracer(tracer);
             }
 
+            /// <summary>
+            /// Installs a midlayer. A midlayer is a protocal layer that resides somewhere between capnp serialization and the raw TCP stream.
+            /// Thus, we have a hook mechanism for transforming data before it is sent to the TCP connection or after it was received
+            /// by the TCP connection, respectively. This mechanism may be used for integrating various (de-)compression algorithms.
+            /// </summary>
+            /// <param name="createFunc">Callback for wrapping the midlayer around its underlying stream</param>
+            /// <exception cref="ArgumentNullException"><paramref name="createFunc"/> is null</exception>
+            public void InjectMidlayer(Func<Stream, Stream> createFunc)
+            {
+                if (createFunc == null)
+                    throw new ArgumentNullException(nameof(createFunc));
+
+                if (State != ConnectionState.Initializing)
+                    throw new InvalidOperationException("Connection is not in state 'Initializing'");
+
+                _stream = createFunc(_stream);
+            }
+
             public void Close()
             {
                 Client.Dispose();
@@ -149,12 +173,7 @@ namespace Capnp.Rpc
                 while (true)
                 {
                     var client = _listener.AcceptTcpClient();
-                    var pump = new FramePump(client.GetStream());
+                    var connection = new Connection(this, client);
-                    var outboundEndpoint = new OutboundTcpEndpoint(this, pump);
-                    var inboundEndpoint = _rpcEngine.AddEndpoint(outboundEndpoint);
-                    pump.FrameReceived += inboundEndpoint.Forward;
-
-                    var connection = new Connection(this, client, pump, outboundEndpoint, inboundEndpoint);
 
                     lock (_reentrancyBlocker)
                     {

CapnpC.CSharp.MsBuild.Generation/CapnpC.CSharp.MsBuild.Generation.csproj

@@ -9,12 +9,8 @@
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
-    <AssemblyVersion>1.3.0.0</AssemblyVersion>
-    <FileVersion>1.3.0.0</FileVersion>
-    <Version>1.3-local$([System.DateTime]::UtcNow.ToString(yyMMddHHmm))</Version>
 
     <NuspecFile>$(MSBuildThisFileDirectory)CapnpC.CSharp.MsBuild.Generation.nuspec</NuspecFile>
-    <NuspecProperties>version=$(Version);configuration=$(Configuration)</NuspecProperties>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageOutputPath>..\bin\$(Configuration)</PackageOutputPath>
     <Configurations>Debug;Release</Configurations>

CapnpC.CSharp.MsBuild.Generation/CapnpC.CSharp.MsBuild.Generation.nuspec

@@ -21,9 +21,9 @@
   <files>
     <file src="build\**\*" target="build" />
     <file src="buildMultiTargeting\**\*" target="buildMultiTargeting" />
-    <file src="bin\$configuration$\net471\*.dll" target="tasks\net471" />
+    <file src="bin\$config$\net471\*.dll" target="tasks\net471" />
-    <file src="bin\$configuration$\netcoreapp2.1\*.dll" target="tasks\netcoreapp2.1" />
+    <file src="bin\$config$\netcoreapp2.1\*.dll" target="tasks\netcoreapp2.1" />
-    <file src="bin\$configuration$\netcoreapp2.1\*.deps.json" target="tasks\netcoreapp2.1" />
+    <file src="bin\$config$\netcoreapp2.1\*.deps.json" target="tasks\netcoreapp2.1" />
     
     <file src="..\Licenses\**\*" target="licenses" />
     <file src="..\LICENSE" target="LICENSE" />

Directory.Build.props

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <PackageReference Include="Nerdbank.GitVersioning">
+      <Version>3.0.28</Version>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+</Project>

Directory.Build.targets

@@ -0,0 +1,23 @@
+<Project>
+  <Target Name="SetNuspecProperties" BeforeTargets="GenerateNuspec" DependsOnTargets="GetBuildVersion">
+    <PropertyGroup>
+      <NuspecProperties>$(NuspecProperties);config=$(Configuration)</NuspecProperties>
+      <NuspecProperties>$(NuspecProperties);version=$(NuGetPackageVersion)</NuspecProperties>
+      <NuspecProperties>$(NuspecProperties);SolutionDir=$(SolutionDir)</NuspecProperties>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="CleanupNupkgs" BeforeTargets="Build" >
+    <ItemGroup>
+      <Nupkgs2Delete Include="$(PackageOutputAbsolutePath)\*.nupkg"/>
+    </ItemGroup>
+    <Delete Files="@(Nupkgs2Delete)" />
+  </Target>
+
+  <Target Name="CopyNupkgToFolder" AfterTargets="Pack">
+    <ItemGroup>
+      <GeneratedNupkgs Include="$(PackageOutputAbsolutePath)\*.nupkg"/>
+    </ItemGroup>
+    <Copy SourceFiles="@(GeneratedNupkgs)" DestinationFolder="$(SolutionDir)GeneratedNuGetPackages\$(Configuration)" />
+  </Target>
+</Project>

MsBuildGenerationTest/MsBuildGenerationTest.csproj

@@ -5,7 +5,6 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
     <LangVersion>8.0</LangVersion>
     <Nullable>Enable</Nullable>
-    <Version>1.3-local</Version>
     <Configurations>Debug;Release</Configurations>
     <PackageReferenceVersion Condition="'$(PackageReferenceVersion)'==''">$(Version)*</PackageReferenceVersion>
   </PropertyGroup>

README.md

@@ -43,6 +43,9 @@ choco install capnpc-csharp-win-x86
 
 Both versions will also download and install the [Cap'n Proto tool set Chocolatey package](https://www.chocolatey.org/packages/capnproto). Note that the author does not maintain this package and has no influence on its contents.
 
+*Announcement: There is currently an experimental branch for packaging the code generator back end as .NET Core CLI tool. If this approach turns out to be viable, it will probably be superior to the Chocolatey deployment. In that case, the CLI tool capnpc-csharp will be deployed on NuGet.org and the former Chocolatey package will be deprecated. capnpc-csharp-win-x86 will probably be kept (and also maintained).*
+
+
 ### Code generator back end: Other OSes
 
 Currently, you are on yourself. Compile the `capnpc-csharp` VS project and install the resulting .NET application manually on your system. This should not be that complicated, see also the [Wiki](https://github.com/c80k/capnproto-dotnetcore/wiki). It would be great to support other package managers, especially [APT](https://wiki.debian.org/Apt). Consider contributing? Author would be happy!

appveyor.yml

@@ -1,27 +1,21 @@
-version: '1.3.{build}'
 image: Visual Studio 2019
 # branches:
 #  only:
 # - master
+version: '{build}'
 cache:
   - c:\Tools\vcpkg\installed
-init:
-  # Good practise, because Windows line endings are different from Unix/Linux ones
-  - cmd: git config --global core.autocrlf true
 install:
   - cd c:\tools\vcpkg
   - vcpkg integrate install
   - vcpkg install capnproto
   - cd %APPVEYOR_BUILD_FOLDER%
-dotnet_csproj:
+  - dotnet tool install -g nbgv
-  patch: true
+init:
-  file: '**\*.csproj;**\*.props;**\*.fsproj;**\*.xml'
+  # Good practise, because Windows line endings are different from Unix/Linux ones
-  version: '{version}'
+  - cmd: git config --global core.autocrlf true
-  package_version: '{version}'
-  assembly_version: '{version}'
-  file_version: '{version}'
-  informational_version: '{version}'
 before_build:
+  - cmd: nbgv cloud
   - cmd: dotnet --version
   - cmd: msbuild -ver
   - cmd: dotnet restore ./Capnp.Net.Runtime/Capnp.Net.Runtime.csproj --verbosity m
@@ -38,48 +32,54 @@ build_script:
   - cmd: msbuild ./Capnp.Net.sln /p:Configuration="Release"
   - cmd: msbuild ./CapnpCompatTest.sln /p:Configuration="Debug"
   - cmd: msbuild ./CapnpCompatTest.sln /p:Configuration="Release"
-  - ps: scripts\capnpc-csharp-pack.ps1 $env:appveyor_build_version
+  - ps: scripts\capnpc-csharp-pack.ps1
 after_build:
   # For once the build has completed
 artifacts:
- - path: bin\Release\Capnp.Net.Runtime.$(appveyor_build_version).nupkg
+ - path: bin\Release\Capnp.Net.Runtime.*.nupkg
    name: Capnp.Net.Runtime
    type: NuGetPackage
- - path: bin\Release\CapnpC.CSharp.MsBuild.Generation.$(appveyor_build_version).nupkg
+ - path: bin\Release\CapnpC.CSharp.MsBuild.Generation.*.nupkg
    name: CapnpC.CSharp.MsBuild.Generation
    type: NuGetPackage
- - path: chocolatey\install\capnpc-csharp-win-x86.$(appveyor_build_version).nupkg
+ - path: chocolatey\install\capnpc-csharp-win-x86.*.nupkg
    name: capnpc-csharp-win-x86
    type: NuGetPackage
- - path: chocolatey\install\capnpc-csharp.$(appveyor_build_version).nupkg
+ - path: chocolatey\install\capnpc-csharp.*-deprecated.nupkg
+   name: capnpc-csharp-deprecated
+   type: NuGetPackage
+ - path: capnpc-csharp\nupkg\*.nupkg
    name: capnpc-csharp
    type: NuGetPackage
-clone_depth: 1
 test_script:
-  - cmd: vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.Generator.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.Generator.Tests.dll
+  - cmd: |
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%\chocolatey\install
+      nbgv get-version -v NuGetPackageVersion >> version.txt
-  - cmd: choco install capnpc-csharp --source=".;https://chocolatey.org/api/v2" --force -y
+      set /P VERSION=< version.txt
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%\install-test
+      vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.Generator.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.Generator.Tests.dll
-  - cmd: compile-test
+      choco install capnproto --source="https://chocolatey.org/api/v2" --force -y
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%
+      cd %APPVEYOR_BUILD_FOLDER%\capnpc-csharp
-  - cmd: vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.Generator.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.Generator.Tests.dll
+      dotnet tool install --global --add-source ./nupkg capnpc-csharp --version %VERSION%
-  - cmd: choco uninstall capnpc-csharp -y
+      cd %APPVEYOR_BUILD_FOLDER%\install-test
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%\install-test
+      compile-test
-  - cmd: notinstalled-test
+      cd %APPVEYOR_BUILD_FOLDER%
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%\chocolatey\install
+      vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.Generator.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.Generator.Tests.dll
-  - cmd: choco install capnpc-csharp-win-x86 --source=".;https://chocolatey.org/api/v2" --force -y
+      dotnet tool uninstall --global capnpc-csharp
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%\install-test
+      cd %APPVEYOR_BUILD_FOLDER%\install-test
-  - cmd: compile-test
+      notinstalled-test
-  - cmd: choco uninstall capnpc-csharp-win-x86 -y
+      cd %APPVEYOR_BUILD_FOLDER%\chocolatey\install
-  - cmd: notinstalled-test
+      choco install capnpc-csharp-win-x86 --source=".;https://chocolatey.org/api/v2" --force -y --version %VERSION% --pre
-  - cmd: cd %APPVEYOR_BUILD_FOLDER%
+      cd %APPVEYOR_BUILD_FOLDER%\install-test
-  - cmd: vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.MsBuild.Generation.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.MsBuild.Generation.Tests.dll
+      compile-test
-  - cmd: msbuild -t:restore ./MsBuildGenerationTest/MsBuildGenerationTest.csproj /p:Configuration="Debug" /p:PackageReferenceVersion="%APPVEYOR_BUILD_VERSION%"
+      choco uninstall capnpc-csharp-win-x86 -y
-  - cmd: msbuild ./MsBuildGenerationTest/MsBuildGenerationTest.sln /p:Configuration="Debug" /p:PackageReferenceVersion="%APPVEYOR_BUILD_VERSION%"
+      notinstalled-test
-  - cmd: vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests\bin\Debug\net471\Capnp.Net.Runtime.Tests.Std20.dll
+      cd %APPVEYOR_BUILD_FOLDER%
-  - cmd: vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests\bin\Release\net471\Capnp.Net.Runtime.Tests.Std20.dll
+      vstest.console /logger:Appveyor /inIsolation CapnpC.CSharp.MsBuild.Generation.Tests\bin\Release\netcoreapp3.0\CapnpC.CSharp.MsBuild.Generation.Tests.dll
-  - cmd: vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests.Core21\bin\Debug\netcoreapp2.1\Capnp.Net.Runtime.Tests.Core21.dll
+      msbuild -t:restore ./MsBuildGenerationTest/MsBuildGenerationTest.csproj /p:Configuration="Debug" /p:PackageReferenceVersion="%VERSION%"
-  - cmd: vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests.Core21\bin\Release\netcoreapp2.1\Capnp.Net.Runtime.Tests.Core21.dll
+      msbuild ./MsBuildGenerationTest/MsBuildGenerationTest.sln /p:Configuration="Debug" /p:PackageReferenceVersion="%VERSION%"
+      vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests\bin\Debug\net471\Capnp.Net.Runtime.Tests.Std20.dll
+      vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests\bin\Release\net471\Capnp.Net.Runtime.Tests.Std20.dll
+      vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests.Core21\bin\Debug\netcoreapp2.1\Capnp.Net.Runtime.Tests.Core21.dll
+      vstest.console /logger:Appveyor /inIsolation Capnp.Net.Runtime.Tests.Core21\bin\Release\netcoreapp2.1\Capnp.Net.Runtime.Tests.Core21.dll
 on_finish :
   # any cleanup in here
 deploy:

capnpc-csharp/capnpc-csharp.csproj

@@ -5,7 +5,10 @@
     <TargetFramework>netcoreapp2.1</TargetFramework>
     <RootNamespace>CapnpC</RootNamespace>
     <LangVersion>7.1</LangVersion>
-    <GeneratePackageOnBuild>false</GeneratePackageOnBuild>
+    <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+    <PackAsTool>true</PackAsTool>
+    <ToolCommandName>capnpc-csharp</ToolCommandName>
+    <PackageOutputPath>./nupkg</PackageOutputPath>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <Authors>Christian Köllner and contributors</Authors>
     <Description>Cap'n Proto C# code generator backend</Description>
@@ -13,7 +16,6 @@
     <PackageProjectUrl>https://github.com/c80k/capnproto-dotnetcore</PackageProjectUrl>
     <RepositoryType>Git</RepositoryType>
     <PackageTags>capnp capnpc RPC serialization cerealization</PackageTags>
-    <Version>1.3.0</Version>
     <Configurations>Debug;Release</Configurations>
   </PropertyGroup>
 

chocolatey/capnpc-csharp/capnpc-csharp.nuspec

@@ -10,8 +10,11 @@
     <!--<license type="expression">MIT</license>-->
     <licenseUrl>https://github.com/c80k/capnproto-dotnetcore/blob/master/LICENSE</licenseUrl>
     <projectUrl>https://github.com/c80k/capnproto-dotnetcore</projectUrl>
-    <title>capnpc-csharp (Install)</title>
+    <title>[Deprecated] capnpc-csharp (Install)</title>
-    <description>Cap'n Proto C# code generator backend, portable .NET Core 2.1</description>
+    <description>Cap'n Proto C# code generator backend, portable .NET Core 2.1
+    This package is deprecated now. It was converted to a .NET Core CLI tool which is now hosted here: https://www.nuget.org/packages/capnpc-csharp
+    To install, type: dotnet tool install -g capnpc-csharp
+    </description>
     <summary>Cap'n Proto C# code generator backend. This is the portable variant which depends on .NET Core 2.1 (as opposed to capnpc-csharp-win-x86)</summary>
     <copyright>Christian Köllner and contributors</copyright>
     <projectSourceUrl>https://github.com/c80k/capnproto-dotnetcore</projectSourceUrl>
@@ -22,15 +25,7 @@
     <releaseNotes>https://github.com/c80k/capnproto-dotnetcore/releases/tag/v$version$</releaseNotes>
     <dependencies>
         <dependency id="capnproto" version="0.7.0" />
-        <dependency id="dotnetcore-runtime.install" version="2.1.6" />
     </dependencies>
   </metadata>
-<files>
+<files/>
-    <file src="chocolateyinstall.ps1" target="tools" />
-    <file src="chocolateyuninstall.ps1" target="tools" />
-    <file src=".\capnpc-csharp.ps1" target="tools" />
-    <file src="..\LICENSE.txt" target="tools" />
-    <file src="..\VERIFICATION.txt" target="tools" />
-    <file src=".\bin\**" target="bin" />
-  </files>
 </package>

scripts/capnpc-csharp-pack.ps1

@@ -1,5 +1,5 @@
-param($version = "1.0.0")
+$jversion = nbgv get-version -f json | ConvertFrom-Json
-
+$version = $($jversion.NuGetPackageVersion)
 $id = "capnpc-csharp"
 $id_win_x86 = "capnpc-csharp-win-x86"
 $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
@@ -10,7 +10,7 @@ $installDir = "$chocoDir\install"
 
 dotnet build -c Release "$scriptDir\..\Capnp.Net.sln"
 dotnet publish -c Release -r win-x86 --self-contained -o "$chocoDir\$id_win_x86\bin" "$csprojDir\$csprojFile"
-dotnet publish -c Release -o "$chocoDir\$id\bin" "$csprojDir\$csprojFile"
+# dotnet publish -c Release -o "$chocoDir\$id\bin" "$csprojDir\$csprojFile"
 
 If(!(test-path $installDir))
 {
@@ -21,3 +21,4 @@ Copy-Item "$scriptDir\..\LICENSE" -Destination "$chocoDir\LICENSE.txt"
 
 choco pack "$chocoDir\$id\$id.nuspec" --version $version --outputdirectory $installDir
 choco pack "$chocoDir\$id_win_x86\$id_win_x86.nuspec" --version $version --outputdirectory $installDir
+Rename-Item -Path "$installDir\$id.$version.nupkg" -NewName "$id.$version-deprecated.nupkg"

version.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "https://raw.githubusercontent.com/AArnott/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json",
+  "version": "1.3",
+  "publicReleaseRefSpec": [
+    "^refs/heads/master$",
+    "^refs/heads/v\\d+(?:\\.\\d+)?$"
+  ],
+  "cloudBuild": {
+    "buildNumber": {
+      "enabled": true,
+      "setVersionVariables": true
+    }
+  }
+}