Expose HTTP.sys kernel response buffering control for HttpListener on Windows

src/libraries/System.Net.HttpListener/src/System/Net/Windows/HttpListener.Windows.cs

@@ -31,6 +31,13 @@ public sealed unsafe partial class HttpListener
         // flag is only used on Win8 and later.
         internal static readonly bool SkipIOCPCallbackOnSuccess = Environment.OSVersion.Version >= new Version(6, 2);
 
+        // Enable buffering of response data in the Kernel. The default value is false.
+        // It should be used by an application doing synchronous I/O or by an application doing asynchronous I/O with
+        // no more than one outstanding write at a time, and can significantly improve throughput over high-latency connections.
+        // Applications that use asynchronous I/O and that may have more than one send outstanding at a time should not use this flag.
+        // Enabling this can result in higher CPU and memory usage by Http.sys.
+        internal static bool EnableKernelResponseBuffering { get; } = AppContext.TryGetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", out bool enabled) && enabled;
+
         // Mitigate potential DOS attacks by limiting the number of unknown headers we accept.  Numerous header names
         // with hash collisions will cause the server to consume excess CPU.  1000 headers limits CPU time to under
         // 0.5 seconds per request.  Respond with a 400 Bad Request.

src/libraries/System.Net.HttpListener/src/System/Net/Windows/HttpResponseStream.Windows.cs

@@ -31,6 +31,10 @@ internal Interop.HttpApi.HTTP_FLAGS ComputeLeftToWrite()
             {
                 flags = _httpContext.Response.ComputeHeaders();
             }
+            if (HttpListener.EnableKernelResponseBuffering)
+            {
+                flags |= Interop.HttpApi.HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA;
+            }
             if (_leftToWrite == long.MinValue)
             {
                 Interop.HttpApi.HTTP_VERB method = _httpContext.GetKnownMethod();

src/libraries/System.Net.HttpListener/tests/HttpListenerTests.Windows.cs

@@ -0,0 +1,75 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Reflection;
+using System.Threading.Tasks;
+using Microsoft.DotNet.RemoteExecutor;
+using Xunit;
+
+namespace System.Net.Tests
+{
+    [PlatformSpecific(TestPlatforms.Windows)]
+    [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
+    public class HttpListenerWindowsTests
+    {
+        [Fact]
+        public void EnableKernelResponseBuffering_DefaultIsDisabled()
+        {
+            using (var listener = new HttpListener())
+            {
+                listener.Start();
+                Assert.False(GetEnableKernelResponseBufferingValue());
+            }
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public async Task EnableKernelResponseBuffering_Enabled()
+        {
+            await RemoteExecutor.Invoke(() =>
+            {
+                AppContext.SetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", true);
+
+                using (var listener = new HttpListener())
+                {
+                    listener.Start();
+                    Assert.True(GetEnableKernelResponseBufferingValue());
+                }
+            }).DisposeAsync();
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public async Task EnableKernelResponseBuffering_ImmutableAfterStart()
+        {
+            await RemoteExecutor.Invoke(() =>
+            {
+                AppContext.SetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", false);
+
+                using (var listener = new HttpListener())
+                {
+                    listener.Start();
+                    Assert.False(GetEnableKernelResponseBufferingValue());
+
+                    AppContext.SetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", true);
+
+                    // Assert internal value wasn't updated, despite updating the AppContext switch.
+                    Assert.False(GetEnableKernelResponseBufferingValue());
+                }
+            }).DisposeAsync();
+        }
+
+        private bool GetEnableKernelResponseBufferingValue()
+        {
+            // We need EnableKernelResponseBuffering which is internal so we get it using reflection.
+            var prop = typeof(HttpListener).GetProperty(
+                "EnableKernelResponseBuffering",
+                BindingFlags.Static | BindingFlags.NonPublic);
+
+            Assert.NotNull(prop);
+
+            object? value = prop!.GetValue(obj: null);
+            Assert.NotNull(value);
+
+            return Assert.IsType<bool>(value);
+        }
+    }
+}

src/libraries/System.Net.HttpListener/tests/HttpResponseStreamTests.Windows.cs

@@ -0,0 +1,248 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.IO;
+using System.Net.Http;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.DotNet.RemoteExecutor;
+using Xunit;
+using static System.Net.Tests.HttpListenerTimeoutManagerWindowsTests;
+
+namespace System.Net.Tests
+{
+    [PlatformSpecific(TestPlatforms.Windows)]
+    [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsNanoServer))] // httpsys component missing in Nano.
+    public class HttpResponseStreamWindowsTests : IDisposable
+    {
+        private HttpListenerFactory _factory;
+        private HttpListener _listener;
+        private GetContextHelper _helper;
+
+        public HttpResponseStreamWindowsTests()
+        {
+            _factory = new HttpListenerFactory();
+            _listener = _factory.GetListener();
+            _helper = new GetContextHelper(_listener, _factory.ListeningUrl);
+        }
+
+        public void Dispose()
+        {
+            _factory.Dispose();
+            _helper.Dispose();
+        }
+
+        [Fact] // [ActiveIssue("https://github.com/dotnet/runtime/issues/21918", TestPlatforms.AnyUnix)]
+        public async Task Write_TooMuch_ThrowsProtocolViolationException()
+        {
+            using (HttpClient client = new HttpClient())
+            {
+                _ = client.GetStringAsync(_factory.ListeningUrl);
+
+                HttpListenerContext serverContext = await _listener.GetContextAsync();
+                using (HttpListenerResponse response = serverContext.Response)
+                {
+                    Stream output = response.OutputStream;
+                    byte[] responseBuffer = "A long string"u8.ToArray();
+                    response.ContentLength64 = responseBuffer.Length - 1;
+                    try
+                    {
+                        Assert.Throws<ProtocolViolationException>(() => output.Write(responseBuffer, 0, responseBuffer.Length));
+                        await Assert.ThrowsAsync<ProtocolViolationException>(() => output.WriteAsync(responseBuffer, 0, responseBuffer.Length));
+                    }
+                    finally
+                    {
+                        // Write the remaining bytes to guarantee a successful shutdown.
+                        output.Write(responseBuffer, 0, (int)response.ContentLength64);
+                        output.Close();
+                    }
+                }
+            }
+        }
+
+        [Fact] // [ActiveIssue("https://github.com/dotnet/runtime/issues/21918", TestPlatforms.AnyUnix)]
+        public async Task Write_TooLittleAsynchronouslyAndClose_ThrowsInvalidOperationException()
+        {
+            using (HttpClient client = new HttpClient())
+            {
+                _ = client.GetStringAsync(_factory.ListeningUrl);
+
+                HttpListenerContext serverContext = await _listener.GetContextAsync();
+                using (HttpListenerResponse response = serverContext.Response)
+                {
+                    Stream output = response.OutputStream;
+
+                    byte[] responseBuffer = "A long string"u8.ToArray();
+                    response.ContentLength64 = responseBuffer.Length + 1;
+
+                    // Throws when there are bytes left to write
+                    await output.WriteAsync(responseBuffer, 0, responseBuffer.Length);
+                    Assert.Throws<InvalidOperationException>(() => output.Close());
+
+                    // Write the final byte and make sure we can close.
+                    await output.WriteAsync(new byte[1], 0, 1);
+                    output.Close();
+                }
+            }
+        }
+
+        [Fact] // [ActiveIssue("https://github.com/dotnet/runtime/issues/21918", TestPlatforms.AnyUnix)]
+        public async Task Write_TooLittleSynchronouslyAndClose_ThrowsInvalidOperationException()
+        {
+            using (HttpClient client = new HttpClient())
+            {
+                _ = client.GetStringAsync(_factory.ListeningUrl);
+
+                HttpListenerContext serverContext = await _listener.GetContextAsync();
+                using (HttpListenerResponse response = serverContext.Response)
+                {
+                    Stream output = response.OutputStream;
+
+                    byte[] responseBuffer = "A long string"u8.ToArray();
+                    response.ContentLength64 = responseBuffer.Length + 1;
+
+                    // Throws when there are bytes left to write
+                    output.Write(responseBuffer, 0, responseBuffer.Length);
+                    Assert.Throws<InvalidOperationException>(() => output.Close());
+
+                    // Write the final byte and make sure we can close.
+                    output.Write(new byte[1], 0, 1);
+                    output.Close();
+                }
+            }
+        }
+
+        // Windows only test as Unix implementation uses Socket.Begin/EndSend, which doesn't fail in this case
+        [Fact]
+        public async Task EndWrite_InvalidAsyncResult_ThrowsArgumentException()
+        {
+            using (HttpListenerResponse response1 = await _helper.GetResponse())
+            using (Stream outputStream1 = response1.OutputStream)
+            using (HttpListenerResponse response2 = await _helper.GetResponse())
+            using (Stream outputStream2 = response2.OutputStream)
+            {
+                IAsyncResult beginWriteResult = outputStream1.BeginWrite(new byte[0], 0, 0, null, null);
+
+                AssertExtensions.Throws<ArgumentException>("asyncResult", () => outputStream2.EndWrite(new CustomAsyncResult()));
+                AssertExtensions.Throws<ArgumentException>("asyncResult", () => outputStream2.EndWrite(beginWriteResult));
+            }
+        }
+
+        // Windows only test as Unix implementation uses Socket.Begin/EndSend, which doesn't fail in this case
+        [Fact]
+        public async Task EndWrite_CalledTwice_ThrowsInvalidOperationException()
+        {
+            using (HttpListenerResponse response1 = await _helper.GetResponse())
+            using (Stream outputStream = response1.OutputStream)
+            {
+                IAsyncResult beginWriteResult = outputStream.BeginWrite(new byte[0], 0, 0, null, null);
+                outputStream.EndWrite(beginWriteResult);
+
+                Assert.Throws<InvalidOperationException>(() => outputStream.EndWrite(beginWriteResult));
+            }
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public async Task KernelResponseBufferingEnabled_WriteAsynchronouslyInParts_SetsFlagAndSucceeds()
+        {
+            await RemoteExecutor.Invoke(async () =>
+            {
+                AppContext.SetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", true);
+
+                using (var listenerFactory = new HttpListenerFactory())
+                {
+                    HttpListener listener = listenerFactory.GetListener();
+
+                    const string expectedResponse = "hello from HttpListener";
+                    Task<HttpListenerContext> serverContextTask = listener.GetContextAsync();
+
+                    using (HttpClient client = new HttpClient())
+                    {
+                        Task<string> clientTask = client.GetStringAsync(listenerFactory.ListeningUrl);
+
+                        HttpListenerContext serverContext = await serverContextTask;
+                        using (HttpListenerResponse response = serverContext.Response)
+                        {
+                            byte[] responseBuffer = Encoding.UTF8.GetBytes(expectedResponse);
+                            response.ContentLength64 = responseBuffer.Length;
+
+                            using (Stream outputStream = response.OutputStream)
+                            {
+                                AssertBufferDataFlagIsSet(outputStream);
+
+                                await outputStream.WriteAsync(responseBuffer, 0, 5);
+                                await outputStream.WriteAsync(responseBuffer, 5, responseBuffer.Length - 5);
+                            }
+                        }
+
+                        var clientString = await clientTask;
+
+                        Assert.Equal(expectedResponse, clientString);
+                    }
+                }
+            }).DisposeAsync();
+        }
+
+        [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))]
+        public async Task KernelResponseBufferingEnabled_WriteSynchronouslyInParts_SetsFlagAndSucceeds()
+        {
+            await RemoteExecutor.Invoke(async () =>
+            {
+                AppContext.SetSwitch("System.Net.HttpListener.EnableKernelResponseBuffering", true);
+                using (var listenerFactory = new HttpListenerFactory())
+                {
+                    HttpListener listener = listenerFactory.GetListener();
+
+                    const string expectedResponse = "hello from HttpListener";
+                    Task<HttpListenerContext> serverContextTask = listener.GetContextAsync();
+
+                    using (HttpClient client = new HttpClient())
+                    {
+                        Task<string> clientTask = client.GetStringAsync(listenerFactory.ListeningUrl);
+
+                        HttpListenerContext serverContext = await serverContextTask;
+                        using (HttpListenerResponse response = serverContext.Response)
+                        {
+                            byte[] responseBuffer = Encoding.UTF8.GetBytes(expectedResponse);
+                            response.ContentLength64 = responseBuffer.Length;
+
+                            using (Stream outputStream = response.OutputStream)
+                            {
+                                AssertBufferDataFlagIsSet(outputStream);
+
+                                outputStream.Write(responseBuffer, 0, 5);
+                                outputStream.Write(responseBuffer, 5, responseBuffer.Length - 5);
+                            }
+                        }
+
+                        var clientString = await clientTask;
+
+                        Assert.Equal(expectedResponse, clientString);
+                    }
+                }
+            }).DisposeAsync();
+        }
+
+        private static void AssertBufferDataFlagIsSet(Stream outputStream)
+        {
+            MethodInfo compute =
+                outputStream.GetType().GetMethod("ComputeLeftToWrite", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
+
+            Assert.NotNull(compute);
+
+            object flagsObj = compute.Invoke(outputStream, parameters: null);
+            Assert.NotNull(flagsObj);
+
+            Assert.True(flagsObj is Enum, "Expected ComputeLeftToWrite to return an enum");
+
+            Enum flagsEnum = (Enum)flagsObj;
+
+            Type enumType = flagsEnum.GetType();
+            Enum bufferDataEnum =
+                (Enum)Enum.Parse(enumType, nameof(HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA), ignoreCase: false);
+
+            Assert.True(flagsEnum.HasFlag(bufferDataEnum));
+        }
+    }
+}