Scatter/gather support

docs/scatter-gather.md

@@ -0,0 +1,104 @@
+# Scatter/Gather
+
+ServiceCompose natively supports scatter/gather scenarios. Scatter/gather is supported through a fanout approach. Given an incoming HTTP request, ServiceComposer will issue as many downstream HTTP requests to fetch data from downstream endpoints. Once all data has been retrieved, they are composed and returned to the original upstream caller.
+
+The following configuration configures a scatter/gather endpoint:
+
+<!-- snippet: scatter-gather-basic-usage -->
+<a id='snippet-scatter-gather-basic-usage'></a>
+```cs
+public void Configure(IApplicationBuilder app, ILoggerFactory loggerFactory)
+{
+    app.UseRouting();
+    app.UseEndpoints(builder => builder.MapScatterGather(template: "api/scatter-gather", new ScatterGatherOptions()
+    {
+        Gatherers = new List<Gatherer>
+        {
+            new HttpGatherer(key: "ASamplesSource", destinationUrl: "https://a.web.server/api/samples/ASamplesSource"),
+            new HttpGatherer(key: "AnotherSamplesSource", destinationUrl: "https://another.web.server/api/samples/AnotherSamplesSource")
+        }
+    }));
+}
+```
+<sup><a href='/src/Snippets/ScatterGather/Startup.cs#L11-L24' title='Snippet source file'>snippet source</a> | <a href='#snippet-scatter-gather-basic-usage' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+The above configuration snippet configures ServiceComposer to handle HTTP requests matching the template. Each time a matching request is dealt with, ServiceComposer invokes each configured gatherer and merges responses from each one into a response returned to the original issuer.
+
+The `Key` and `Destination` properties are mandatory. The key uniquely identifies each gatherer in the context of a specific request. The destination is the downstream URL of the endpoint to invoke to retrieve data.
+
+## Customizing downstream URLs
+
+If the incoming request contains a query string, the query string and its values are automatically appended to downstream URLs as is. It is possible to override that behavior by setting the `DownstreamUrlMapper` delegate as presented in the following snippet:
+
+<!-- snippet: scatter-gather-customizing-downstream-urls -->
+<a id='snippet-scatter-gather-customizing-downstream-urls'></a>
+```cs
+public void Configure(IApplicationBuilder app, ILoggerFactory loggerFactory)
+{
+    app.UseEndpoints(builder => builder.MapScatterGather(template: "api/scatter-gather", new ScatterGatherOptions()
+    {
+        Gatherers = new List<Gatherer>
+        {
+            new HttpGatherer("ASamplesSource", "https://a.web.server/api/samples/ASamplesSource")
+            {
+                DestinationUrlMapper = (request, destination) => destination.Replace(
+                    "{this-is-contextual}", 
+                    request.HttpContext.Request.Query["this-is-contextual"])
+            }
+        }
+    }));
+}
+```
+<sup><a href='/src/Snippets/ScatterGather/CustomizingDownstreamURLs.cs#L10-L26' title='Snippet source file'>snippet source</a> | <a href='#snippet-scatter-gather-customizing-downstream-urls' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+The same approach can be used to customize the downstream URL before invocation.
+
+## Data format
+
+ServiceComposer scatter/gather support works only with JSON data. Gatherers must return an `IEnumerable<JsonNode>`. By default, gatherers assume that the downstream endpoint result can be converted into a `JsonArray`.
+
+### Transforming returned data
+
+If there is a need to transform downstream data to respect the expected format, it's possible to create a custom gatherer and override the `TransformResponse` method:
+
+<!-- snippet: scatter-gather-transform-response -->
+<a id='snippet-scatter-gather-transform-response'></a>
+```cs
+public class CustomHttpGatherer : HttpGatherer
+{
+    public CustomHttpGatherer(string key, string destination) : base(key, destination) { }
+
+    protected override Task<IEnumerable<JsonNode>> TransformResponse(HttpResponseMessage responseMessage)
+    {
+        // retrieve the response as a string from the HttpResponseMessage
+        // and parse it as a JsonNode enumerable.
+        return base.TransformResponse(responseMessage);
+    }
+}
+```
+<sup><a href='/src/Snippets/ScatterGather/TransformResponse.cs#L12-L24' title='Snippet source file'>snippet source</a> | <a href='#snippet-scatter-gather-transform-response' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->
+
+### Taking control of the downstream invocation process
+
+If transforming returned data is not enough, it's possible to take full control over the downstream service invocation process by overriding the `Gather` method:
+
+<!-- snippet: scatter-gather-gather-override -->
+<a id='snippet-scatter-gather-gather-override'></a>
+```cs
+public class CustomHttpGatherer : HttpGatherer
+{
+    public CustomHttpGatherer(string key, string destination) : base(key, destination) { }
+
+    public override Task<IEnumerable<JsonNode>> Gather(HttpContext context)
+    {
+        // by overriding this method we can implement custom logic
+        // to gather the responses from the downstream service.
+        return base.Gather(context);
+    }
+}
+```
+<sup><a href='/src/Snippets/ScatterGather/GatherMethodOverride.cs#L12-L24' title='Snippet source file'>snippet source</a> | <a href='#snippet-scatter-gather-gather-override' title='Start of snippet'>anchor</a></sup>
+<!-- endSnippet -->

src/ServiceComposer.AspNetCore.Tests/API/APIApprovals.Approve_API.verified.txt

@@ -60,6 +60,22 @@ namespace ServiceComposer.AspNetCore
         " removed in v3. Use attribute routing based composition, and CompositionEventHan" +
         "dler<TEvent>.", true)]
     public delegate System.Threading.Tasks.Task EventHandler<TEvent>(string requestId, [System.Runtime.CompilerServices.Dynamic] object viewModel, TEvent @event, Microsoft.AspNetCore.Routing.RouteData routeData, Microsoft.AspNetCore.Http.HttpRequest httpRequest);
+    public abstract class Gatherer
+    {
+        protected Gatherer(string key) { }
+        public string Key { get; }
+        public abstract System.Threading.Tasks.Task<System.Collections.Generic.IEnumerable<System.Text.Json.Nodes.JsonNode>> Gather(Microsoft.AspNetCore.Http.HttpContext context);
+    }
+    public class HttpGatherer : ServiceComposer.AspNetCore.Gatherer
+    {
+        public HttpGatherer(string key, string destinationUrl) { }
+        public System.Func<Microsoft.AspNetCore.Http.HttpRequest, string, string> DefaultDestinationUrlMapper { get; }
+        public string DestinationUrl { get; }
+        public System.Func<Microsoft.AspNetCore.Http.HttpRequest, string, string> DestinationUrlMapper { get; init; }
+        public override System.Threading.Tasks.Task<System.Collections.Generic.IEnumerable<System.Text.Json.Nodes.JsonNode>> Gather(Microsoft.AspNetCore.Http.HttpContext context) { }
+        protected virtual string MapDestinationUrl(Microsoft.AspNetCore.Http.HttpRequest request, string destination) { }
+        protected virtual System.Threading.Tasks.Task<System.Collections.Generic.IEnumerable<System.Text.Json.Nodes.JsonNode>> TransformResponse(System.Net.Http.HttpResponseMessage responseMessage) { }
+    }
     public static class HttpRequestExtensions
     {
         [return: System.Runtime.CompilerServices.Dynamic]
@@ -74,6 +90,11 @@ namespace ServiceComposer.AspNetCore
         public static System.Threading.Tasks.Task<T> Bind<T>(this Microsoft.AspNetCore.Http.HttpRequest request)
             where T : new() { }
     }
+    public interface IAggregator
+    {
+        void Add(System.Collections.Generic.IEnumerable<System.Text.Json.Nodes.JsonNode> nodes);
+        System.Threading.Tasks.Task<System.Text.Json.Nodes.JsonArray> Aggregate();
+    }
     public interface ICompositionContext
     {
         string RequestId { get; }
@@ -159,6 +180,16 @@ namespace ServiceComposer.AspNetCore
         public bool UseOutputFormatters { get; set; }
         public void UseCustomJsonSerializerSettings(System.Func<Microsoft.AspNetCore.Http.HttpRequest, Newtonsoft.Json.JsonSerializerSettings> jsonSerializerSettingsConfig) { }
     }
+    public static class ScatterGatherEndpointBuilderExtensions
+    {
+        public static Microsoft.AspNetCore.Builder.IEndpointConventionBuilder MapScatterGather(this Microsoft.AspNetCore.Routing.IEndpointRouteBuilder builder, string template, ServiceComposer.AspNetCore.ScatterGatherOptions options) { }
+    }
+    public class ScatterGatherOptions
+    {
+        public ScatterGatherOptions() { }
+        public System.Type CustomAggregator { get; set; }
+        public System.Collections.Generic.IList<ServiceComposer.AspNetCore.Gatherer> Gatherers { get; set; }
+    }
     public static class ServiceCollectionExtensions
     {
         public static void AddViewModelComposition(this Microsoft.Extensions.DependencyInjection.IServiceCollection services, Microsoft.Extensions.Configuration.IConfiguration configuration = null) { }

src/ServiceComposer.AspNetCore.Tests/ScatterGather/Get_with_2_gatherers.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.DependencyInjection;
+using ServiceComposer.AspNetCore.Testing;
+using Xunit;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using ServiceComposer.AspNetCore.Tests.Utils;
+
+namespace ServiceComposer.AspNetCore.Tests.ScatterGather;
+
+public class Get_with_2_gatherers
+{
+    [Fact]
+    public async Task Returns_expected_response()
+    {
+        // Arrange
+        var aSampleSourceClient = new SelfContainedWebApplicationFactoryWithWebHost<Dummy>
+        (
+            configureServices: services =>
+            {
+                services.AddRouting();
+            },
+            configure: app =>
+            {
+                app.UseRouting();
+                app.UseEndpoints(builder =>
+                {
+                    builder.MapGet("/samples/ASamplesSource", () =>
+                    {
+                        return new []{ new { Value = "ASample" } };
+                    });
+                });
+            }
+        ).CreateClient();
+
+        var anotherSampleSourceClient = new SelfContainedWebApplicationFactoryWithWebHost<Dummy>
+        (
+            configureServices: services =>
+            {
+                services.AddRouting();
+            },
+            configure: app =>
+            {
+                app.UseRouting();
+                app.UseEndpoints(builder =>
+                {
+                    builder.MapGet("/samples/AnotherSamplesSource", () =>
+                    {
+                        return new []{ new { Value = "AnotherSample" } };
+                    });
+                });
+            }
+        ).CreateClient();
+
+        var client = new SelfContainedWebApplicationFactoryWithWebHost<Dummy>
+        (
+            configureServices: services =>
+            {
+                HttpClient ClientProvider(string name) =>
+                    name switch
+                    {
+                        "ASamplesSource" => aSampleSourceClient,
+                        "AnotherSamplesSource" => anotherSampleSourceClient,
+                        _ => throw new NotSupportedException($"Missing HTTP client for {name}")
+                    };
+
+                // TODO: does this need to register a default HTTP client?
+                // services.AddScatterGather();
+                services.AddRouting();
+                services.Replace(
+                    new ServiceDescriptor(typeof(IHttpClientFactory), 
+                    new DelegateHttpClientFactory(ClientProvider)));
+            },
+            configure: app =>
+            {
+                app.UseRouting();
+                app.UseEndpoints(builder =>
+                {
+                    builder.MapScatterGather(template: "/samples", new ScatterGatherOptions
+                    {
+                        Gatherers = new List<Gatherer>
+                        {
+                            new HttpGatherer(key: "ASamplesSource", destinationUrl: "/samples/ASamplesSource"),
+                            new HttpGatherer(key: "AnotherSamplesSource", destinationUrl: "/samples/AnotherSamplesSource")
+                        }
+                    });
+                });
+            }
+        ).CreateClient();
+
+        // Act
+        var response = await client.GetAsync("/samples");
+
+        // Assert
+        Assert.True(response.IsSuccessStatusCode);
+
+        var responseString = await response.Content.ReadAsStringAsync();
+        var responseArray = JsonNode.Parse(responseString)!.AsArray();
+        var responseArrayAsJsonStrings = new HashSet<string>(responseArray.Select(n=>n.ToJsonString()));
+
+        var expectedArray = JsonNode.Parse(JsonSerializer.Serialize( new[]
+        {
+            new {Value = "ASample"},
+            new {Value = "AnotherSample"}
+        }, new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+        }))!.AsArray();
+        var expectedArrayAsJsonStrings = new HashSet<string>(expectedArray.Select(n=>n.ToJsonString()));
+
+        Assert.Equal(2, responseArray.Count);
+        Assert.Equivalent(expectedArrayAsJsonStrings, responseArrayAsJsonStrings);
+    }
+}
+