ci: Run the persistence contract tests

.github/actions/contract-tests/action.yml

@@ -15,6 +15,14 @@ inputs:
     description: 'Whether to run contract tests from the latest v3 release'
     required: false
     default: 'false'
+  enable_persistence_tests:
+    description: 'Whether to start Redis, DynamoDB and Consul on their default ports and run the persistent store tests.'
+    required: false
+    default: 'false'
+  test_service_port:
+    description: 'The port of the contract test service. A caller that enables the persistence tests must pass another port than 8000, the port that the harness expects DynamoDB on.'
+    required: false
+    default: '8000'
 
 runs:
   using: composite
@@ -36,10 +44,31 @@ runs:
       shell: bash
       run: dotnet build -p:Configuration=debug ${{ inputs.service_project_file }}
 
+    - name: Start Persistent Stores
+      if: inputs.enable_persistence_tests == 'true'
+      uses: launchdarkly/gh-actions/actions/persistent-stores@persistent-stores-v0
+      with:
+        redis: true
+        dynamodb: true
+        consul: true
+
+    # The stores start in the background. The harness's DynamoDB setup fails the whole DynamoDB suite if the store
+    # does not answer yet, and Consul rejects writes until it has elected a leader.
+    - name: Wait for Persistent Stores
+      if: inputs.enable_persistence_tests == 'true'
+      shell: bash
+      run: |
+        for port in 6379 8000 8500; do
+          timeout 120 bash -c "until (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; do sleep 1; done" \
+            || { echo "The store on port $port did not start"; exit 1; }
+        done
+        timeout 120 bash -c 'until curl -sf http://127.0.0.1:8500/v1/status/leader | grep -q ":"; do sleep 1; done' \
+          || { echo "Consul did not elect a leader"; exit 1; }
+
     - name: Launch Contract Tests
       id: launch-contract-tests
       shell: bash
-      run: dotnet ${{ inputs.service_dll_file }} > test-service.log 2>&1 & disown
+      run: dotnet ${{ inputs.service_dll_file }} ${{ inputs.test_service_port }} > test-service.log 2>&1 & disown
 
     - name: Suppression Path
       shell: bash
@@ -50,8 +79,8 @@ runs:
     - name: Run Contract Tests
       uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1
       with:
-        test_service_port: 8000
-        enable_persistence_tests: 'false'
+        test_service_port: ${{ inputs.test_service_port }}
+        enable_persistence_tests: ${{ inputs.enable_persistence_tests }}
         # Leave the service up for the FDv2 run that follows.
         stop_service: ${{ inputs.run_fdv2_tests == 'true' && 'false' || 'true' }}
         extra_params: '-status-timeout=360 -skip-from=${{ env.SUPPRESSION_FILE }}'
@@ -61,10 +90,10 @@ runs:
       if: inputs.run_fdv2_tests == 'true'
       uses: launchdarkly/gh-actions/actions/contract-tests@contract-tests-v1
       with:
-        test_service_port: 8000
+        test_service_port: ${{ inputs.test_service_port }}
         version: 'v3'
         debug_logging: 'true'
-        enable_persistence_tests: 'false'
+        enable_persistence_tests: ${{ inputs.enable_persistence_tests }}
         extra_params: '-status-timeout=360 -skip-from=${{ env.SUPPRESSION_FILE_FDV2 }}'
         token: ${{ inputs.token }}
 

.github/workflows/sdk-server-ci.yml

@@ -37,6 +37,11 @@ jobs:
           service_dll_file: ${{ env.CONTRACT_TEST_DLL_FILE}}
           token: ${{ secrets.GITHUB_TOKEN }}
           run_fdv2_tests: 'true'
+          # The persistence tests run on the Linux runner only, where the stores start quickly in Docker. Their
+          # behavior does not depend on the platform.
+          enable_persistence_tests: ${{ runner.os == 'Linux' && 'true' || 'false' }}
+          # DynamoDB uses port 8000, the service's default port.
+          test_service_port: '9000'
 
       - uses: ./.github/actions/build-docs
         with:

pkgs/sdk/server/contract-tests/README.md

@@ -7,3 +7,7 @@ To run these tests locally, run `make contract-tests` from the SDK project root
 Or, to test against an in-progress local version of the test harness, run `make start-contract-test-service` from the SDK project root directory; then, in the root directory of the `sdk-test-harness` project, build the test harness and run it from the command line.
 
 Currently, the project does _not_ automatically detect the available target frameworks. It will default to building and running for .NET 8.0. To use a different target framework, set the environment variable `TESTFRAMEWORK` to the name of the application runtime framework (such as `net8.0`), and set the environment variable `BUILDFRAMEWORKS` (note the S at the end) to the target framework that the SDK should be built for (which may or may not be the same).
+
+The service compiles the sources of the Redis, DynamoDB, and Consul integrations in this repository, so that they run against the local SDK rather than the published SDK package that their projects reference.
+
+To run the persistent store tests, start the stores on their default ports, for example with `docker run -p 127.0.0.1:6379:6379 redis`, `docker run -p 127.0.0.1:8000:8000 amazon/dynamodb-local`, and `docker run -p 127.0.0.1:8500:8500 hashicorp/consul`, and pass `-enable-persistence-tests` to the test harness. DynamoDB uses port 8000, the service's default port, so pass another port as the service's first argument, for example `dotnet bin/Debug/net8.0/ContractTestService.dll 9000`.

pkgs/sdk/server/contract-tests/Representations.cs

@@ -36,6 +36,7 @@ public class SdkConfigParams
         public SdkHookParams Hooks { get; set; }
         public SdkConfigDataSystemParams DataSystem { get; set; }
         public SdkConfigOverridesParams Overrides { get; set; }
+        public SdkConfigPersistentDataStoreParams PersistentDataStore { get; set; }
     }
 
     public class SdkConfigOverridesParams

pkgs/sdk/server/contract-tests/SdkClientEntity.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
+using Amazon.DynamoDBv2;
+using Amazon.Runtime;
 using LaunchDarkly.Logging;
 using LaunchDarkly.Sdk;
 using LaunchDarkly.Sdk.Json;
@@ -436,6 +438,18 @@ private static Configuration BuildSdkConfig(SdkConfigParams sdkParams, ILogAdapt
                 builder.DataSource(dataSource);
             }
 
+            if (sdkParams.PersistentDataStore != null)
+            {
+                builder.DataStore(CreatePersistentDataStore(sdkParams.PersistentDataStore));
+                // With a persistent store and no data source, the SDK runs in daemon mode: it reads the data
+                // that another process, such as the Relay Proxy, writes to the store.
+                if (streamingParams == null && pollingParams == null &&
+                    serviceEndpointsParams?.Streaming == null && serviceEndpointsParams?.Polling == null)
+                {
+                    builder.DataSource(Components.ExternalUpdatesOnly);
+                }
+            }
+
             var eventParams = sdkParams.Events;
             if (eventParams == null)
             {
@@ -516,81 +530,14 @@ private static Configuration BuildSdkConfig(SdkConfigParams sdkParams, ILogAdapt
             {
                 var dataSystemBuilder = Components.DataSystem().Custom();
 
-                // TODO: re-enable this code in the future and determine which dependencies on persistent stores need to be added to contract test build process
-                // Configure persistent store if provided
-                // if (sdkParams.DataSystem.Store?.PersistentDataStore != null)
-                // {
-                //     var storeConfig = sdkParams.DataSystem.Store.PersistentDataStore;
-                //     var storeType = storeConfig.Store.Type.ToLower();
-                //     IComponentConfigurer<IDataStore> persistentStore = null;
-
-                //     PersistentDataStoreBuilder persistentStoreBuilder = null;
-
-                //     switch (storeType)
-                //     {
-                //         case "redis":
-                //             var redisBuilder = Redis.DataStore();
-                //             if (!string.IsNullOrEmpty(storeConfig.Store.DSN))
-                //             {
-                //                 redisBuilder.Uri(storeConfig.Store.DSN);
-                //             }
-                //             if (!string.IsNullOrEmpty(storeConfig.Store.Prefix))
-                //             {
-                //                 redisBuilder.Prefix(storeConfig.Store.Prefix);
-                //             }
-                //             persistentStoreBuilder = Components.PersistentDataStore(redisBuilder);
-                //             break;
-
-                //         case "dynamodb":
-                //             // For DynamoDB, DSN is the table name
-                //             var dynamoBuilder = DynamoDB.DataStore(storeConfig.Store.DSN ?? "sdk-contract-tests");
-                //             if (!string.IsNullOrEmpty(storeConfig.Store.Prefix))
-                //             {
-                //                 dynamoBuilder.Prefix(storeConfig.Store.Prefix);
-                //             }
-                //             // DynamoDB uses IPersistentDataStoreAsync, so use the async overload
-                //             persistentStoreBuilder = Components.PersistentDataStore(dynamoBuilder);
-                //             break;
-
-                //         case "consul":
-                //             var consulBuilder = Consul.DataStore();
-                //             if (!string.IsNullOrEmpty(storeConfig.Store.DSN))
-                //             {
-                //                 consulBuilder.Address(storeConfig.Store.DSN);
-                //             }
-                //             if (!string.IsNullOrEmpty(storeConfig.Store.Prefix))
-                //             {
-                //                 consulBuilder.Prefix(storeConfig.Store.Prefix);
-                //             }
-                //             persistentStoreBuilder = Components.PersistentDataStore(consulBuilder);
-                //             break;
-                //     }
-
-                //     if (persistentStoreBuilder != null)
-                //     {
-                //         // Configure cache
-                //         var cacheMode = storeConfig.Cache?.Mode?.ToLower();
-                //         if (cacheMode == "off")
-                //         {
-                //             persistentStoreBuilder.NoCaching();
-                //         }
-                //         else if (cacheMode == "ttl" && storeConfig.Cache.TTL.HasValue)
-                //         {
-                //             persistentStoreBuilder.CacheTime(TimeSpan.FromSeconds(storeConfig.Cache.TTL.Value));
-                //         }
-                //         else if (cacheMode == "infinite")
-                //         {
-                //             persistentStoreBuilder.CacheForever();
-                //         }
-
-                //         // Determine store mode
-                //         var storeMode = sdkParams.DataSystem.StoreMode == 0
-                //             ? DataSystemConfiguration.DataStoreMode.ReadOnly 
-                //             : DataSystemConfiguration.DataStoreMode.ReadWrite;
-
-                //         dataSystemBuilder.PersistentStore(persistentStoreBuilder, storeMode);
-                //     }
-                // }
+                var storeParams = sdkParams.DataSystem.Store?.PersistentDataStore;
+                if (storeParams != null)
+                {
+                    var storeMode = sdkParams.DataSystem.StoreMode == StoreMode.ReadWrite
+                        ? DataSystemConfiguration.DataStoreMode.ReadWrite
+                        : DataSystemConfiguration.DataStoreMode.ReadOnly;
+                    dataSystemBuilder.PersistentStore(CreatePersistentDataStore(storeParams), storeMode);
+                }
 
                 // Configure initializers
                 if (sdkParams.DataSystem.Initializers != null && sdkParams.DataSystem.Initializers.Length > 0)
@@ -721,6 +668,89 @@ private static FileOverrideSourceBuilder BuildOverrideSource(SdkConfigOverridesP
             return source;
         }
 
+        // The Redis and DynamoDB clients are configured so that a failed operation surfaces well within a
+        // second, as the harness's store recovery tests require: no retries, and short timeouts. The Consul
+        // client does not retry.
+        private static readonly TimeSpan StoreOperationTimeout = TimeSpan.FromMilliseconds(500);
+
+        // The table that the test harness creates in DynamoDB.
+        private const string DynamoDBTableName = "sdk-contract-tests";
+
+        private static PersistentDataStoreBuilder CreatePersistentDataStore(SdkConfigPersistentDataStoreParams storeParams)
+        {
+            var store = storeParams.Store;
+            if (string.IsNullOrEmpty(store.DSN))
+            {
+                throw new ArgumentException("The persistent store configuration has no DSN");
+            }
+            PersistentDataStoreBuilder builder;
+            switch (store.Type)
+            {
+                case PersistentStoreType.Redis:
+                    var redis = Redis.DataStore()
+                        .Uri(store.DSN)
+                        .ConnectTimeout(StoreOperationTimeout)
+                        .OperationTimeout(StoreOperationTimeout);
+                    if (!string.IsNullOrEmpty(store.Prefix))
+                    {
+                        redis.Prefix(store.Prefix);
+                    }
+                    builder = Components.PersistentDataStore(redis);
+                    break;
+
+                case PersistentStoreType.DynamoDB:
+                    // The DSN is the endpoint of the local DynamoDB instance.
+                    var dynamoDB = DynamoDB.DataStore(DynamoDBTableName)
+                        .Configuration(new AmazonDynamoDBConfig
+                        {
+                            ServiceURL = store.DSN,
+                            AuthenticationRegion = "us-east-1",
+                            MaxErrorRetry = 0,
+                            Timeout = StoreOperationTimeout
+                        })
+                        .Credentials(new BasicAWSCredentials("dummy", "dummy"));
+                    if (!string.IsNullOrEmpty(store.Prefix))
+                    {
+                        dynamoDB.Prefix(store.Prefix);
+                    }
+                    builder = Components.PersistentDataStore(dynamoDB);
+                    break;
+
+                case PersistentStoreType.Consul:
+                    // The DSN is a host and port, without a scheme. The Consul package's namespace is also
+                    // named Consul, so the SDK's class is named in full.
+                    var consul = LaunchDarkly.Sdk.Server.Integrations.Consul.DataStore()
+                        .Address(store.DSN.Contains("://") ? store.DSN : "http://" + store.DSN);
+                    if (!string.IsNullOrEmpty(store.Prefix))
+                    {
+                        consul.Prefix(store.Prefix);
+                    }
+                    builder = Components.PersistentDataStore(consul);
+                    break;
+
+                default:
+                    throw new ArgumentException("Unsupported persistent store type: " + store.Type);
+            }
+
+            switch (storeParams.Cache?.Mode)
+            {
+                case PersistentCacheMode.Off:
+                    builder.NoCaching();
+                    break;
+                case PersistentCacheMode.TTL:
+                    if (storeParams.Cache.TTL.HasValue)
+                    {
+                        builder.CacheTime(TimeSpan.FromSeconds(storeParams.Cache.TTL.Value));
+                    }
+                    break;
+                case PersistentCacheMode.Infinite:
+                    builder.CacheForever();
+                    break;
+            }
+
+            return builder;
+        }
+
         private static IComponentConfigurer<IDataSource> CreateSynchronizer(
             SdkConfigDataSynchronizerParams synchronizer,
             string payloadFilter)

pkgs/sdk/server/contract-tests/Stores/Consul/ContractTestService.Consul.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!-- The Consul integration of this repository, built from its sources against the local SDK for the contract test
+       service. The integration's own project references the published SDK package. -->
+  <PropertyGroup>
+    <TestFramework Condition="'$(TESTFRAMEWORK)' == ''">net8.0</TestFramework>
+    <TargetFrameworks>$(TESTFRAMEWORK)</TargetFrameworks>
+    <AssemblyName>ContractTestService.Consul</AssemblyName>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\..\..\..\dotnet-server-sdk-consul\src\**\*.cs" Exclude="..\..\..\..\..\dotnet-server-sdk-consul\src\bin\**;..\..\..\..\..\dotnet-server-sdk-consul\src\obj\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\LaunchDarkly.ServerSdk.csproj" />
+    <!-- The same version range as the integration's project. -->
+    <PackageReference Include="Consul" Version="[1.6.1.1,]" />
+  </ItemGroup>
+
+</Project>

pkgs/sdk/server/contract-tests/Stores/DynamoDB/ContractTestService.DynamoDB.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!-- The DynamoDB integration of this repository, built from its sources against the local SDK for the contract test
+       service. The integration's own project references the published SDK package. -->
+  <PropertyGroup>
+    <TestFramework Condition="'$(TESTFRAMEWORK)' == ''">net8.0</TestFramework>
+    <TargetFrameworks>$(TESTFRAMEWORK)</TargetFrameworks>
+    <AssemblyName>ContractTestService.DynamoDB</AssemblyName>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\..\..\..\dotnet-server-sdk-dynamodb\src\**\*.cs" Exclude="..\..\..\..\..\dotnet-server-sdk-dynamodb\src\AssemblyInfo.cs;..\..\..\..\..\dotnet-server-sdk-dynamodb\src\bin\**;..\..\..\..\..\dotnet-server-sdk-dynamodb\src\obj\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\LaunchDarkly.ServerSdk.csproj" />
+    <!-- The same version range as the integration's project. -->
+    <PackageReference Include="AWSSDK.DynamoDBv2" Version="[4.0.0.0,5.0.0.0)" />
+  </ItemGroup>
+
+</Project>

pkgs/sdk/server/contract-tests/Stores/Redis/ContractTestService.Redis.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <!-- The Redis integration of this repository, built from its sources against the local SDK for the contract test
+       service. The integration's own project references the published SDK package. -->
+  <PropertyGroup>
+    <TestFramework Condition="'$(TESTFRAMEWORK)' == ''">net8.0</TestFramework>
+    <TargetFrameworks>$(TESTFRAMEWORK)</TargetFrameworks>
+    <AssemblyName>ContractTestService.Redis</AssemblyName>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\..\..\..\dotnet-server-sdk-redis\src\**\*.cs" Exclude="..\..\..\..\..\dotnet-server-sdk-redis\src\AssemblyInfo.cs;..\..\..\..\..\dotnet-server-sdk-redis\src\bin\**;..\..\..\..\..\dotnet-server-sdk-redis\src\obj\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\LaunchDarkly.ServerSdk.csproj" />
+    <!-- The same version range as the integration's project. -->
+    <PackageReference Include="StackExchange.Redis" Version="[2.0.513,]" />
+  </ItemGroup>
+
+</Project>

pkgs/sdk/server/contract-tests/TestService.cs

@@ -12,7 +12,8 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            // An optional first argument selects the port, so that several test services can run at once.
+            // An optional first argument selects the port. In CI, port 8000 belongs to the local DynamoDB
+            // instance that the persistent store tests use.
             var port = args.Length > 0 ? int.Parse(args[0]) : 8000;
             var quitSignal = new EventWaitHandle(false, EventResetMode.AutoReset);
             var app = new Webapp(quitSignal);
@@ -52,7 +53,11 @@ public class Webapp
             "instance-id",
             "retry-conformance-fdv1-streaming",
             "retry-conformance-fdv1-polling",
-            "flag-overrides"
+            "flag-overrides",
+            "persistent-data-store-redis",
+            "persistent-data-store-dynamodb",
+            "persistent-data-store-consul",
+            "persistent-data-store-recovery"
         };
 
         public readonly Handler Handler;