Centralize shutdown data-finalization timeout and add forced-shutdown path

Add configurable data-finalization timeout and wire it through CLI, defaults.conf,
and GarnetServerOptions so shutdown behavior stays consistent across hosts.

- Add --data-finalization-timeout / DataFinalizationTimeoutSeconds (default 15s)
- Replace hardcoded 15s finalize CTS in ShutdownAsync with linked caller token
  plus configured cap; skip persistence on noSave or forced cancellation
- Define defaults on GarnetServerOptions (DefaultShutdownTimeoutSeconds,
  DefaultDataFinalizationTimeoutSeconds) as single source of truth
- Windows service: HostOptions.ShutdownTimeout = drain + finalize + 5s margin
  (replaces fixed 20s buffer); pre-parse both timeout flags
- Console host: second Ctrl+C / ProcessExit during shutdown cancels persistence
  (Redis-like fast path) via shutdownForceCts
- Tests: forced cancellation, noSave, and AOF shutdown coverage; fix CS1998
  by using void tests with Assert.DoesNotThrowAsync

Author: yuseok-kim-edushare

hosting/Windows/Garnet.worker/Program.cs

@@ -3,30 +3,32 @@
 
 using System;
 using Garnet;
+using Garnet.server;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
 class Program
 {
-    // Data finalization (AOF commit / checkpoint) uses up to 15 seconds internally (see GarnetServer.FinalizeDataAsync).
-    // Add this buffer on top of the connection-drain timeout so the host shutdown budget covers the full shutdown sequence.
-    private const int DataFinalizationBufferSeconds = 20;
+    // Extra host budget beyond connection drain + data finalization (quiesce, stop listening, etc.).
+    private const int HostShutdownMarginSeconds = 5;
 
     static void Main(string[] args)
     {
-        // Pre-parse only the shutdown-timeout argument so we can configure both
-        // the host shutdown budget and the Worker's connection-drain timeout from a single value.
-        var shutdownTimeoutSeconds = ParseShutdownTimeoutSeconds(args, defaultSeconds: 5);
+        // Pre-parse shutdown-related arguments so we can configure both the host shutdown budget
+        // and the Worker's connection-drain timeout before the generic host is built.
+        var shutdownTimeoutSeconds = ParseIntOption(args, "--shutdown-timeout", "-shutdown-timeout", GarnetServerOptions.DefaultShutdownTimeoutSeconds);
+        var dataFinalizationTimeoutSeconds = ParseIntOption(args, "--data-finalization-timeout", "-data-finalization-timeout", GarnetServerOptions.DefaultDataFinalizationTimeoutSeconds);
         var shutdownTimeout = TimeSpan.FromSeconds(shutdownTimeoutSeconds);
 
         var builder = Host.CreateApplicationBuilder(args);
 
         // Tell the .NET host (and the Windows SCM via WindowsServiceLifetime) how long to wait
-        // before forcibly killing the process. We add DataFinalizationBufferSeconds so that AOF
-        // commit / checkpoint can complete after connection draining finishes.
+        // before forcibly killing the process: drain + data finalization + margin.
         builder.Services.Configure<HostOptions>(opts =>
         {
-            opts.ShutdownTimeout = shutdownTimeout + TimeSpan.FromSeconds(DataFinalizationBufferSeconds);
+            opts.ShutdownTimeout = shutdownTimeout
+                + TimeSpan.FromSeconds(dataFinalizationTimeoutSeconds)
+                + TimeSpan.FromSeconds(HostShutdownMarginSeconds);
         });
 
         builder.Services.AddHostedService(_ => new Worker(args, shutdownTimeout));
@@ -41,15 +43,14 @@ static void Main(string[] args)
     }
 
     /// <summary>
-    /// Scans <paramref name="args"/> for <c>--shutdown-timeout &lt;value&gt;</c> and returns
-    /// the parsed integer, or <paramref name="defaultSeconds"/> if the argument is absent or invalid.
-    /// This lightweight pre-parse avoids a full CommandLineParser pass before the host is built.
+    /// Scans <paramref name="args"/> for <paramref name="longName"/> (or <paramref name="shortName"/>) and returns
+    /// the parsed positive integer, or <paramref name="defaultSeconds"/> if the argument is absent or invalid.
     /// </summary>
-    private static int ParseShutdownTimeoutSeconds(string[] args, int defaultSeconds)
+    private static int ParseIntOption(string[] args, string longName, string shortName, int defaultSeconds)
     {
         for (var i = 0; i < args.Length - 1; i++)
         {
-            if (args[i] is "--shutdown-timeout" or "-shutdown-timeout" &&
+            if (args[i] is var name && (name == longName || name == shortName) &&
                 int.TryParse(args[i + 1], out var value) && value > 0)
             {
                 return value;

hosting/Windows/Garnet.worker/Worker.cs

@@ -19,8 +19,8 @@ public class Worker : BackgroundService
         /// <param name="args">Command line arguments forwarded to <see cref="GarnetServer"/>.</param>
         /// <param name="shutdownTimeout">
         /// How long to wait for active connections to drain during graceful shutdown.
-        /// Must be less than the host <see cref="Microsoft.Extensions.Hosting.HostOptions.ShutdownTimeout"/>
-        /// so that data finalization (AOF commit / checkpoint) can also complete within the host budget.
+        /// Must fit within the host <see cref="Microsoft.Extensions.Hosting.HostOptions.ShutdownTimeout"/>,
+        /// which also budgets for data finalization and a small margin (see <c>Program.cs</c>).
         /// </param>
         public Worker(string[] args, TimeSpan shutdownTimeout)
         {

libs/host/Configuration/Options.cs

@@ -382,6 +382,10 @@ internal sealed class Options : ICloneable
             "The Windows SCM default pre-kill wait is 5 seconds, so values below 5 are not recommended when running as a Windows service.")]
         public int ShutdownTimeoutSeconds { get; set; }
 
+        [IntRangeValidation(1, int.MaxValue)]
+        [Option("data-finalization-timeout", Required = false, HelpText = "Timeout in seconds for AOF commit and checkpoint during graceful shutdown data finalization.")]
+        public int DataFinalizationTimeoutSeconds { get; set; }
+
         [OptionValidation]
         [Option("use-azure-storage", Required = false, HelpText = "Use Azure Page Blobs for storage instead of local storage.")]
         public bool? UseAzureStorage { get; set; }
@@ -924,6 +928,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno
                 ThreadPoolMaxIOCompletionThreads = ThreadPoolMaxIOCompletionThreads,
                 NetworkConnectionLimit = NetworkConnectionLimit,
                 ShutdownTimeoutSeconds = ShutdownTimeoutSeconds,
+                DataFinalizationTimeoutSeconds = DataFinalizationTimeoutSeconds,
                 DeviceFactoryCreator = deviceType == DeviceType.AzureStorage ? azureFactoryCreator()
                     : new LocalStorageNamedDeviceFactoryCreator(
                         deviceType: deviceType,

libs/host/GarnetServer.cs

@@ -75,7 +75,12 @@ static string GetVersion()
         /// <summary>
         /// Configured shutdown drain timeout in seconds.
         /// </summary>
-        public int ShutdownTimeoutSeconds => opts.ShutdownTimeoutSeconds > 0 ? opts.ShutdownTimeoutSeconds : 5;
+        public int ShutdownTimeoutSeconds => opts.ShutdownTimeoutSeconds > 0 ? opts.ShutdownTimeoutSeconds : GarnetServerOptions.DefaultShutdownTimeoutSeconds;
+
+        /// <summary>
+        /// Configured data-finalization timeout in seconds (AOF commit / checkpoint during shutdown).
+        /// </summary>
+        public int DataFinalizationTimeoutSeconds => opts.DataFinalizationTimeoutSeconds > 0 ? opts.DataFinalizationTimeoutSeconds : GarnetServerOptions.DefaultDataFinalizationTimeoutSeconds;
 
         /// <summary>
         /// Create Garnet Server instance using specified command line arguments; use Start to start the server.
@@ -507,11 +512,12 @@ public void Start()
         /// </summary>
         /// <param name="timeout">Timeout for waiting on active connections (default: configured <see cref="ShutdownTimeoutSeconds"/> value)</param>
         /// <param name="noSave">If true, skip data persistence (AOF commit and checkpoint) during shutdown</param>
-        /// <param name="token">Cancellation token</param>
+        /// <param name="token">Cancellation token; when cancelled (e.g. second Ctrl+C), connection draining and data finalization are aborted</param>
         /// <returns>Task representing the async shutdown operation</returns>
         public async Task ShutdownAsync(TimeSpan? timeout = null, bool noSave = false, CancellationToken token = default)
         {
             var shutdownTimeout = timeout ?? TimeSpan.FromSeconds(ShutdownTimeoutSeconds);
+            var skipPersistence = noSave;
 
             try
             {
@@ -534,9 +540,14 @@ public async Task ShutdownAsync(TimeSpan? timeout = null, bool noSave = false, C
                 {
                     await WaitForActiveConnectionsAsync(shutdownTimeout, token).ConfigureAwait(false);
                 }
+                catch (OperationCanceledException) when (token.IsCancellationRequested)
+                {
+                    skipPersistence = true;
+                    logger?.LogWarning("Connection draining was cancelled due to forced shutdown.");
+                }
                 catch (OperationCanceledException)
                 {
-                    logger?.LogWarning("Connection draining was cancelled. Proceeding with data finalization...");
+                    logger?.LogWarning("Connection draining timed out. Proceeding with data finalization...");
                 }
             }
             catch (Exception ex)
@@ -545,25 +556,34 @@ public async Task ShutdownAsync(TimeSpan? timeout = null, bool noSave = false, C
             }
             finally
             {
-                if (!noSave)
+                if (skipPersistence)
+                {
+                    logger?.LogInformation("Skipping data persistence during shutdown.");
+                }
+                else
                 {
                     // Attempt AOF commit or checkpoint as best-effort,
-                    // even if connection draining was cancelled or failed.
-                    // Use a bounded timeout instead of the caller's token to ensure completion.
-                    using var finalizeCts = new CancellationTokenSource(TimeSpan.FromSeconds(15));
+                    // even if connection draining timed out. Honor caller cancellation (forced shutdown)
+                    // and cap total finalize time with DataFinalizationTimeoutSeconds.
+                    using var finalizeCts = CancellationTokenSource.CreateLinkedTokenSource(token);
+                    finalizeCts.CancelAfter(TimeSpan.FromSeconds(DataFinalizationTimeoutSeconds));
                     try
                     {
                         await FinalizeDataAsync(finalizeCts.Token).ConfigureAwait(false);
                     }
+                    catch (OperationCanceledException) when (token.IsCancellationRequested)
+                    {
+                        logger?.LogWarning("Data finalization was cancelled due to forced shutdown.");
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        logger?.LogWarning("Data finalization timed out after {TimeoutSeconds} seconds.", DataFinalizationTimeoutSeconds);
+                    }
                     catch (Exception ex)
                     {
                         logger?.LogError(ex, "Error during data finalization");
                     }
                 }
-                else
-                {
-                    logger?.LogInformation("Shutdown with noSave flag - skipping data persistence.");
-                }
             }
         }
 

libs/host/defaults.conf

@@ -282,6 +282,9 @@
 	/* Timeout in seconds to wait for active connections to drain during graceful shutdown. */
 	"ShutdownTimeoutSeconds" : 5,
 
+	/* Timeout in seconds for AOF commit and checkpoint during graceful shutdown data finalization. */
+	"DataFinalizationTimeoutSeconds" : 15,
+
 	/* Use Azure Page Blobs for storage instead of local storage. */
 	"UseAzureStorage" : false,
 

libs/server/Servers/GarnetServerOptions.cs

@@ -268,10 +268,25 @@ public class GarnetServerOptions : ServerOptions
         /// </summary>
         public bool QuietMode = false;
 
+        /// <summary>
+        /// Default connection-drain timeout in seconds when not configured.
+        /// </summary>
+        public const int DefaultShutdownTimeoutSeconds = 5;
+
+        /// <summary>
+        /// Default data-finalization timeout in seconds when not configured.
+        /// </summary>
+        public const int DefaultDataFinalizationTimeoutSeconds = 15;
+
         /// <summary>
         /// Timeout (in seconds) for waiting on active connections to drain during graceful shutdown.
         /// </summary>
-        public int ShutdownTimeoutSeconds = 5;
+        public int ShutdownTimeoutSeconds = DefaultShutdownTimeoutSeconds;
+
+        /// <summary>
+        /// Timeout (in seconds) for AOF commit and checkpoint during graceful shutdown data finalization.
+        /// </summary>
+        public int DataFinalizationTimeoutSeconds = DefaultDataFinalizationTimeoutSeconds;
 
         /// <summary>
         /// SAVE and BGSAVE: We will take a full (index + log) checkpoint when ReadOnlyAddress of log increases by this amount, from the last full checkpoint.

main/GarnetServer/Program.cs

@@ -22,28 +22,35 @@ static async Task Main(string[] args)
                 // Start the server
                 server.Start();
 
-                using var cts = new CancellationTokenSource();
+                using var runCts = new CancellationTokenSource();
+                using var shutdownForceCts = new CancellationTokenSource();
+                var shutdownSignals = 0;
 
-                ConsoleCancelEventHandler cancelKeyPressHandler = (sender, e) =>
+                void OnShutdownSignal()
                 {
-                    e.Cancel = true; // Prevent the process from terminating immediately
-                    if (!cts.IsCancellationRequested)
-                        cts.Cancel();
-                };
+                    // First signal: graceful shutdown. Second signal (or ProcessExit while shutting down): fast path.
+                    if (Interlocked.Increment(ref shutdownSignals) >= 2)
+                        shutdownForceCts.Cancel();
 
-                EventHandler processExitHandler = (sender, e) =>
-                {
                     try
                     {
-                        if (!cts.IsCancellationRequested)
-                            cts.Cancel();
+                        if (!runCts.IsCancellationRequested)
+                            runCts.Cancel();
                     }
                     catch (ObjectDisposedException)
                     {
                         // The cancellation source may already be disposed during process teardown.
                     }
+                }
+
+                ConsoleCancelEventHandler cancelKeyPressHandler = (sender, e) =>
+                {
+                    e.Cancel = true; // Prevent the process from terminating immediately
+                    OnShutdownSignal();
                 };
 
+                EventHandler processExitHandler = (sender, e) => OnShutdownSignal();
+
                 // Signal cancellation on Ctrl+C; avoid blocking the event handler with async work
                 Console.CancelKeyPress += cancelKeyPressHandler;
 
@@ -55,15 +62,14 @@ static async Task Main(string[] args)
                 {
                     try
                     {
-                        await Task.Delay(Timeout.Infinite, cts.Token).ConfigureAwait(false);
+                        await Task.Delay(Timeout.Infinite, runCts.Token).ConfigureAwait(false);
                     }
                     catch (OperationCanceledException)
                     {
-                        // Graceful shutdown: drain connections, commit AOF, take checkpoint
+                        // Graceful shutdown on first signal; second signal cancels persistence via shutdownForceCts.
                         await server.ShutdownAsync(
                             TimeSpan.FromSeconds(server.ShutdownTimeoutSeconds),
-                            token: CancellationToken.None
-                            ).ConfigureAwait(false);
+                            token: shutdownForceCts.Token).ConfigureAwait(false);
                     }
                 }
                 finally

test/standalone/Garnet.test/GarnetServerTcpTests.cs

@@ -167,24 +167,38 @@ public async Task ShutdownAsyncRespectsTimeout()
         }
 
         [Test]
-        public async Task ShutdownAsyncRespectsCancellation()
+        public void ShutdownAsyncRespectsCancellation()
         {
             // Arrange
             using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig());
             redis.GetDatabase(0).Ping();
 
             using var cts = new CancellationTokenSource();
 
-            // Act - Cancel immediately
+            // Act - Cancel immediately (forced shutdown: skip data finalization)
             cts.Cancel();
+            var sw = System.Diagnostics.Stopwatch.StartNew();
             Assert.DoesNotThrowAsync(async () =>
             {
                 await server.ShutdownAsync(timeout: TimeSpan.FromSeconds(30), token: cts.Token).ConfigureAwait(false);
             });
+            sw.Stop();
+
+            // Assert - Should not wait for the full data-finalization timeout
+            ClassicAssert.Less(sw.ElapsedMilliseconds, 5_000);
+        }
+
+        [Test]
+        public void ShutdownAsyncNoSaveSkipsPersistence()
+        {
+            Assert.DoesNotThrowAsync(async () =>
+            {
+                await server.ShutdownAsync(timeout: TimeSpan.FromSeconds(5), noSave: true).ConfigureAwait(false);
+            });
         }
 
         [Test]
-        public async Task ShutdownAsyncWithAofCommit()
+        public void ShutdownAsyncWithAofCommit()
         {
             // Arrange - Create server with AOF enabled
             server?.Dispose();
@@ -201,7 +215,7 @@ public async Task ShutdownAsyncWithAofCommit()
                 db.StringSet($"aof-key-{i}", $"value-{i}");
             }
 
-            // Act - Shutdown should commit AOF without errors
+            // Act & Assert - Shutdown should commit AOF without errors
             Assert.DoesNotThrowAsync(async () =>
             {
                 await server.ShutdownAsync(timeout: TimeSpan.FromSeconds(5)).ConfigureAwait(false);