[BRE-1058] fix alpine race condition (#6156)

* alpine race condition during shutdown fix

* change catch to only be for relevant task cancelled, added a debug log

* test commit for build and test

* remove testing comment

src/Core/HostedServices/ApplicationCacheHostedService.cs

@@ -67,19 +67,25 @@ public class ApplicationCacheHostedService : IHostedService, IDisposable
 
     public virtual async Task StopAsync(CancellationToken cancellationToken)
     {
+        // Step 1: Signal ExecuteAsync to stop gracefully
+        _cts?.Cancel();
+
+        // Step 2: Wait for ExecuteAsync to finish cleanly
+        if (_executingTask != null)
+        {
+            await _executingTask;
+        }
+
+        // Step 3: Now safely dispose resources (ExecuteAsync is done)
         await _subscriptionReceiver.CloseAsync(cancellationToken);
         await _serviceBusClient.DisposeAsync();
-        _cts?.Cancel();
+
+        // Step 4: Clean up subscription
         try
         {
             await _serviceBusAdministrationClient.DeleteSubscriptionAsync(_topicName, _subName, cancellationToken);
         }
         catch { }
-
-        if (_executingTask != null)
-        {
-            await _executingTask;
-        }
     }
 
     public virtual void Dispose()
@@ -87,7 +93,18 @@ public class ApplicationCacheHostedService : IHostedService, IDisposable
 
     private async Task ExecuteAsync(CancellationToken cancellationToken)
     {
-        await foreach (var message in _subscriptionReceiver.ReceiveMessagesAsync(cancellationToken))
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            try
+            {
+                var messages = await _subscriptionReceiver.ReceiveMessagesAsync(
+                    maxMessages: 1,
+                    maxWaitTime: TimeSpan.FromSeconds(30),
+                    cancellationToken);
+
+                if (messages?.Any() == true)
+                {
+                    foreach (var message in messages)
                     {
                         try
                         {
@@ -99,6 +116,19 @@ public class ApplicationCacheHostedService : IHostedService, IDisposable
                         }
                     }
                 }
+            }
+            catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("ServiceBus receiver disposed during Alpine container shutdown");
+                break;
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("ServiceBus operation cancelled during Alpine container shutdown");
+                break;
+            }
+        }
+    }
 
     private async Task ProcessMessageAsync(ServiceBusReceivedMessage message, CancellationToken cancellationToken)
     {

src/EventsProcessor/AzureQueueHostedService.cs

@@ -86,12 +86,25 @@ public class AzureQueueHostedService : IHostedService, IDisposable
                     await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                 }
             }
+            catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                break;
+            }
             catch (Exception ex)
             {
                 _logger.LogError(ex, "Error occurred processing message block.");
 
+                try
+                {
                     await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                 }
+                catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+                {
+                    _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                    break;
+                }
+            }
         }
 
         _logger.LogWarning("Done processing messages.");

src/Notifications/AzureQueueHostedService.cs

@@ -87,6 +87,11 @@ public class AzureQueueHostedService : IHostedService, IDisposable
                     await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                 }
             }
+            catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                break;
+            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error processing messages.");