Move long-running delay task fix to AsyncDelayer

Fixes #1985
This commit is contained in:
Jordan Dominion
2024-11-03 10:28:09 -05:00
parent d98bc2918e
commit be6960d942
17 changed files with 71 additions and 47 deletions
@@ -710,7 +710,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
Task.WhenAll(
disconnectTask,
listenTask ?? Task.CompletedTask),
AsyncDelayer.Delay(TimeSpan.FromSeconds(5), cancellationToken));
AsyncDelayer.Delay(TimeSpan.FromSeconds(5), cancellationToken).AsTask());
}
/// <summary>
@@ -135,7 +135,7 @@ namespace Tgstation.Server.Host.Components.Engine
const int MaximumTerminationSeconds = 5;
logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds);
var timeout = asyncDelayer.Delay(TimeSpan.FromSeconds(MaximumTerminationSeconds), cancellationToken);
var timeout = asyncDelayer.Delay(TimeSpan.FromSeconds(MaximumTerminationSeconds), cancellationToken).AsTask();
var lifetime = process.Lifetime;
if (lifetime.IsCompleted)
logger.LogTrace("Robust.Server already exited");
@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
@@ -522,30 +521,6 @@ namespace Tgstation.Server.Host.Components
logger.LogInformation("Next auto-update will occur at {time}", DateTimeOffset.UtcNow + delay);
// https://learn.microsoft.com/en-us/dotnet/api/system.threading.tasks.task.delay?view=net-8.0#system-threading-tasks-task-delay(system-timespan)
const uint DelayMinutesLimit = UInt32.MaxValue - 1;
Debug.Assert(DelayMinutesLimit == 4294967294, "Delay limit assertion failure!");
var maxDelayIterations = 0UL;
if (delay.TotalMilliseconds >= UInt32.MaxValue)
{
maxDelayIterations = (ulong)Math.Floor(delay.TotalMilliseconds / DelayMinutesLimit);
logger.LogDebug("Breaking interval into {iterationCount} iterations", maxDelayIterations + 1);
delay = TimeSpan.FromMilliseconds(delay.TotalMilliseconds - (maxDelayIterations * DelayMinutesLimit));
}
if (maxDelayIterations > 0)
{
var longDelayTimeSpan = TimeSpan.FromMilliseconds(DelayMinutesLimit);
for (var i = 0UL; i < maxDelayIterations; ++i)
{
logger.LogTrace("Long delay #{iteration}...", i + 1);
await asyncDelayer.Delay(longDelayTimeSpan, cancellationToken);
}
logger.LogTrace("Final delay iteration #{iteration}...", maxDelayIterations + 1);
}
await asyncDelayer.Delay(delay, cancellationToken);
logger.LogInformation("Beginning auto update...");
await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, Enumerable.Empty<string>(), true, cancellationToken);
@@ -541,7 +541,7 @@ namespace Tgstation.Server.Host.Components
loggedDelay = true;
}
delayTask = asyncDelayer.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
delayTask = asyncDelayer.Delay(TimeSpan.FromMilliseconds(100), cancellationToken).AsTask();
}
await delayTask;
@@ -547,7 +547,8 @@ namespace Tgstation.Server.Host.Components.Session
toAwait,
asyncDelayer.Delay(
TimeSpan.FromSeconds(startupTimeout.Value),
CancellationToken.None)); // DCT: None available, task will clean up after delay
CancellationToken.None)
.AsTask()); // DCT: None available, task will clean up after delay
Logger.LogTrace(
"Waiting for LaunchResult based on {launchResultCompletionCause}{possibleTimeout}...",
@@ -611,7 +612,7 @@ namespace Tgstation.Server.Host.Components.Session
const int GracePeriodSeconds = 30;
Logger.LogDebug("Server will terminated in {gracePeriodSeconds}s if it does not exit...", GracePeriodSeconds);
var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(GracePeriodSeconds), CancellationToken.None); // DCT: None available
var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(GracePeriodSeconds), CancellationToken.None).AsTask(); // DCT: None available
await Task.WhenAny(process.Lifetime, delayTask);
if (!process.Lifetime.IsCompleted)
@@ -185,7 +185,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
lingeringDeploymentExpirySeconds);
// DCT: A cancel firing here can result in us leaving a dmbprovider undisposed, localDeploymentCleanupGate will always fire in that case
var timeout = AsyncDelayer.Delay(TimeSpan.FromSeconds(lingeringDeploymentExpirySeconds), CancellationToken.None);
var timeout = AsyncDelayer.Delay(TimeSpan.FromSeconds(lingeringDeploymentExpirySeconds), CancellationToken.None).AsTask();
var completedTask = await Task.WhenAny(
localDeploymentCleanupGate.Task,
@@ -86,7 +86,7 @@ namespace Tgstation.Server.Host.Security
var timeTillSessionExpiry = authenticationContext.SessionExpiry - DateTimeOffset.UtcNow;
if (timeTillSessionExpiry > TimeSpan.Zero)
{
var delayTask = asyncDelayer.Delay(timeTillSessionExpiry, applicationLifetime.ApplicationStopping);
var delayTask = asyncDelayer.Delay(timeTillSessionExpiry, applicationLifetime.ApplicationStopping).AsTask();
await Task.WhenAny(delayTask, otherCancellationReason);
@@ -298,7 +298,7 @@ namespace Tgstation.Server.Host.Swarm
var timeoutTask = swarmController
? asyncDelayer.Delay(
TimeSpan.FromMinutes(SwarmConstants.UpdateCommitTimeoutMinutes),
cancellationToken)
cancellationToken).AsTask()
: Extensions.TaskExtensions.InfiniteTask.WaitAsync(cancellationToken);
var commitTask = Task.WhenAny(localUpdateOperation.CommitGate, timeoutTask);
@@ -1512,7 +1512,8 @@ namespace Tgstation.Server.Host.Swarm
var delayTask = asyncDelayer.Delay(
delay,
cancellationToken);
cancellationToken)
.AsTask();
var awakeningTask = Task.WhenAny(
delayTask,
@@ -1,13 +1,57 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace Tgstation.Server.Host.Utils
{
/// <inheritdoc />
sealed class AsyncDelayer : IAsyncDelayer
{
/// <summary>
/// The <see cref="ILogger"/> for the <see cref="AsyncDelayer"/>.
/// </summary>
readonly ILogger<AsyncDelayer> logger;
/// <summary>
/// Initializes a new instance of the <see cref="AsyncDelayer"/> class.
/// </summary>
/// <param name="logger">The value of <see cref="logger"/>.</param>
public AsyncDelayer(ILogger<AsyncDelayer> logger)
{
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc />
public Task Delay(TimeSpan timeSpan, CancellationToken cancellationToken) => Task.Delay(timeSpan, cancellationToken);
public async ValueTask Delay(TimeSpan timeSpan, CancellationToken cancellationToken)
{
// https://learn.microsoft.com/en-us/dotnet/api/system.threading.tasks.task.delay?view=net-8.0#system-threading-tasks-task-delay(system-timespan)
const uint DelayMinutesLimit = UInt32.MaxValue - 1;
Debug.Assert(DelayMinutesLimit == 4294967294, "Delay limit assertion failure!");
var maxDelayIterations = 0UL;
if (timeSpan.TotalMilliseconds >= UInt32.MaxValue)
{
maxDelayIterations = (ulong)Math.Floor(timeSpan.TotalMilliseconds / DelayMinutesLimit);
logger.LogDebug("Breaking interval into {iterationCount} iterations", maxDelayIterations + 1);
timeSpan = TimeSpan.FromMilliseconds(timeSpan.TotalMilliseconds - (maxDelayIterations * DelayMinutesLimit));
}
if (maxDelayIterations > 0)
{
var longDelayTimeSpan = TimeSpan.FromMilliseconds(DelayMinutesLimit);
for (var i = 0UL; i < maxDelayIterations; ++i)
{
logger.LogTrace("Long delay #{iteration}...", i + 1);
await Task.Delay(longDelayTimeSpan, cancellationToken);
}
logger.LogTrace("Final delay iteration #{iteration}...", maxDelayIterations + 1);
}
await Task.Delay(timeSpan, cancellationToken);
}
}
}
@@ -14,7 +14,7 @@ namespace Tgstation.Server.Host.Utils
/// </summary>
/// <param name="timeSpan">The <see cref="TimeSpan"/> that must elapse.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
Task Delay(TimeSpan timeSpan, CancellationToken cancellationToken);
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
ValueTask Delay(TimeSpan timeSpan, CancellationToken cancellationToken);
}
}
@@ -25,7 +25,7 @@ namespace Tgstation.Server.Host.Tests.Signals
.Returns(ValueTask.CompletedTask);
var mockAsyncDelayer = new Mock<IAsyncDelayer>();
mockAsyncDelayer.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
mockAsyncDelayer.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns(ValueTask.CompletedTask);
using var signalHandler = new PosixSignalHandler(mockServerControl.Object, mockAsyncDelayer.Object, Mock.Of<ILogger<PosixSignalHandler>>());
Assert.IsFalse(tcs.Task.IsCompleted);
@@ -89,7 +89,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
Instance = new Models.Instance(),
};
await using var provider = new IrcProvider(mockJobManager, new AsyncDelayer(), loggerFactory.CreateLogger<IrcProvider>(), Mock.Of<IAssemblyInformationProvider>(), chatBot, new FileLoggingConfiguration());
await using var provider = new IrcProvider(mockJobManager, new AsyncDelayer(loggerFactory.CreateLogger<AsyncDelayer>()), loggerFactory.CreateLogger<IrcProvider>(), Mock.Of<IAssemblyInformationProvider>(), chatBot, new FileLoggingConfiguration());
Assert.IsFalse(provider.Connected);
await InvokeConnect(provider);
Assert.IsTrue(provider.Connected);
@@ -91,7 +91,7 @@ namespace Tgstation.Server.Host.Setup.Tests
mockInternalConfigurationOptions.Object);
mockPlatformIdentifier.SetupGet(x => x.IsWindows).Returns(true).Verifiable();
mockAsyncDelayer.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns(Task.CompletedTask).Verifiable();
mockAsyncDelayer.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns(ValueTask.CompletedTask).Verifiable();
await RunWizard();
@@ -129,7 +129,7 @@ namespace Tgstation.Server.Host.Swarm.Tests
mockAsyncDelayer.Setup(
x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.Returns<TimeSpan, CancellationToken>(
(delay, ct) => Task.Delay(TimeSpan.FromMilliseconds(100), ct));
async (delay, ct) => await Task.Delay(TimeSpan.FromMilliseconds(100), ct));
var mockServerUpdater = new Mock<IServerUpdater>();
@@ -152,7 +152,7 @@ namespace Tgstation.Server.Host.Swarm.Tests
new CryptographySuite(
Mock.Of<IPasswordHasher<Models.User>>()),
Mock.Of<IIOManager>(),
new AsyncDelayer(), // use a real one here because otherwise tickets expire too fast
new AsyncDelayer(Mock.Of<ILogger<AsyncDelayer>>()), // use a real one here because otherwise tickets expire too fast
CreateLoggerFactoryForLogger(loggerFactory.CreateLogger($"FileTransferService-{swarmConfiguration.Identifier}"), out var mockLoggerFactory).CreateLogger<FileTransferService>());
RpcMapper = new SwarmRpcMapper(
@@ -2,8 +2,11 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
namespace Tgstation.Server.Host.Utils.Tests
{
[TestClass]
@@ -12,7 +15,7 @@ namespace Tgstation.Server.Host.Utils.Tests
[TestMethod]
public async Task TestDelay()
{
var delayer = new AsyncDelayer();
var delayer = new AsyncDelayer(Mock.Of<ILogger<AsyncDelayer>>());
var startDelay = delayer.Delay(TimeSpan.FromSeconds(1), CancellationToken.None);
var checkDelay = Task.Delay(TimeSpan.FromSeconds(1) - TimeSpan.FromMilliseconds(100), CancellationToken.None);
await startDelay;
@@ -22,10 +25,10 @@ namespace Tgstation.Server.Host.Utils.Tests
[TestMethod]
public async Task TestCancel()
{
var delayer = new AsyncDelayer();
var delayer = new AsyncDelayer(Mock.Of<ILogger<AsyncDelayer>>());
using var cts = new CancellationTokenSource();
cts.Cancel();
await Assert.ThrowsExceptionAsync<TaskCanceledException>(() => delayer.Delay(TimeSpan.FromSeconds(1), cts.Token));
await Assert.ThrowsExceptionAsync<TaskCanceledException>(() => delayer.Delay(TimeSpan.FromSeconds(1), cts.Token).AsTask());
}
}
}
@@ -51,7 +51,7 @@ namespace Tgstation.Server.Tests.Live
// at time of writing, this is used exclusively for the reconnection interval which works in minutes
// shorten it to 3s
var mock = new Mock<IAsyncDelayer>();
mock.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns<TimeSpan, CancellationToken>((delay, cancellationToken) => Task.Delay(TimeSpan.FromSeconds(3), cancellationToken));
mock.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns<TimeSpan, CancellationToken>(async (delay, cancellationToken) => await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken));
return mock.Object;
}
@@ -309,7 +309,7 @@ namespace Tgstation.Server.Tests.Live.Instance
? await session.TopicSendSemaphore.Lock(cancellationToken)
: null)
return await topicClient.SendWithOptionalPriority(
new AsyncDelayer(),
new AsyncDelayer(loggerFactory.CreateLogger<AsyncDelayer>()),
loggerFactory.CreateLogger<WatchdogTest>(),
queryString,
topicPort,