Prevent reboot bridge request completing too soon

This commit is contained in:
Jordan
2023-06-18 18:33:59 -04:00
parent db488ea01d
commit 08ea2b7764
3 changed files with 41 additions and 0 deletions
@@ -69,6 +69,11 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
Task OnReboot { get; }
/// <summary>
/// A <see cref="Task"/> that must complete before a TgsReboot() bridge request can complete.
/// </summary>
Task RebootGate { set; }
/// <summary>
/// A <see cref="Task"/> that completes when the server calls /world/TgsInitializationComplete().
/// </summary>
@@ -75,6 +75,26 @@ namespace Tgstation.Server.Host.Components.Session
/// <inheritdoc />
public Task OnReboot => rebootTcs.Task;
/// <inheritdoc />
public Task RebootGate
{
get => rebootGate;
set
{
var tcs = new TaskCompletionSource();
Task toAwait = null;
async Task Wrap()
{
await tcs.Task;
await toAwait;
await value;
}
toAwait = Interlocked.Exchange(ref rebootGate, Wrap());
tcs.SetResult();
}
}
/// <inheritdoc />
public Task OnPrime => primeTcs.Task;
@@ -164,6 +184,11 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
volatile TaskCompletionSource primeTcs;
/// <summary>
/// Backing field for <see cref="RebootGate"/>.
/// </summary>
volatile Task rebootGate;
/// <summary>
/// The number of currently active calls to <see cref="ProcessBridgeRequest(BridgeParameters, CancellationToken)"/> from TgsReboot().
/// </summary>
@@ -254,6 +279,8 @@ namespace Tgstation.Server.Host.Components.Session
rebootTcs = new TaskCompletionSource();
primeTcs = new TaskCompletionSource();
rebootGate = Task.CompletedTask;
// Run this asynchronously because we want to try to avoid any effects sending topics to the server while the initial bridge request is processing
// It MAY be the source of a DD crash. See this gist https://gist.github.com/Cyberboss/7776bbeff3a957d76affe0eae95c9f14
// Worth further investigation as to if that sequence of events is a reliable crash vector and opening a BYOND bug if it is
@@ -751,6 +778,7 @@ namespace Tgstation.Server.Host.Components.Session
}
Interlocked.Exchange(ref rebootTcs, new TaskCompletionSource()).SetResult();
await RebootGate;
}
finally
{
@@ -804,6 +804,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
var ranInitialDmbCheck = false;
for (ulong iteration = 1; nextAction != MonitorAction.Exit; ++iteration)
using (LogContext.PushProperty(SerilogContextHelper.WatchdogMonitorIterationContextProperty, iteration))
{
TaskCompletionSource nextMonitorWakeupTcs = new TaskCompletionSource();
try
{
Logger.LogTrace("Iteration {iteration} of monitor loop", iteration);
@@ -821,6 +823,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
oldTask = newTaskFactory();
}
controller.RebootGate = nextMonitorWakeupTcs.Task;
if (lastController == controller)
{
TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime);
@@ -967,6 +970,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
nextAction = MonitorAction.Continue;
}
}
finally
{
nextMonitorWakeupTcs.SetResult();
}
}
}
catch (OperationCanceledException)
{