Handle startup jobs a bit better

This commit is contained in:
Jordan Dominion
2023-12-16 10:07:18 -05:00
parent ba454f0ac0
commit aa51b67ea1
4 changed files with 42 additions and 16 deletions
@@ -0,0 +1,25 @@
using System;
using Tgstation.Server.Api.Models;
namespace Tgstation.Server.Api.Extensions
{
/// <summary>
/// Extension methods for the <see cref="JobCode"/> <see langword="enum"/>.
/// </summary>
public static class JobCodeExtensions
{
/// <summary>
/// If a given <paramref name="jobCode"/> can be triggered by TGS startup.
/// </summary>
/// <param name="jobCode">The <see cref="JobCode"/>.</param>
/// <returns><see langword="true"/> if the <see cref="JobCode"/> can trigger before startup, <see langword="false"/> otherwise.</returns>
public static bool IsServerStartupJob(this JobCode jobCode)
=> jobCode switch
{
JobCode.Unknown or JobCode.Move or JobCode.RepositoryClone or JobCode.RepositoryUpdate or JobCode.RepositoryAutoUpdate or JobCode.RepositoryDelete or JobCode.EngineOfficialInstall or JobCode.EngineCustomInstall or JobCode.EngineDelete or JobCode.Deployment or JobCode.AutomaticDeployment or JobCode.WatchdogLaunch or JobCode.WatchdogRestart or JobCode.WatchdogDump => false,
JobCode.StartupWatchdogLaunch or JobCode.StartupWatchdogReattach or JobCode.ReconnectChatBot => true,
_ => throw new InvalidOperationException($"Invalid JobCode: {jobCode}"),
};
}
}
+7 -1
View File
@@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging;
using Serilog.Context;
using Tgstation.Server.Api.Extensions;
using Tgstation.Server.Api.Hubs;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Common.Extensions;
@@ -452,7 +453,12 @@ namespace Tgstation.Server.Host.Jobs
}
}
var instanceCoreProvider = await activationTcs.Task.WaitAsync(cancellationToken);
var activationTask = activationTcs.Task;
Debug.Assert(activationTask.IsCompleted || job.JobCode.Value.IsServerStartupJob(), "Non-server startup job registered before activation!");
var instanceCoreProvider = await activationTask.WaitAsync(cancellationToken);
QueueHubUpdate(job.ToApi(), false);
logger.LogTrace("Starting job...");
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
@@ -8,6 +8,7 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.SignalR.Client;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Tgstation.Server.Api.Extensions;
using Tgstation.Server.Api.Hubs;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Request;
@@ -209,9 +210,7 @@ namespace Tgstation.Server.Tests.Live.Instance
}
else
{
var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot
|| job.JobCode == JobCode.StartupWatchdogLaunch
|| job.JobCode == JobCode.StartupWatchdogReattach;
var wasMissableJob = job.JobCode.Value.IsServerStartupJob();
Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: #{job.Id.Value} - {job.JobCode} - {job.Description}");
++missableMissedJobs;
}
@@ -28,6 +28,7 @@ using Newtonsoft.Json;
using Npgsql;
using Tgstation.Server.Api;
using Tgstation.Server.Api.Extensions;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Request;
using Tgstation.Server.Api.Models.Response;
@@ -1653,16 +1654,11 @@ namespace Tgstation.Server.Tests.Live
{
var jobs = await instanceClient.Jobs.ListActive(null, cancellationToken);
if (jobs.Count == 0)
{
var entities = await instanceClient.Jobs.List(null, cancellationToken);
var getTasks = entities
.Select(e => instanceClient.Jobs.GetId(e, cancellationToken))
.ToList();
jobs = (await ValueTaskExtensions.WhenAll(getTasks))
jobs = (await instanceClient.Jobs.List(null, cancellationToken))
.Where(x => x.StartedAt.Value > preStartupTime)
.ToList();
}
.ToList();
else
jobs = jobs.Where(x => x.JobCode.Value.IsServerStartupJob()).ToList();
await using var jrt = new JobsRequiredTest(instanceClient.Jobs);
foreach (var job in jobs)
@@ -1679,9 +1675,9 @@ namespace Tgstation.Server.Tests.Live
var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken);
await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken))
{
await jobsHubTest.WaitForReconnect(cancellationToken);
var instanceClient = adminClient.Instances.CreateClient(instance);
await WaitForInitialJobs(instanceClient);
await jobsHubTest.WaitForReconnect(cancellationToken);
var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
@@ -1722,9 +1718,9 @@ namespace Tgstation.Server.Tests.Live
serverTask = server.Run(cancellationToken).AsTask();
await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken))
{
await jobsHubTest.WaitForReconnect(cancellationToken);
var instanceClient = adminClient.Instances.CreateClient(instance);
await WaitForInitialJobs(instanceClient);
await jobsHubTest.WaitForReconnect(cancellationToken);
var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value);