Merge branch 'dev' into 1009-FixInstanceRestore

This commit is contained in:
Jordan Brown
2020-06-15 14:46:37 -04:00
committed by GitHub
17 changed files with 111 additions and 26 deletions
+2
View File
@@ -95,6 +95,8 @@
#define TGS_EVENT_WATCHDOG_SHUTDOWN 15
/// Before the watchdog detaches for a TGS update/restart. No parameters.
#define TGS_EVENT_WATCHDOG_DETACH 16
// We don't actually implement this value as the DMAPI can never receive it
// #define TGS_EVENT_WATCHDOG_LAUNCH 17
// OTHER ENUMS
@@ -11,14 +11,14 @@ namespace Tgstation.Server.Api.Models.Internal
/// The revision sha
/// </summary>
[Required]
[StringLength(40)]
[StringLength(Limits.MaximumCommitShaLength)]
public string? CommitSha { get; set; }
/// <summary>
/// The sha of the most recent remote commit
/// </summary>
[Required]
[StringLength(40)]
[StringLength(Limits.MaximumCommitShaLength)]
public string? OriginCommitSha { get; set; }
}
}
@@ -14,5 +14,10 @@
/// Length limit for <see cref="Internal.ChatBot.Name"/>s.
/// </summary>
public const int MaximumIndexableStringLength = 100;
/// <summary>
/// Length limit for git commit SHAs.
/// </summary>
public const int MaximumCommitShaLength = 40;
}
}
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace Tgstation.Server.Api.Models
{
@@ -12,9 +13,15 @@ namespace Tgstation.Server.Api.Models
/// </summary>
public string? Origin { get; set; }
/// <summary>
/// If submodules should be recursively cloned.
/// </summary>
public bool? RecurseSubmodules { get; set; }
/// <summary>
/// The commit HEAD should point to. Not populated in responses, use <see cref="RevisionInformation"/> instead for retrieval
/// </summary>
[StringLength(Limits.MaximumCommitShaLength)]
public string? CheckoutSha { get; set; }
/// <summary>
@@ -45,6 +52,7 @@ namespace Tgstation.Server.Api.Models
/// <summary>
/// The branch or tag HEAD points to
/// </summary>
[StringLength(Limits.MaximumStringLength)]
public string? Reference { get; set; }
/// <summary>
@@ -33,19 +33,16 @@ namespace Tgstation.Server.Client.Components
/// <inheritdoc />
public IJobsClient Jobs { get; }
/// <summary>
/// The <see cref="IApiClient"/> for the <see cref="InstanceClient"/>
/// </summary>
readonly IApiClient apiClient;
/// <summary>
/// Construct a <see cref="InstanceClient"/>
/// </summary>
/// <param name="apiClient">The value of <see cref="apiClient"/></param>
/// <param name="apiClient">The <see cref="IApiClient"/> used to construct component clients.</param>
/// <param name="instance">The value of <see cref="Metadata"/></param>
public InstanceClient(IApiClient apiClient, Instance instance)
{
this.apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
if (apiClient == null)
throw new ArgumentNullException(nameof(apiClient));
Metadata = instance ?? throw new ArgumentNullException(nameof(instance));
Byond = new ByondClient(apiClient, instance);
@@ -106,5 +106,11 @@
/// </summary>
[EventScript("WatchdogDetach")]
WatchdogDetach,
/// <summary>
/// Before the watchdog launches. No parameters.
/// </summary>
[EventScript("WatchdogLaunch")]
WatchdogLaunch
}
}
@@ -250,6 +250,7 @@ namespace Tgstation.Server.Host.Components
sessionControllerFactory,
gameIoManager,
diagnosticsIOManager,
eventConsumer,
metadata.CloneMetadata(),
metadata.DreamDaemonSettings);
eventConsumer.SetWatchdog(watchdog);
@@ -15,7 +15,7 @@ namespace Tgstation.Server.Host.Components.Repository
bool InUse { get; }
/// <summary>
/// If a <see cref="CloneRepository(Uri, string, string, string, Action{int}, CancellationToken)"/> operation is in progress
/// If a <see cref="CloneRepository(Uri, string, string, string, Action{int}, bool, CancellationToken)"/> operation is in progress.
/// </summary>
bool CloneInProgress { get; }
@@ -34,9 +34,17 @@ namespace Tgstation.Server.Host.Components.Repository
/// <param name="username">The username to clone from <paramref name="url"/></param>
/// <param name="password">The password to clone from <paramref name="url"/></param>
/// <param name="progressReporter">A function to report 0-100 progress of the clone</param>
/// <param name="recurseSubmodules">If submodules should be recusively cloned and initialized.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>The newly cloned <see cref="IRepository"/>, <see langword="null"/> if one already exists</returns>
Task<IRepository> CloneRepository(Uri url, string initialBranch, string username, string password, Action<int> progressReporter, CancellationToken cancellationToken);
Task<IRepository> CloneRepository(
Uri url,
string initialBranch,
string username,
string password,
Action<int> progressReporter,
bool recurseSubmodules,
CancellationToken cancellationToken);
/// <summary>
/// Delete the current repository
@@ -89,7 +89,14 @@ namespace Tgstation.Server.Host.Components.Repository
}
/// <inheritdoc />
public async Task<IRepository> CloneRepository(Uri url, string initialBranch, string username, string password, Action<int> progressReporter, CancellationToken cancellationToken)
public async Task<IRepository> CloneRepository(
Uri url,
string initialBranch,
string username,
string password,
Action<int> progressReporter,
bool recurseSubmodules,
CancellationToken cancellationToken)
{
if (url == null)
throw new ArgumentNullException(nameof(url));
@@ -122,7 +129,7 @@ namespace Tgstation.Server.Host.Components.Repository
progressReporter((int)percentage);
return !cancellationToken.IsCancellationRequested;
},
RecurseSubmodules = true,
RecurseSubmodules = recurseSubmodules,
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested,
BranchName = initialBranch,
@@ -7,6 +7,7 @@ using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
@@ -51,6 +52,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
@@ -65,6 +67,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IServerControl serverControl,
IAsyncDelayer asyncDelayer,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
ILogger<BasicWatchdog> logger,
DreamDaemonLaunchParameters initialLaunchParameters,
Api.Models.Instance instance,
@@ -79,6 +82,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
serverControl,
asyncDelayer,
diagnosticsIOManager,
eventConsumer,
logger,
initialLaunchParameters,
instance,
@@ -8,6 +8,7 @@ using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
@@ -62,6 +63,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
@@ -76,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IServerControl serverControl,
IAsyncDelayer asyncDelayer,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
ILogger<ExperimentalWatchdog> logger,
DreamDaemonLaunchParameters initialLaunchParameters,
Api.Models.Instance instance, bool autoStart)
@@ -89,6 +92,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
serverControl,
asyncDelayer,
diagnosticsIOManager,
eventConsumer,
logger,
initialLaunchParameters,
instance,
@@ -1,6 +1,7 @@
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.IO;
@@ -20,6 +21,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="IWatchdog"/></param>
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the Game directory for the <see cref="IWatchdog"/>.</param>
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="IWatchdog"/>.</param>
/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/></param>
/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/></param>
/// <returns>A new <see cref="IWatchdog"/></returns>
@@ -30,6 +32,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ISessionControllerFactory sessionControllerFactory,
IIOManager gameIOManager,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
Api.Models.Instance instance,
DreamDaemonSettings settings);
}
@@ -120,6 +120,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
readonly IIOManager diagnosticsIOManager;
/// <summary>
/// The <see cref="IEventConsumer"/> that is not the <see cref="WatchdogBase"/>
/// </summary>
readonly IEventConsumer eventConsumer;
/// <summary>
/// <see langword="lock"/> <see cref="object"/> used for <see cref="DisposeAndNullControllers"/>.
/// </summary>
@@ -177,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="serverControl">The <see cref="IServerControl"/> to populate <see cref="restartRegistration"/> with</param>
/// <param name="asyncDelayer">The value of <see cref="AsyncDelayer"/>.</param>
/// <param name="diagnosticsIOManager">The value of <see cref="diagnosticsIOManager"/>.</param>
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
/// <param name="logger">The value of <see cref="Logger"/></param>
/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified</param>
/// <param name="instance">The value of <see cref="instance"/></param>
@@ -191,6 +197,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IServerControl serverControl,
IAsyncDelayer asyncDelayer,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
ILogger logger,
DreamDaemonLaunchParameters initialLaunchParameters,
Api.Models.Instance instance,
@@ -204,6 +211,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
@@ -255,7 +263,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
return;
if (!graceful)
{
var eventTask = HandleEvent(releaseServers ? EventType.WatchdogDetach : EventType.WatchdogShutdown, null, cancellationToken);
var eventTask = eventConsumer.HandleEvent(releaseServers ? EventType.WatchdogDetach : EventType.WatchdogShutdown, null, cancellationToken);
var chatTask = announce ? Chat.SendWatchdogMessage("Shutting down...", false, cancellationToken) : Task.CompletedTask;
@@ -353,13 +361,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
Task chatTask;
Task announceTask;
if (startMonitor && await StopMonitor().ConfigureAwait(false))
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
announceTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
else if (announce)
chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
{
announceTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
if (reattachInfo == null)
announceTask = Task.WhenAll(
eventConsumer.HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), cancellationToken),
announceTask);
}
else
chatTask = Task.CompletedTask; // no announce
announceTask = Task.CompletedTask; // no announce
// since neither server is running, this is safe to do
LastLaunchParameters = ActiveLaunchParameters;
@@ -369,11 +383,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
var recursiveCallToHappen = false;
try
{
await InitControllers(() => recursiveCallToHappen = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
await InitControllers(() => recursiveCallToHappen = true, announceTask, reattachInfo, cancellationToken).ConfigureAwait(false);
if (recursiveCallToHappen)
return;
await chatTask.ConfigureAwait(false);
await announceTask.ConfigureAwait(false);
Logger.LogInformation("Launched servers successfully");
Running = true;
@@ -389,14 +403,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
// don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested)
{
var originalChatTask = chatTask;
var originalChatTask = announceTask;
async Task ChainChatTaskWithErrorMessage()
{
await originalChatTask.ConfigureAwait(false);
await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
}
chatTask = ChainChatTaskWithErrorMessage();
announceTask = ChainChatTaskWithErrorMessage();
Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
}
@@ -407,9 +421,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
// finish the chat task that's in flight
try
{
await chatTask.ConfigureAwait(false);
await announceTask.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
Logger.LogTrace("Announcement task canceled!");
}
catch (OperationCanceledException) { }
}
}
@@ -751,8 +768,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
/// <inheritdoc />
public async Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
async Task IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
{
// Method explicitly implemented to prevent accidental calls when this.eventConsumer should be used.
var activeServer = GetActiveController();
// Server may have ended
@@ -4,6 +4,7 @@ using System;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.Core;
@@ -79,6 +80,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ISessionControllerFactory sessionControllerFactory,
IIOManager gameIOManager,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
Api.Models.Instance instance,
DreamDaemonSettings settings)
{
@@ -93,6 +95,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ServerControl,
AsyncDelayer,
diagnosticsIOManager,
eventConsumer,
LoggerFactory.CreateLogger<ExperimentalWatchdog>(),
settings,
instance,
@@ -105,6 +108,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
sessionControllerFactory,
gameIOManager,
diagnosticsIOManager,
eventConsumer,
instance,
settings);
}
@@ -118,6 +122,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="IWatchdog"/></param>
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the Game directory for the <see cref="IWatchdog"/>.</param>
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="IWatchdog"/>.</param>
/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/></param>
/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/></param>
/// <returns>A new <see cref="IWatchdog"/></returns>
@@ -128,6 +133,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ISessionControllerFactory sessionControllerFactory,
IIOManager gameIOManager,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
Api.Models.Instance instance,
DreamDaemonSettings settings)
=> new BasicWatchdog(
@@ -140,6 +146,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ServerControl,
AsyncDelayer,
diagnosticsIOManager,
eventConsumer,
LoggerFactory.CreateLogger<BasicWatchdog>(),
settings,
instance,
@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
@@ -55,6 +56,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="gameIOManager">The value of <see cref="gameIOManager"/>.</param>
/// <param name="symlinkFactory">The value of <see cref="symlinkFactory"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -71,6 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IServerControl serverControl,
IAsyncDelayer asyncDelayer,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IIOManager gameIOManager,
ISymlinkFactory symlinkFactory,
ILogger<WindowsWatchdog> logger,
@@ -86,6 +89,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
serverControl,
asyncDelayer,
diagnosticsIOManager,
eventConsumer,
logger,
initialLaunchParameters,
instance,
@@ -4,6 +4,7 @@ using System;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.Core;
@@ -60,6 +61,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ISessionControllerFactory sessionControllerFactory,
IIOManager gameIOManager,
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
Api.Models.Instance instance,
DreamDaemonSettings settings)
=> new WindowsWatchdog(
@@ -72,6 +74,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ServerControl,
AsyncDelayer,
diagnosticsIOManager,
eventConsumer,
gameIOManager,
symlinkFactory,
LoggerFactory.CreateLogger<WindowsWatchdog>(),
@@ -204,7 +204,15 @@ namespace Tgstation.Server.Host.Controllers
var api = currentModel.ToApi();
await jobManager.RegisterOperation(job, async (paramJob, databaseContextFactory, progressReporter, ct) =>
{
using var repos = await repoManager.CloneRepository(new Uri(origin), cloneBranch, currentModel.AccessUser, currentModel.AccessToken, progressReporter, ct).ConfigureAwait(false);
using var repos = await repoManager.CloneRepository(
new Uri(origin),
cloneBranch,
currentModel.AccessUser,
currentModel.AccessToken,
progressReporter,
model.RecurseSubmodules ?? true,
ct)
.ConfigureAwait(false);
if (repos == null)
throw new JobException(ErrorCode.RepoExists);
var instance = new Models.Instance