Merge pull request #572 from Cyberboss/MoreWatchdog

More watchdog work
This commit is contained in:
Jordan Brown
2018-08-19 09:11:56 -04:00
committed by GitHub
26 changed files with 409 additions and 120 deletions
+32 -21
View File
@@ -72,7 +72,7 @@
TGS_INFO_LOG("Validating API and exiting...")
Export(TGS4_COMM_VALIDATE)
del(world)
chat_channels_json_path = cached_json["chatChannelsJson"]
chat_commands_json_path = cached_json["chatCommandsJson"]
src.event_handler = event_handler
@@ -80,31 +80,34 @@
ListCustomCommands()
. = TRUE
var/list/revisionData = cached_json["revision"]
if(!revisionData)
return
cached_revision = new
cached_revision.commit = revisionData["commitSha"]
cached_revision.origin_commit = revisionData["originCommitSha"]
if(revisionData)
cached_revision = new
cached_revision.commit = revisionData["commitSha"]
cached_revision.origin_commit = revisionData["originCommitSha"]
cached_test_merges = list()
var/json = cached_json["testMerges"]
for(var/I in json)
var/list/json = cached_json["testMerges"]
for(var/entry in json)
var/datum/tgs_revision_information/test_merge/tm = new
tm.number = text2num(I)
var/list/entry = json[I]
tm.pull_request_commit = entry["pullRequestRevision"]
tm.author = entry["author"]
tm.title = entry["title"]
var/list/revInfo = entry["revision"]
tm.commit = revInfo["commitSha"]
tm.origin_commit = entry["originCommitSha"]
tm.time_merged = text2num(entry["timeMerged"])
tm.comment = entry["comment"]
var/list/revInfo = entry["revision"]
if(revInfo)
tm.commit = revInfo["commitSha"]
tm.origin_commit = revInfo["originCommitSha"]
tm.title = entry["titleAtMerge"]
tm.body = entry["bodyAtMerge"]
tm.url = entry["url"]
tm.author = entry["author"]
tm.number = entry["number"]
tm.pull_request_commit = entry["pullRequestRevision"]
tm.comment = entry["comment"]
cached_test_merges += tm
return TRUE
/datum/tgs_api/v4/OnInitializationComplete()
Export(TGS4_COMM_SERVER_PRIMED)
@@ -138,7 +141,15 @@
return result
if(TGS4_TOPIC_EVENT)
intercepted_message_queue = list()
event_handler.HandleEvent(text2num(params[TGS4_PARAMETER_DATA]))
var/list/event_notification = json_decode(params[TGS4_PARAMETER_DATA])
var/list/event_parameters = event_notification["Parameters"]
var/list/event_call = list(event_notification["Type"])
if(event_parameters)
event_call += event_parameters
event_handler.HandleEvent(arglist(event_call))
. = json_encode(intercepted_message_queue)
intercepted_message_queue = null
return
@@ -7,12 +7,6 @@ namespace Tgstation.Server.Api.Models
/// </summary>
public sealed class DreamMaker : DreamMakerSettings
{
/// <summary>
/// The last <see cref="CompileJob"/> ran
/// </summary>
[Permissions(DenyWrite = true)]
public CompileJob LastJob { get; set; }
/// <summary>
/// The <see cref="CompilerStatus"/> of the compiler
/// </summary>
@@ -134,7 +134,7 @@ namespace Tgstation.Server.Host.Components.Byond
cancellationToken.ThrowIfCancellationRequested();
if (exitCode != 0)
throw new Exception(String.Format(CultureInfo.InvariantCulture, "Failed to install included DirectX! Exit code: {0}", exitCode));
throw new JobException(String.Format(CultureInfo.InvariantCulture, "Failed to install included DirectX! Exit code: {0}", exitCode));
installedDirectX = true;
}
}
@@ -15,7 +15,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
public string Name => "byond";
/// <inheritdoc />
public string HelpText => "Displays the installed Byond version";
public string HelpText => "Displays the active Byond version";
/// <inheritdoc />
public bool AdminOnly => false;
@@ -77,6 +77,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
{
new VersionCommand(application),
new ByondCommand(byondManager),
new RevisionCommand(watchdog, repositoryManager, instance),
new PullRequestsCommand(watchdog, repositoryManager, databaseContextFactory, instance),
new KekCommand()
};
@@ -92,7 +92,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
if (!results.Any())
return "None!";
return String.Join(", ", results.Select(x => String.Format(CultureInfo.InvariantCulture, "#{0} as {1}", x.Number, x.PullRequestRevision.Substring(0, 7))));
return String.Join(", ", results.Select(x => String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7))));
}
}
}
@@ -0,0 +1,75 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Components.Watchdog;
namespace Tgstation.Server.Host.Components.Chat.Commands
{
/// <summary>
/// For displaying <see cref="Api.Models.Internal.RevisionInformation"/>
/// </summary>
sealed class RevisionCommand : ICommand
{
/// <inheritdoc />
public string Name => "revision";
/// <inheritdoc />
public string HelpText => "Display live commit sha. Add --repo to view repository revision";
/// <inheritdoc />
public bool AdminOnly => false;
/// <summary>
/// The <see cref="IWatchdog"/> for the <see cref="PullRequestsCommand"/>
/// </summary>
readonly IWatchdog watchdog;
/// <summary>
/// The <see cref="IRepositoryManager"/> for the <see cref="PullRequestsCommand"/>
/// </summary>
readonly IRepositoryManager repositoryManager;
/// <summary>
/// The <see cref="Models.Instance"/> for the <see cref="PullRequestsCommand"/>
/// </summary>
readonly Models.Instance instance;
/// <summary>
/// Construct a <see cref="RevisionCommand"/>
/// </summary>
/// <param name="watchdog">The value of <see cref="watchdog"/></param>
/// <param name="repositoryManager">The value of <see cref="repositoryManager"/></param>
/// <param name="instance">The value of <see cref="instance"/></param>
public RevisionCommand(IWatchdog watchdog, IRepositoryManager repositoryManager, Models.Instance instance)
{
this.watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog));
this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
}
/// <inheritdoc />
public async Task<string> Invoke(string arguments, User user, CancellationToken cancellationToken)
{
string result;
if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO"))
using (var repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false))
{
if (repo == null)
return "Repository unavailable!";
result = repo.Head;
}
else
{
if (!watchdog.Running)
return "Server offline!";
result = watchdog.ActiveCompileJob?.RevisionInformation.CommitSha;
}
return String.Format(CultureInfo.InvariantCulture, "^{0}", result);
}
}
}
@@ -280,9 +280,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
}
client.Listen(false);
if (client.Nickname != nickname)
//run this now cause it has to go up and down the pipe for us to get a proper check
client.GetIrcUser(nickname);
listenTask = Task.Factory.StartNew(() =>
{
@@ -135,14 +135,17 @@ namespace Tgstation.Server.Host.Components.Compiler
var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);
using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
var now = DateTimeOffset.Now;
if (now < timeoutAt)
if (now < timeoutAt && launchResult.StartupTime.HasValue)
{
var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);
await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
if (!controller.Lifetime.IsCompleted)
{
logger.LogDebug("API validation timed out!");
@@ -173,7 +176,7 @@ namespace Tgstation.Server.Host.Components.Compiler
logger.LogDebug("DreamMaker exit code: {0}", exitCode);
job.Output = dm.GetCombinedOutput();
logger.LogTrace("DreamMaker output: {0}", job.Output);
logger.LogTrace("DreamMaker output: {0}{1}", Environment.NewLine, job.Output);
return exitCode;
}
}
@@ -260,11 +263,7 @@ namespace Tgstation.Server.Host.Components.Compiler
lock (this)
{
if (Status != CompilerStatus.Idle)
{
job.Output = "There is already a compile in progress!";
logger.LogInformation(job.Output);
return job;
}
throw new JobException("There is already a compile in progress!");
Status = CompilerStatus.Copying;
}
@@ -337,11 +336,7 @@ namespace Tgstation.Server.Host.Components.Compiler
logger.LogTrace("Searching for available .dmes...");
var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault();
if (path == default)
{
job.Output = "Unable to find any .dme!";
logger.LogWarning(job.Output);
return job;
}
throw new JobException("Unable to find any .dme!");
var dmeWithExtension = ioManager.GetFileName(path);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
@@ -369,7 +364,7 @@ namespace Tgstation.Server.Host.Components.Compiler
{
//server never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List<string> { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false);
throw new Exception(exitCode == 0 ? "Validation of the TGS api failed!" : String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}", exitCode, job.Output));
throw new JobException(exitCode == 0 ? "Validation of the TGS api failed!" : String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}", exitCode, job.Output));
}
logger.LogTrace("Running post compile event...");
@@ -0,0 +1,20 @@
using System.Collections.Generic;
namespace Tgstation.Server.Host.Components.Interop
{
/// <summary>
/// For notifying DD of <see cref="EventType"/>s
/// </summary>
sealed class EventNotification
{
/// <summary>
/// The <see cref="EventType"/>
/// </summary>
public EventType Type { get; set; }
/// <summary>
/// The event parameters
/// </summary>
public IEnumerable<string> Parameters { get; set; }
}
}
@@ -1,4 +1,6 @@
using Tgstation.Server.Api.Models.Internal;
using System;
using System.Globalization;
using Tgstation.Server.Api.Models.Internal;
namespace Tgstation.Server.Host.Components.Interop
{
@@ -10,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Interop
/// <summary>
/// The unix time of when the test merge was applied
/// </summary>
public long TimeMerged { get; set; }
public string TimeMerged { get; set; }
/// <summary>
/// The <see cref="RevisionInformation"/> of the <see cref="TestMerge"/>
@@ -21,10 +23,11 @@ namespace Tgstation.Server.Host.Components.Interop
/// Construct a <see cref="TestMerge"/>
/// </summary>
/// <param name="testMerge">The <see cref="Models.TestMerge"/> to build from</param>
public TestMerge(Models.TestMerge testMerge) : base(testMerge)
/// <param name="revision">The value of <see cref="Revision"/></param>
public TestMerge(Models.TestMerge testMerge, RevisionInformation revision) : base(testMerge)
{
TimeMerged = testMerge.MergedAt.Ticks;
Revision = testMerge.PrimaryRevisionInformation;
TimeMerged = testMerge.MergedAt.Ticks.ToString(CultureInfo.InvariantCulture);
Revision = revision ?? throw new ArgumentNullException(nameof(revision));
}
}
}
@@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Repository
public string GitHubRepoName { get; }
/// <inheritdoc />
public bool Tracking => repository.Head.IsTracking;
public bool Tracking => Reference != null && repository.Head.IsTracking;
/// <inheritdoc />
public string Head => repository.Head.Tip.Sha;
@@ -9,9 +9,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
public sealed class LaunchResult
{
/// <summary>
/// The time it took for <see cref="System.Diagnostics.Process.WaitForInputIdle()"/> to return
/// The time it took for <see cref="System.Diagnostics.Process.WaitForInputIdle()"/> to return. If <see langword="null"/> the startup timed out
/// </summary>
public TimeSpan StartupTime { get; set; }
public TimeSpan? StartupTime { get; set; }
/// <summary>
/// The <see cref="System.Diagnostics.Process.ExitCode"/> if it exited
@@ -19,6 +19,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
public int? ExitCode { get; set; }
/// <inheritdoc />
public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, Time {1}ms", ExitCode, StartupTime.TotalMilliseconds);
public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, Time {1}ms", ExitCode, StartupTime?.TotalMilliseconds);
}
}
@@ -1,16 +1,42 @@
namespace Tgstation.Server.Host.Components.Watchdog
using Newtonsoft.Json;
namespace Tgstation.Server.Host.Components.Watchdog
{
/// <summary>
/// The (absolute) state of the <see cref="Watchdog"/>
/// </summary>
sealed class MonitorState
{
/// <summary>
/// If the inactive server is being rebooted
/// </summary>
public bool RebootingInactiveServer { get; set; }
/// <summary>
/// If the inactive server has a .dmb and needs to be swapped in
/// </summary>
public bool InactiveServerHasStagedDmb { get; set; }
/// <summary>
/// If the inactive server is in an unrecoverable state
/// </summary>
public bool InactiveServerCritFail { get; set; }
/// <summary>
/// The next <see cref="MonitorAction"/> to take in <see cref="Watchdog.MonitorLifetimes(System.Threading.CancellationToken)"/>
/// </summary>
public MonitorAction NextAction { get; set; }
/// <summary>
/// The active <see cref="ISessionController"/>
/// </summary>
[JsonIgnore]
public ISessionController ActiveServer { get; set; }
/// <summary>
/// The inactive <see cref="ISessionController"/>
/// </summary>
[JsonIgnore]
public ISessionController InactiveServer { get; set; }
}
}
@@ -167,7 +167,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="chat">The value of <see cref="chat"/></param>
/// <param name="chatJsonTrackingContext">The value of <see cref="chatJsonTrackingContext"/></param>
/// <param name="logger">The value of <see cref="logger"/></param>
public SessionController(ReattachInformation reattachInformation, IProcess process, IByondExecutableLock byondLock, IByondTopicSender byondTopicSender, IJsonTrackingContext chatJsonTrackingContext, ICommContext interopContext, IChat chat, ILogger<SessionController> logger)
/// <param name="startupTimeout">The optional time to wait before failing the <see cref="LaunchResult"/></param>
public SessionController(ReattachInformation reattachInformation, IProcess process, IByondExecutableLock byondLock, IByondTopicSender byondTopicSender, IJsonTrackingContext chatJsonTrackingContext, ICommContext interopContext, IChat chat, ILogger<SessionController> logger, uint? startupTimeout)
{
this.chatJsonTrackingContext = chatJsonTrackingContext; //null valid
this.reattachInformation = reattachInformation ?? throw new ArgumentNullException(nameof(reattachInformation));
@@ -190,11 +191,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
async Task<LaunchResult> GetLaunchResult()
{
var startTime = DateTimeOffset.Now;
await process.Startup.ConfigureAwait(false);
Task toAwait = process.Startup;
if (startupTimeout.HasValue)
toAwait = Task.WhenAny(process.Startup, Task.Delay(startTime.AddSeconds(startupTimeout.Value) - startTime));
await toAwait.ConfigureAwait(false);
var result = new LaunchResult
{
ExitCode = process.Lifetime.IsCompleted ? (int?)await process.Lifetime.ConfigureAwait(false) : null,
StartupTime = DateTimeOffset.Now - startTime
StartupTime = process.Startup.IsCompleted ? (TimeSpan?)(DateTimeOffset.Now - startTime) : null
};
return result;
};
@@ -420,5 +427,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
CheckDisposed();
reattachInformation.RebootState = RebootState.Normal;
}
/// <inheritdoc />
public void SetHighPriority() => process.SetHighPriority();
}
}
@@ -131,6 +131,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
//i changed this back from guids, hopefully i don't regret that
string JsonFile(string name) => String.Format(CultureInfo.InvariantCulture, "{0}.{1}", name, JsonPostfix);
//setup interop files
var interopInfo = new JsonFile
{
AccessIdentifier = accessIdentifier,
@@ -139,10 +140,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
ChatCommandsJson = JsonFile("chat_commands"),
ServerCommandsJson = JsonFile("server_commands"),
InstanceName = instance.Name,
Revision = dmbProvider.CompileJob.RevisionInformation
Revision = new Api.Models.Internal.RevisionInformation
{
CommitSha = dmbProvider.CompileJob.RevisionInformation.CommitSha,
OriginCommitSha = dmbProvider.CompileJob.RevisionInformation.OriginCommitSha
}
};
interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new Interop.TestMerge(x)));
interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new Interop.TestMerge(x, interopInfo.Revision)));
var interopJsonFile = JsonFile("interop");
@@ -163,15 +168,18 @@ namespace Tgstation.Server.Host.Components.Watchdog
var chatJsonTrackingContext = await chatJsonTrackingTask.ConfigureAwait(false);
try
{
//get the byond lock
var byondLock = currentByondLock ?? await byond.UseExecutables(Version.Parse(dmbProvider.CompileJob.ByondVersion), cancellationToken).ConfigureAwait(false);
try
{
//more sanitization here cause it uses the same scheme
var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson));
//create interop context
var context = new CommContext(ioManager, loggerFactory.CreateLogger<CommContext>(), basePath, interopInfo.ServerCommandsJson);
try
{
//set command line options
//more sanitization here cause it uses the same scheme
var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson));
var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} {2}-close -{3} -verbose -public -params \"{4}\"",
dmbProvider.DmbName,
primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort,
@@ -179,9 +187,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
SecurityWord(launchParameters.SecurityLevel.Value),
parameters);
//launch dd
var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments);
try
{
//return the session controller for it
return new SessionController(new ReattachInformation
{
AccessIdentifier = accessIdentifier,
@@ -192,7 +202,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ChatChannelsJson = interopInfo.ChatChannelsJson,
ChatCommandsJson = interopInfo.ChatCommandsJson,
ServerCommandsJson = interopInfo.ServerCommandsJson,
}, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger<SessionController>());
}, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger<SessionController>(), launchParameters.StartupTimeout);
}
catch
{
@@ -239,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
var process = processExecutor.GetProcess(reattachInformation.ProcessId);
try
{
return new SessionController(reattachInformation, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger<SessionController>());
return new SessionController(reattachInformation, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger<SessionController>(), null);
}
catch
{
@@ -20,9 +20,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
sealed class Watchdog : IWatchdog, ICustomCommandHandler
{
/// <summary>
/// The time in milliseconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
/// The time in seconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
/// </summary>
const int AlphaBravoStartupSeperationInterval = 3000;
const int AlphaBravoStartupSeperationInterval = 3;
/// <inheritdoc />
public bool Running { get; private set; }
@@ -187,6 +187,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
alphaServer = null;
bravoServer?.Dispose();
bravoServer = null;
Running = false;
}
/// <summary>
@@ -232,7 +233,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
var chatTask = announce ? chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask;
await StopMonitor().ConfigureAwait(false);
DisposeAndNullControllers();
Running = false;
await chatTask.ConfigureAwait(false);
return;
}
@@ -320,7 +320,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (dmbBackup == null) //NANI!?
//just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
throw new Exception("Creating backup DMB provider failed!");
throw new JobException("Creating backup DMB provider failed!");
monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
usedMostRecentDmb = false;
@@ -411,7 +411,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
case Components.Watchdog.RebootState.Shutdown:
await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
DisposeAndNullControllers();
Running = false;
monitorState.NextAction = MonitorAction.Exit;
return;
}
@@ -438,8 +437,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
monitorState.NextAction = MonitorAction.Break;
break;
case MonitorActivationReason.InactiveServerRebooted:
//should never happen but okay
logger.LogWarning("Inactive server rebooted, this is a bug in DM code!");
monitorState.RebootingInactiveServer = true;
monitorState.InactiveServer.ResetRebootState(); //the DMAPI has already done this internally
monitorState.ActiveServer.ClosePortOnReboot = false;
@@ -492,7 +489,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
var inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
var activeServerReboot = monitorState.ActiveServer.OnReboot;
var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
var inactiveServerStartup = monitorState.InactiveServer.LaunchResult;
var inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
var activeLaunchParametersChanged = activeParametersUpdated.Task;
var newDmbAvailable = dmbFactory.OnNewerDmb;
@@ -550,9 +547,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
else
moreActivationsToProcess = false;
if(moreActivationsToProcess)
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
}
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
//writeback alphaServer and bravoServer
alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
bravoServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
@@ -576,7 +575,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
monitorState = new MonitorState(); //clean the slate
}
await chatTask.ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
if(!Running)
{
logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString());
@@ -626,9 +625,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
async Task<WatchdogLaunchResult> LaunchNoLock(bool startMonitor, bool announce, bool doReattach, CancellationToken cancellationToken)
{
logger.LogTrace("Begin LaunchNoLock");
using (var alphaStartCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
logger.LogTrace("Begin LaunchNoLock");
if (Running)
{
logger.LogTrace("Aborted due to already running!");
@@ -655,32 +654,43 @@ namespace Tgstation.Server.Host.Components.Watchdog
var doesntNeedNewDmb = doReattach && reattachInfo.Alpha != null && reattachInfo.Bravo != null;
var dmbToUse = doesntNeedNewDmb ? null : dmbFactory.LockNextDmb(2);
Task<ISessionController> alphaServerTask = null;
try
{
Task<ISessionController> alphaServerTask;
if (!doesntNeedNewDmb)
alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, alphaStartCts.Token);
else
alphaServerTask = sessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
//do a few seconds of delay so that any backends the servers use know that alpha came first
await Task.Delay(AlphaBravoStartupSeperationInterval, cancellationToken).ConfigureAwait(false);
//wait until this boy officially starts so as not to confuse the servers as to who came first
var startTime = DateTimeOffset.Now;
alphaServer = await alphaServerTask.ConfigureAwait(false);
alphaServer.SetHighPriority();
//extra delay for total ordering
var now = DateTimeOffset.Now;
var delay = now - startTime;
if (delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
await Task.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
Task<ISessionController> bravoServerTask;
if (!doesntNeedNewDmb)
bravoServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken);
else
bravoServerTask = sessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken);
await Task.WhenAll(alphaServerTask, bravoServerTask).ConfigureAwait(false);
alphaServer = alphaServerTask.Result;
bravoServer = bravoServerTask.Result;
bravoServer = await bravoServerTask.ConfigureAwait(false);
bravoServer.SetHighPriority();
async Task<LaunchResult> CheckLaunch(ISessionController controller, string serverName)
{
var launch = await controller.LaunchResult.ConfigureAwait(false);
if (launch.ExitCode.HasValue)
//you killed us ray...
throw new Exception(String.Format(CultureInfo.InvariantCulture, "{1} server failed to start: {0}", launch.ToString(), serverName));
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server failed to start: {0}", launch.ToString(), serverName));
if (!launch.StartupTime.HasValue)
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server timed out on startup: {0}s", launch.ToString(), ActiveLaunchParameters.StartupTimeout.Value));
return launch;
}
@@ -793,12 +803,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
return true;
var builder = new StringBuilder(Constants.DMTopicEvent);
if (parameters != null)
foreach (var I in parameters)
{
builder.Append("&");
builder.Append(byondTopicSender.SanitizeString(I));
}
builder.Append("&");
var notification = new EventNotification
{
Type = eventType,
Parameters = parameters
};
var json = JsonConvert.SerializeObject(notification);
builder.Append(byondTopicSender.SanitizeString(json));
var activeServer = AlphaIsActive ? alphaServer : bravoServer;
results = await activeServer.SendCommand(builder.ToString(), cancellationToken).ConfigureAwait(false);
@@ -49,12 +49,10 @@ namespace Tgstation.Server.Host.Controllers
public override async Task<IActionResult> Read(CancellationToken cancellationToken)
{
var instance = instanceManager.GetInstance(Instance);
var projectNameTask = DatabaseContext.DreamMakerSettings.Where(x => x.InstanceId == Instance.Id).Select(x => x.ProjectName).FirstOrDefaultAsync(cancellationToken);
var job = await DatabaseContext.CompileJobs.OrderByDescending(x => x.Job.StartedAt).Include(x => x.Job).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
var projectName = await DatabaseContext.DreamMakerSettings.Where(x => x.InstanceId == Instance.Id).Select(x => x.ProjectName).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
return Json(new Api.Models.DreamMaker
{
LastJob = job?.ToApi(),
ProjectName = await projectNameTask.ConfigureAwait(false),
ProjectName = projectName,
Status = instance.DreamMaker.Status
});
}
@@ -162,7 +162,7 @@ namespace Tgstation.Server.Host.Controllers
using (var repos = await repoManager.CloneRepository(new Uri(origin), cloneBranch, currentModel.AccessUser, currentModel.AccessToken, progressReporter, cancellationToken).ConfigureAwait(false))
{
if (repos == null)
throw new Exception("Filesystem conflict while cloning repository!");
throw new JobException("Filesystem conflict while cloning repository!");
var db = serviceProvider.GetRequiredService<IDatabaseContext>();
if (await PopulateApi(api, repos, db, Instance, null, null, cancellationToken).ConfigureAwait(false))
await db.Save(cancellationToken).ConfigureAwait(false);
@@ -315,7 +315,7 @@ namespace Tgstation.Server.Host.Controllers
using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(ct).ConfigureAwait(false))
{
if (repo == null)
throw new InvalidOperationException("Repository could not be loaded!");
throw new JobException("Repository could not be loaded!");
var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null;
@@ -323,7 +323,7 @@ namespace Tgstation.Server.Host.Controllers
var startSha = repo.Head;
if (newTestMerges && !repo.IsGitHubRepository)
throw new InvalidOperationException("Cannot test merge on a non GitHub based repository!");
throw new JobException("Cannot test merge on a non GitHub based repository!");
var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName;
@@ -357,15 +357,15 @@ namespace Tgstation.Server.Host.Controllers
//fetch/pull
if (model.UpdateFromOrigin == true)
{
if (!repo.Tracking && model.Reference == null)
throw new InvalidOperationException("Not on an updatable reference!");
if (!repo.Tracking)
throw new JobException("Not on an updatable reference!");
await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), ct).ConfigureAwait(false);
doneFetches = 1;
if (!modelHasShaOrReference)
{
var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, ct).ConfigureAwait(false);
if (!fastForward.HasValue)
throw new InvalidOperationException("Merge conflict occurred during origin update!");
throw new JobException("Merge conflict occurred during origin update!");
await UpdateRevInfo().ConfigureAwait(false);
if (fastForward.Value)
{
@@ -388,7 +388,7 @@ namespace Tgstation.Server.Host.Controllers
if (model.UpdateFromOrigin == true && model.Reference != null)
{
if (!repo.Tracking)
throw new InvalidOperationException("Checked out reference does not track a remote object!");
throw new JobException("Checked out reference does not track a remote object!");
await repo.ResetToOrigin(ct).ConfigureAwait(false);
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, ct).ConfigureAwait(false);
await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false);
@@ -412,6 +412,7 @@ namespace Tgstation.Server.Host.Controllers
var repoName = repo.GitHubRepoName;
Models.RevisionInformation revInfoWereLookingFor = null;
bool needToApplyRemainingPrs = true;
//optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out
if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha)
{
@@ -445,20 +446,61 @@ namespace Tgstation.Server.Host.Controllers
if (!cantSearch)
{
var dbPull = await databaseContext.RevisionInformations
.Where(x => x.Instance.Id == Instance.Id
&& x.OriginCommitSha == lastRevisionInfo.OriginCommitSha
&& x.ActiveTestMerges.Count == model.NewTestMerges.Count)
.Where(x => x.Instance.Id == Instance.Id
&& x.OriginCommitSha == lastRevisionInfo.OriginCommitSha
&& x.ActiveTestMerges.Count <= model.NewTestMerges.Count
&& x.ActiveTestMerges.Count > 0)
.Include(x => x.ActiveTestMerges)
.ThenInclude(x => x.TestMerge)
.ToListAsync(cancellationToken).ConfigureAwait(false);
//split here cause this bit has to be done locally
revInfoWereLookingFor = dbPull
.Where(x => x.ActiveTestMerges.Select(y => y.TestMerge)
.All(y => model.NewTestMerges.Any(z =>
y.Number == z.Number
&& y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)
&& y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null)))
.Where(x => x.ActiveTestMerges.Count == model.NewTestMerges.Count
&& x.ActiveTestMerges.Select(y => y.TestMerge)
.All(y => model.NewTestMerges.Any(z =>
y.Number == z.Number
&& y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)
&& (y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null))))
.FirstOrDefault();
if (revInfoWereLookingFor == null && model.NewTestMerges.Count > 1)
{
//okay try to add at least SOME prs we've seen before
var search = model.NewTestMerges.ToList();
var appliedTestMergeIds = new List<long>();
Models.RevisionInformation lastGoodRevInfo = null;
do
{
foreach (var I in search)
{
revInfoWereLookingFor = dbPull
.Where(x => model.NewTestMerges.Any(z =>
x.PrimaryTestMerge.Number == z.Number
&& x.PrimaryTestMerge.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)
&& (x.PrimaryTestMerge.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null))
&& x.ActiveTestMerges.Select(y => y.TestMerge).All(y => appliedTestMergeIds.Contains(y.Id)))
.FirstOrDefault();
if (revInfoWereLookingFor != null)
{
lastGoodRevInfo = revInfoWereLookingFor;
appliedTestMergeIds.Add(revInfoWereLookingFor.PrimaryTestMerge.Id);
search.Remove(I);
break;
}
}
} while (revInfoWereLookingFor != null && search.Count > 0);
revInfoWereLookingFor = lastGoodRevInfo;
needToApplyRemainingPrs = search.Count != 0;
if (needToApplyRemainingPrs)
model.NewTestMerges = search;
}
else if (revInfoWereLookingFor != null)
needToApplyRemainingPrs = false;
}
}
@@ -468,7 +510,8 @@ namespace Tgstation.Server.Host.Controllers
await repo.ResetToSha(revInfoWereLookingFor.CommitSha, cancellationToken).ConfigureAwait(false);
lastRevisionInfo = revInfoWereLookingFor;
}
else
if(needToApplyRemainingPrs)
{
var contextUser = new Models.User
{
@@ -12,5 +12,10 @@ namespace Tgstation.Server.Host.Core
/// The <see cref="Task{TResult}"/> resulting in the exit code of the process
/// </summary>
Task<int> Lifetime { get; }
/// <summary>
/// Set's the owned <see cref="System.Diagnostics.Process.PriorityClass"/> to <see cref="System.Diagnostics.ProcessPriorityClass.AboveNormal"/>
/// </summary>
void SetHighPriority();
}
}
@@ -0,0 +1,34 @@
using System;
namespace Tgstation.Server.Host
{
/// <summary>
/// Operation exceptions thrown from the context of a <see cref="Models.Job"/>
/// </summary>
public sealed class JobException : Exception
{
/// <summary>
/// Construct a <see cref="JobException"/>
/// </summary>
public JobException()
{
}
/// <summary>
/// Construct a <see cref="JobException"/> with a <paramref name="message"/>
/// </summary>
/// <param name="message">The message for the <see cref="Exception"/></param>
public JobException(string message) : base(message)
{
}
/// <summary>
/// Construct a <see cref="JobException"/> with a <paramref name="message"/> and <paramref name="innerException"/>
/// </summary>
/// <param name="message">The message for the <see cref="Exception"/></param>
/// <param name="innerException">The inner <see cref="Exception"/> for the nase <see cref="Exception"/></param>
public JobException(string message, Exception innerException) : base(message, innerException)
{
}
}
}
+1 -1
View File
@@ -95,7 +95,7 @@ namespace Tgstation.Server.Host.Core
catch (Exception e)
{
logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e);
job.ExceptionDetails = e.ToString();
job.ExceptionDetails = e is JobException ? e.Message : e.ToString();
}
job.StoppedAt = DateTimeOffset.Now;
await databaseContext.Save(default).ConfigureAwait(false);
+12 -3
View File
@@ -50,7 +50,7 @@ namespace Tgstation.Server.Host.Core
{
if (combinedStringBuilder == null)
throw new InvalidOperationException("Output/Error reading was not enabled!");
return combinedStringBuilder.ToString();
return combinedStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
}
/// <inheritdoc />
@@ -58,7 +58,7 @@ namespace Tgstation.Server.Host.Core
{
if (errorStringBuilder == null)
throw new InvalidOperationException("Error reading was not enabled!");
return errorStringBuilder.ToString();
return errorStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
}
/// <inheritdoc />
@@ -66,7 +66,7 @@ namespace Tgstation.Server.Host.Core
{
if (outputStringBuilder == null)
throw new InvalidOperationException("Output reading was not enabled!");
return errorStringBuilder.ToString();
return errorStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
}
/// <inheritdoc />
@@ -79,5 +79,14 @@ namespace Tgstation.Server.Host.Core
}
catch (InvalidOperationException) { }
}
public void SetHighPriority()
{
try
{
handle.PriorityClass = System.Diagnostics.ProcessPriorityClass.AboveNormal;
}
catch (InvalidOperationException) { }
}
}
}
@@ -20,7 +20,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Byond.TopicSender" Version="1.1.1" />
<PackageReference Include="Byond.TopicSender" Version="1.1.2" />
<PackageReference Include="Cyberboss.AspNetCore.AsyncInitializer" Version="1.2.0" />
<PackageReference Include="Cyberboss.SmartIrc4net.Standard" Version="0.4.6" />
<PackageReference Include="Discord.Net.WebSocket" Version="1.0.2" />
+47 -4
View File
@@ -1,6 +1,6 @@
{
"info": {
"_postman_id": "6defe202-821a-488f-b8fe-e4017992655f",
"_postman_id": "b7e61e38-6891-473e-8ab4-869e5cf41a96",
"name": "TGS",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
@@ -1254,7 +1254,7 @@
],
"body": {
"mode": "raw",
"raw": "{\n\t\"userId\": 1,\n\t\"byondRights\": 4294967294,\n\t\"dreamDaemonRights\": 4294967294,\n\t\"dreamMakerRights\": 4294967294,\n\t\"repositoryRights\": 4294967294\n\t\"chatBotRights\": 4294967294,\n\t\"configurationRights\": 4294967294\n}"
"raw": "{\n\t\"userId\": 1,\n\t\"byondRights\": 4294967295,\n\t\"dreamDaemonRights\": 4294967295,\n\t\"dreamMakerRights\": 4294967295,\n\t\"repositoryRights\": 4294967295\n\t\"chatBotRights\": 4294967295,\n\t\"configurationRights\": 4294967295\n}"
},
"url": {
"raw": "localhost:5000/InstanceUser",
@@ -1297,7 +1297,7 @@
],
"body": {
"mode": "raw",
"raw": "{\n\t\"userId\": 1,\n\t\"byondRights\": 4294967294,\n\t\"dreamDaemonRights\": 4294967294,\n\t\"dreamMakerRights\": 4294967294,\n\t\"repositoryRights\": 4294967294,\n\t\"chatBotRights\": 4294967294,\n\t\"configurationRights\": 4294967294\n}"
"raw": "{\n\t\"userId\": 1,\n\t\"byondRights\": 4294967295,\n\t\"dreamDaemonRights\": 4294967295,\n\t\"dreamMakerRights\": 4294967295,\n\t\"repositoryRights\": 4294967295,\n\t\"chatBotRights\": 4294967295,\n\t\"configurationRights\": 4294967295\n}"
},
"url": {
"raw": "localhost:5000/InstanceUser",
@@ -1901,7 +1901,7 @@
],
"body": {
"mode": "raw",
"raw": "{\n\t\"updateFromOrigin\": true,\n\t\"reference\": \"master\",\n\t\"newTestMerges\": [\n\t\t{\n\t\t\t\"number\": 39147,\n\t\t\t\"comment\": \"electric boogaloo\"\n\t\t}\n\t\t]\n}"
"raw": "{\n\t\"updateFromOrigin\": true,\n\t\"reference\": \"master\",\n\t\"newTestMerges\": [\n\t\t{\n\t\t\t\"number\": 39771,\n\t\t\t\"comment\": \"needful for debugging\"\n\t\t},\n\t\t{\n\t\t\t\"number\": 39777,\n\t\t\t\"comment\": \"also needful\"\n\t\t},\n\t\t{\n\t\t\t\"number\": 39778\n\t\t}\n\t\t]\n}"
},
"url": {
"raw": "localhost:5000/Repository",
@@ -2717,6 +2717,49 @@
},
"response": []
},
{
"name": "Read",
"request": {
"method": "GET",
"header": [
{
"key": "Accept",
"value": "application/json"
},
{
"key": "User-Agent",
"value": "Postman/1.0"
},
{
"key": "Api",
"value": "Tgstation.Server.Api/4.0.0.0"
},
{
"key": "Content-Type",
"value": "application/json"
},
{
"key": "Instance",
"value": "1"
}
],
"body": {
"mode": "raw",
"raw": "{}"
},
"url": {
"raw": "localhost:5000/DreamDaemon",
"host": [
"localhost"
],
"port": "5000",
"path": [
"DreamDaemon"
]
}
},
"response": []
},
{
"name": "Stop",
"request": {
+3
View File
@@ -1,3 +1,6 @@
Verify the byond cache folder location on linux
Test watchdog
Only show user name and ID when serializing to API
In fact remove IApiConvertable<> altogether, it's not required by anything