Always dreaming never stopping

src/Tgstation.Server.Host/Components/Watchdog/Executor.cs

@@ -45,7 +45,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				throw new ArgumentNullException(nameof(parameters));
 
 			var proc = new Process();
-
 			try
 			{
 				proc.StartInfo.FileName = byondLock.DreamDaemonPath;

src/Tgstation.Server.Host/Components/Watchdog/ISessionControllerFactory.cs

@@ -9,8 +9,24 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	/// </summary>
     interface ISessionControllerFactory
     {
+		/// <summary>
+		/// Create a <see cref="ISessionController"/> from a freshly launch DreamDaemon instance
+		/// </summary>
+		/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/> to use</param>
+		/// <param name="dmbProvider">The <see cref="IDmbProvider"/> to use</param>
+		/// <param name="primaryPort">If the <see cref="DreamDaemonLaunchParameters.PrimaryPort"/> of <paramref name="launchParameters"/> should be used</param>
+		/// <param name="primaryDirectory">If the <see cref="IDmbProvider.PrimaryDirectory"/> of <paramref name="dmbProvider"/> should be used</param>
+		/// <param name="apiValidate">If the <see cref="ISessionController"/> should only validate the DMAPI then exit</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in a new <see cref="ISessionController"/></returns>
 		Task<ISessionController> LaunchNew(DreamDaemonLaunchParameters launchParameters, IDmbProvider dmbProvider, bool primaryPort, bool primaryDirectory, bool apiValidate, CancellationToken cancellationToken);
 
+		/// <summary>
+		/// Create a <see cref="ISessionController"/> from an existing DreamDaemon instance
+		/// </summary>
+		/// <param name="reattachInformation">The <see cref="ReattachInformation"/> to use</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in a new <see cref="ISessionController"/></returns>
 		Task<ISessionController> Reattach(ReattachInformation reattachInformation, CancellationToken cancellationToken);
     }
 }

src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
 using Tgstation.Server.Api.Models.Internal;
+using Tgstation.Server.Host.Core;
+using Tgstation.Server.Host.Models;
 
 namespace Tgstation.Server.Host.Components.Watchdog
 {
@@ -9,11 +12,28 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	sealed class Watchdog : IWatchdog
 	{
 		/// <inheritdoc />
-		public DreamDaemonLaunchParameters LaunchParameters
-		{
-			get => launchParameters;
-			set => launchParameters = value ?? throw new ArgumentNullException(nameof(value));
-		}
+		public bool Running { get; private set; }
+
+		/// <inheritdoc />
+		public bool AlphaIsActive { get; private set; }
+
+		/// <inheritdoc />
+		public LaunchResult LastLaunchResult { get; private set; }
+
+		/// <inheritdoc />
+		public Models.CompileJob LiveCompileJob { get; private set; }
+
+		/// <inheritdoc />
+		public Models.CompileJob StagedCompileJob { get; private set; }
+
+		/// <inheritdoc />
+		public DreamDaemonLaunchParameters ActiveLaunchParameters { get; private set; }
+
+		/// <inheritdoc />
+		public DreamDaemonLaunchParameters LastLaunchParameters { get; private set; }
+
+		/// <inheritdoc />
+		public RebootState? RebootState => Running ? (RebootState?)(AlphaIsActive ? alphaServer.RebootState : bravoServer.RebootState) : null;
 
 		/// <summary>
 		/// The <see cref="IChat"/> for the <see cref="Watchdog"/>
@@ -23,7 +43,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// The <see cref="ISessionControllerFactory"/> for the <see cref="Watchdog"/>
 		/// </summary>
-		readonly ISessionControllerFactory sessionManagerFactory;
+		readonly ISessionControllerFactory sessionControllerFactory;
 
 		/// <summary>
 		/// The <see cref="IEventConsumer"/> for the <see cref="Watchdog"/>
@@ -40,42 +60,216 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		readonly IDmbFactory dmbFactory;
 
-		DreamDaemonLaunchParameters launchParameters;
+
+		/// <summary>
+		/// The <see cref="SemaphoreSlim"/> for the <see cref="Watchdog"/>
+		/// </summary>
+		readonly SemaphoreSlim semaphore;
 
 		ISessionController alphaServer;
 		ISessionController bravoServer;
 
-		bool alphaActive;
-
-		bool disposed;
+		/// <summary>
+		/// If the servers should be released instead of shutdown
+		/// </summary>
+		bool releaseServers;
 
 		/// <summary>
 		/// Construct a <see cref="IWatchdog"/>
 		/// </summary>
 		/// <param name="chat">The value of <see cref="chat"/></param>
-		/// <param name="sessionManagerFactory">The value of <see cref="sessionManagerFactory"/></param>
+		/// <param name="sessionControllerFactory">The value of <see cref="sessionControllerFactory"/></param>
 		/// <param name="dmbFactory">The value of <see cref="dmbFactory"/></param>
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
 		/// <param name="interopRegistrar">The value of <see cref="interopRegistrar"/></param>
-		public Watchdog(IByond byond, IChat chat, ISessionControllerFactory sessionManagerFactory, IDmbFactory dmbFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar, DreamDaemonLaunchParameters initialLaunchParameters)
+		/// <param name="serverUpdater">The <see cref="IServerUpdater"/> for the <see cref="Watchdog"/></param>
+		/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/></param>
+		public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar, IServerUpdater serverUpdater, DreamDaemonLaunchParameters initialLaunchParameters)
 		{
 			this.chat = chat ?? throw new ArgumentNullException(nameof(chat));
-			this.sessionManagerFactory = sessionManagerFactory ?? throw new ArgumentNullException(nameof(sessionManagerFactory));
+			this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
 			this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
 			this.interopRegistrar = interopRegistrar ?? throw new ArgumentNullException(nameof(interopRegistrar));
-			LaunchParameters = initialLaunchParameters;
+
+			if(serverUpdater == null)
+				throw new ArgumentNullException(nameof(serverUpdater));
+
+			serverUpdater.RegisterForUpdate(() => releaseServers = true);
+
+			AlphaIsActive = true;
+			ActiveLaunchParameters = initialLaunchParameters;
+			releaseServers = false;
+			semaphore = new SemaphoreSlim(1);
 		}
 
 		/// <inheritdoc />
 		public void Dispose()
 		{
-			lock (this)
+			DisposeAndNullControllers();
+			semaphore.Dispose();
+		}
+
+
+		void DisposeAndNullControllers()
+		{
+			alphaServer?.Dispose();
+			alphaServer = null;
+			bravoServer?.Dispose();
+			bravoServer = null;
+		}
+
+		/// <inheritdoc />
+		public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
+		{
+			using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
 			{
-				alphaServer?.Dispose();
-				bravoServer?.Dispose();
-				disposed = true;
+				ActiveLaunchParameters = launchParameters;
+				if (Running)
+					await RestartNoLock(true, cancellationToken).ConfigureAwait(false);
+
+				//WHEN YOU GET BACK
+				//change graceful restart to immediately restart the inactive server and then relaunch it
+				//this func is done, don't worry
 			}
 		}
+
+		/// <inheritdoc />
+		public async Task<WatchdogLaunchResult> Launch(CancellationToken cancellationToken)
+		{
+			using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
+			using(var alphaStartCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
+			{
+				if (Running)
+					return null;
+				//start both servers
+				LastLaunchParameters = ActiveLaunchParameters;
+				var dmbToUse = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false);
+				Task<ISessionController> alphaServerTask = null;
+
+				try
+				{
+					try
+					{
+						alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, true, true, false, alphaStartCts.Token);
+						//do a few seconds of delay so that any backends the servers use know that alpha came first
+						await Task.Delay(5000, cancellationToken).ConfigureAwait(false);
+						var bravoServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, false, false, false, cancellationToken);
+						bravoServer = await bravoServerTask.ConfigureAwait(false);
+						alphaServer = await alphaServerTask.ConfigureAwait(false);
+					}
+					catch
+					{
+						if (alphaServerTask != null)
+							if (alphaServerTask.Status == TaskStatus.RanToCompletion)
+								alphaServer = await alphaServerTask.ConfigureAwait(false);
+							else
+							{
+								alphaStartCts.Cancel();
+								try
+								{
+									alphaServer = await alphaServerTask.ConfigureAwait(false);
+								}
+								catch { }
+							}
+						throw;
+					}
+
+					async Task<LaunchResult> CheckLaunch(ISessionController controller, string serverName)
+					{
+						var launch = await controller.LaunchResult.ConfigureAwait(false);
+						if (launch.ExitCode.HasValue)
+							//you killed us ray...
+							throw new Exception(String.Format(CultureInfo.InvariantCulture, "{2} server failed to start: Exit Code: {0}, RAM: {1}, Runtime {3}ms", launch.ExitCode, launch.StartupTime, serverName, launch.StartupTime.TotalMilliseconds));
+						return launch;
+					}
+
+					var alphaLrt = CheckLaunch(alphaServer, "Alpha");
+					var bravoLrt = CheckLaunch(bravoServer, "Bravo");
+					//now we have two booting servers, get them up and running
+					var allTask = Task.WhenAll(alphaLrt, bravoLrt);
+
+					//don't forget about the cancelationToken
+					var cancelTcs = new TaskCompletionSource<object>();
+					using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
+						await Task.WhenAny(allTask, cancelTcs.Task).ConfigureAwait(false);
+
+					//both servers are now running, alpha is the active server, huzzah
+					LiveCompileJob = dmbToUse.CompileJob;
+					LastLaunchResult = alphaLrt.Result;
+					StagedCompileJob = null;
+					AlphaIsActive = true;
+					Running = true;
+
+					return new WatchdogLaunchResult
+					{
+						Alpha = alphaLrt.Result,
+						Bravo = bravoLrt.Result
+					};
+				}
+				catch
+				{
+					DisposeAndNullControllers();
+					throw;
+				}
+			}
+		}
+
+		async Task<WatchdogLaunchResult> RestartNoLock(bool graceful, CancellationToken cancellationToken)
+		{
+			var running = Running;
+			if (!graceful || !running)
+			{
+				if (running)
+					await Terminate(false, cancellationToken).ConfigureAwait(false);
+				return await Launch(cancellationToken).ConfigureAwait(false);
+			}
+			var toReboot = AlphaIsActive ? alphaServer : bravoServer;
+			if (toReboot != null)
+				await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
+			return null;
+		}
+
+		/// <inheritdoc />
+		public async Task<WatchdogLaunchResult> Restart(bool graceful, CancellationToken cancellationToken)
+		{
+			using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
+				return await RestartNoLock(graceful, cancellationToken).ConfigureAwait(false);
+		}
+
+		/// <inheritdoc />
+		public async Task Terminate(bool graceful, CancellationToken cancellationToken)
+		{
+			using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
+			{
+				if (!Running)
+					return;
+				if (!graceful)
+				{
+					DisposeAndNullControllers();
+					return;
+				}
+				var toKill = AlphaIsActive ? alphaServer : bravoServer;
+				if (toKill != null)
+					await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
+			}
+		}
+
+		/// <inheritdoc />
+		public Task StartAsync(CancellationToken cancellationToken) => Launch(cancellationToken);
+
+		/// <inheritdoc />
+		public Task StopAsync(CancellationToken cancellationToken)
+		{
+			if (releaseServers)
+			{
+				ReattachInformation reattachInformation;
+				if (AlphaIsActive)
+					reattachInformation = alphaServer?.Release();
+				else
+					reattachInformation = bravoServer?.Release();
+			}
+			return Terminate(false, cancellationToken);
+		}
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs

@@ -1,5 +1,6 @@
 using System;
 using Tgstation.Server.Api.Models.Internal;
+using Tgstation.Server.Host.Core;
 
 namespace Tgstation.Server.Host.Components.Watchdog
 {
@@ -31,6 +32,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		readonly IInteropRegistrar interopRegistrar;
 
+		/// <summary>
+		/// The <see cref="IServerUpdater"/> for the <see cref="WatchdogFactory"/>
+		/// </summary>
+		readonly IServerUpdater serverUpdater;
+
 		/// <summary>
 		/// Construct a <see cref="WatchdogFactory"/>
 		/// </summary>
@@ -39,16 +45,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="sessionManagerFactory">The value of <see cref="sessionManagerFactory"/></param>
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
 		/// <param name="interopRegistrar">The value of <see cref="interopRegistrar"/></param>
-		public WatchdogFactory(IByond byond, IChat chat, ISessionControllerFactory sessionManagerFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar)
+		/// <param name="serverUpdater">The value of <see cref="serverUpdater"/></param>
+		public WatchdogFactory(IChat chat, ISessionControllerFactory sessionManagerFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar, IServerUpdater serverUpdater)
 		{
-			this.byond = byond ?? throw new ArgumentNullException(nameof(byond));
 			this.chat = chat ?? throw new ArgumentNullException(nameof(chat));
 			this.sessionManagerFactory = sessionManagerFactory ?? throw new ArgumentNullException(nameof(sessionManagerFactory));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
 			this.interopRegistrar = interopRegistrar ?? throw new ArgumentNullException(nameof(interopRegistrar));
+			this.serverUpdater = serverUpdater ?? throw new ArgumentNullException(nameof(serverUpdater));
 		}
 
 		/// <inheritdoc />
-		public IWatchdog CreateWatchdog(IDmbFactory dmbFactory, DreamDaemonLaunchParameters launchParameters) => new Watchdog(byond, chat, sessionManagerFactory, dmbFactory, eventConsumer, interopRegistrar, launchParameters);
+		public IWatchdog CreateWatchdog(IDmbFactory dmbFactory, DreamDaemonLaunchParameters launchParameters) => new Watchdog(chat, sessionManagerFactory, dmbFactory, eventConsumer, interopRegistrar, serverUpdater, launchParameters);
 	}
 }

src/Tgstation.Server.Host/Core/IServerUpdater.cs

@@ -1,14 +1,22 @@
-namespace Tgstation.Server.Host.Core
+using System;
+
+namespace Tgstation.Server.Host.Core
 {
 	/// <summary>
 	/// Represents a service that may take an updated <see cref="Host"/> assembly and run it, stopping the current assembly in the process
 	/// </summary>
-    interface IServerUpdateConsumer
+    interface IServerUpdater
     {
 		/// <summary>
 		/// Run a new <see cref="Host"/> assembly and stop the current one. This will likely trigger all active <see cref="System.Threading.CancellationToken"/>s
 		/// </summary>
 		/// <param name="updatePath">The path to the new <see cref="Host"/> assembly</param>
 		void ApplyUpdate(string updatePath);
+
+		/// <summary>
+		/// Register a given <paramref name="action"/> to run before stopping the server for updates
+		/// </summary>
+		/// <param name="action">The <see cref="Action"/> to run</param>
+		void RegisterForUpdate(Action action);
     }
 }

src/Tgstation.Server.Host/Core/SemaphoreContext.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Tgstation.Server.Host.Core
+{
+	/// <summary>
+	/// Async lock context helper
+	/// </summary>
+	public sealed class SemaphoreContext : IDisposable
+	{
+		/// <summary>
+		/// Asyncronously locks a <paramref name="semaphore"/>
+		/// </summary>
+		/// <param name="semaphore">The <see cref="SemaphoreSlim"/> to lock</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="SemaphoreSlimContext"/> for the lock</returns>
+		public static async Task<SemaphoreContext> Lock(SemaphoreSlim semaphore, CancellationToken cancellationToken)
+		{
+			if (semaphore == null)
+				throw new ArgumentNullException(nameof(semaphore));
+			await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
+			cancellationToken.ThrowIfCancellationRequested();
+			return new SemaphoreContext(semaphore);
+		}
+
+		/// <summary>
+		/// The locked <see cref="SemaphoreSlim"/>
+		/// </summary>
+		readonly SemaphoreSlim lockedSemaphore;
+
+		/// <summary>
+		/// If <see cref="Dispose"/> has been called
+		/// </summary>
+		bool disposed;
+
+		/// <summary>
+		/// Construct a <see cref="SemaphoreSlimContext"/>
+		/// </summary>
+		/// <param name="lockedSemaphore">The value of <see cref="lockedSemaphore"/></param>
+		SemaphoreContext(SemaphoreSlim lockedSemaphore) => this.lockedSemaphore = lockedSemaphore;
+
+		/// <summary>
+		/// Finalize the <see cref="SemaphoreSlimContext"/>
+		/// </summary>
+		~SemaphoreContext() => Dispose();
+
+		/// <summary>
+		/// Release the lock on <see cref="lockedSemaphore"/>
+		/// </summary>
+		public void Dispose()
+		{
+			if (disposed)
+				return;
+			lockedSemaphore.Release();
+			GC.SuppressFinalize(this);
+			disposed = true;
+		}
+	}
+}

src/Tgstation.Server.Host/Server.cs

@@ -10,7 +10,7 @@ using Tgstation.Server.Host.Startup;
 namespace Tgstation.Server.Host
 {
 	/// <inheritdoc />
-	sealed class Server : IServer, IServerUpdateConsumer
+	sealed class Server : IServer, IServerUpdater
 	{
 		/// <inheritdoc />
 		public string UpdatePath { get; private set; }
@@ -35,11 +35,10 @@ namespace Tgstation.Server.Host
         [ExcludeFromCodeCoverage]
 		public async Task RunAsync(CancellationToken cancellationToken)
 		{
-			cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-			using (cancellationTokenSource)
+			using (cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
 			using (var webHost = webHostBuilder
 				.UseStartup<Application>()
-				.ConfigureServices((serviceCollection) => serviceCollection.AddSingleton<IServerUpdateConsumer>(this))
+				.ConfigureServices((serviceCollection) => serviceCollection.AddSingleton<IServerUpdater>(this))
 				.Build()
 			)
 				await webHost.RunAsync(cancellationToken).ConfigureAwait(false);
@@ -56,5 +55,8 @@ namespace Tgstation.Server.Host
 			}
 			cancellationTokenSource.Cancel();
 		}
+
+		/// <inheritdoc />
+		public void RegisterForUpdate(Action action) => cancellationTokenSource.Token.Register(action);
 	}
 }

tests/Tgstation.Server.Host.Tests/Core/TestApplication.cs

@@ -7,7 +7,7 @@ using System.Threading.Tasks;
 namespace Tgstation.Server.Host.Core.Tests
 {
 	[TestClass]
-	public sealed class TestApplication : IServerUpdateConsumer
+	public sealed class TestApplication : IServerUpdater
 	{
 		public void ApplyUpdate(string updatePath) => throw new System.NotImplementedException();