From d3b5a10701facb280fbc9623f35819bc84b5af3d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 27 Nov 2018 16:04:17 -0500 Subject: [PATCH] Fix webroot issue in deployments. Propagate startup exceptions to the watchdog --- .../Components/InstanceManager.cs | 20 ++++- .../Core/IServerControl.cs | 7 ++ src/Tgstation.Server.Host/Server.cs | 90 ++++++++++++------- src/Tgstation.Server.Host/ServerFactory.cs | 19 +++- 4 files changed, 98 insertions(+), 38 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index b9b558cba2..b68c16c63d 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -39,6 +39,11 @@ namespace Tgstation.Server.Host.Components /// readonly IJobManager jobManager; + /// + /// The for the + /// + readonly IServerControl serverControl; + /// /// The for the /// @@ -67,7 +72,7 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - /// The used to register the as a + /// The value of /// The value of public InstanceManager(IInstanceFactory instanceFactory, IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IApplication application, IJobManager jobManager, IServerControl serverControl, ILogger logger) { @@ -76,8 +81,7 @@ namespace Tgstation.Server.Host.Components this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.application = application ?? throw new ArgumentNullException(nameof(application)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); - if (serverControl == null) - throw new ArgumentNullException(nameof(serverControl)); + this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); serverControl.RegisterForRestart(this); @@ -215,7 +219,7 @@ namespace Tgstation.Server.Host.Components await factoryStartup.ConfigureAwait(false); await dbInstances.ForEachAsync(metadata => tasks.Add(metadata.Online.Value ? OnlineInstance(metadata, cancellationToken) : Task.CompletedTask), cancellationToken).ConfigureAwait(false); await Task.WhenAll(tasks).ConfigureAwait(false); - logger.LogInformation("Instance manager ready!"); + logger.LogInformation("Server ready!"); application.Ready(null); } catch (OperationCanceledException) @@ -226,6 +230,14 @@ namespace Tgstation.Server.Host.Components { logger.LogCritical("Instance manager startup error! Exception: {0}", e); application.Ready(e); + try + { + await serverControl.Die(e).ConfigureAwait(false); + } + catch (Exception e2) + { + logger.LogCritical("Failed to kill server! Exception: {0}", e2); + } } }); diff --git a/src/Tgstation.Server.Host/Core/IServerControl.cs b/src/Tgstation.Server.Host/Core/IServerControl.cs index 184280b033..03b5b10a67 100644 --- a/src/Tgstation.Server.Host/Core/IServerControl.cs +++ b/src/Tgstation.Server.Host/Core/IServerControl.cs @@ -36,5 +36,12 @@ namespace Tgstation.Server.Host.Core /// /// A representing the running operation Task Restart(); + + /// + /// Kill the server with a fatal exception + /// + /// The to propagate to the watchdog if any + /// A representing the running operation + Task Die(Exception exception); } } diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 1786fd6a19..cee0258581 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -48,6 +48,11 @@ namespace Tgstation.Server.Host /// CancellationTokenSource cancellationTokenSource; + /// + /// The to propagate when the server terminates + /// + Exception propagatedException; + /// /// If a server update has been or is being applied /// @@ -63,6 +68,8 @@ namespace Tgstation.Server.Host this.webHostBuilder = webHostBuilder ?? throw new ArgumentNullException(nameof(webHostBuilder)); this.updatePath = updatePath; + webHostBuilder.ConfigureServices(serviceCollection => serviceCollection.AddSingleton(this)); + restartHandlers = new List(); } @@ -72,13 +79,22 @@ namespace Tgstation.Server.Host /// If should be checked void CheckSanity(bool checkWatchdog) { - if (checkWatchdog && !WatchdogPresent) + if (checkWatchdog && !WatchdogPresent && propagatedException == null) throw new InvalidOperationException("Server restarts are not supported"); if (cancellationTokenSource == null || logger == null) throw new InvalidOperationException("Tried to control a non-running Server!"); } + /// + /// Re-throw if it exists + /// + void CheckExceptionPropagation() + { + if (propagatedException != null) + throw propagatedException; + } + /// public async Task RunAsync(CancellationToken cancellationToken) { @@ -94,16 +110,20 @@ namespace Tgstation.Server.Host }; fsWatcher.EnableRaisingEvents = true; } - using (var webHost = webHostBuilder - .UseStartup() - .ConfigureServices(serviceCollection => serviceCollection.AddSingleton(this)) - .Build() - ) - { - logger = webHost.Services.GetRequiredService>(); - await webHost.RunAsync(cancellationTokenSource.Token).ConfigureAwait(false); - } + + using (var webHost = webHostBuilder.Build()) + try + { + logger = webHost.Services.GetRequiredService>(); + await webHost.RunAsync(cancellationTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + CheckExceptionPropagation(); + throw; + } } + CheckExceptionPropagation(); } /// @@ -163,7 +183,7 @@ namespace Tgstation.Server.Host throw; } - await Restart(version).ConfigureAwait(false); + await Restart(version, null).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -207,14 +227,15 @@ namespace Tgstation.Server.Host } /// - public Task Restart() => Restart(null); + public Task Restart() => Restart(null, null); /// /// Implements /// /// The of any potential updates being applied + /// The potential value of /// - async Task Restart(Version newVersion) + async Task Restart(Version newVersion, Exception exception) { CheckSanity(true); @@ -228,33 +249,36 @@ namespace Tgstation.Server.Host return; } RestartRequested = true; + propagatedException = exception; } - logger.LogInformation("Restarting server..."); - - using (var cts = new CancellationTokenSource()) - { - logger.LogTrace("Running restart handlers..."); - var cancellationToken = cts.Token; - var eventsTask = Task.WhenAll(restartHandlers.Select(x => x.HandleRestart(newVersion, cancellationToken)).ToList()); - //YA GOT 10 SECONDS - var expiryTask = Task.Delay(TimeSpan.FromSeconds(10)); - await Task.WhenAny(eventsTask, expiryTask).ConfigureAwait(false); - logger.LogTrace("Joining restart handlers..."); - cts.Cancel(); - try + if (exception == null) + using (var cts = new CancellationTokenSource()) { - await eventsTask.ConfigureAwait(false); + logger.LogInformation("Restarting server..."); + var cancellationToken = cts.Token; + var eventsTask = Task.WhenAll(restartHandlers.Select(x => x.HandleRestart(newVersion, cancellationToken)).ToList()); + //YA GOT 10 SECONDS + var expiryTask = Task.Delay(TimeSpan.FromSeconds(10)); + await Task.WhenAny(eventsTask, expiryTask).ConfigureAwait(false); + logger.LogTrace("Joining restart handlers..."); + cts.Cancel(); + try + { + await eventsTask.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + catch (Exception e) + { + logger.LogError("Restart handlers error! Exception: {0}", e); + } } - catch (OperationCanceledException) { } - catch (Exception e) - { - logger.LogError("Restart handlers error! Exception: {0}", e); - } - } logger.LogTrace("Stopping host..."); cancellationTokenSource.Cancel(); } + + /// + public Task Die(Exception exception) => Restart(null, exception); } } diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index cefdfd8897..26d77ab31f 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -1,5 +1,10 @@ using Microsoft.AspNetCore; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; using System; +using System.IO; +using System.Reflection; +using Tgstation.Server.Host.Core; namespace Tgstation.Server.Host { @@ -7,6 +12,18 @@ namespace Tgstation.Server.Host public sealed class ServerFactory : IServerFactory { /// - public IServer CreateServer(string[] args, string updatePath) => new Server(WebHost.CreateDefaultBuilder(args ?? throw new ArgumentNullException(nameof(args))), updatePath); + public IServer CreateServer(string[] args, string updatePath) { + + var webHost = WebHost.CreateDefaultBuilder(args ?? throw new ArgumentNullException(nameof(args))) + .ConfigureAppConfiguration((context, configurationBuilder) => configurationBuilder.SetBasePath(Directory.GetCurrentDirectory())) + .UseStartup() + .SuppressStatusMessages(true) + .UseShutdownTimeout(TimeSpan.FromMinutes(1)); + + if(updatePath != null) + webHost.UseContentRoot(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); + + return new Server(webHost, updatePath); + } } }