From 8387eb50adee4235fcd896f68a246db38147e216 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 4 Nov 2024 18:32:34 -0500 Subject: [PATCH 01/25] Add self bootstrapping to host watchdog --- build/Version.props | 2 +- .../Tgstation.Server.Host.Console.csproj | 1 + .../Tgstation.Server.Host.Service.csproj | 1 + .../BootstrapSettings.cs | 38 ++++++ .../Watchdog.cs | 112 +++++++++++++++--- 5 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs diff --git a/build/Version.props b/build/Version.props index 21e5da987b..02bbc0e30a 100644 --- a/build/Version.props +++ b/build/Version.props @@ -12,7 +12,7 @@ 19.3.0 7.3.0 5.10.0 - 1.5.0 + 1.6.0 8.0.0 1.2.1 2.0.0 diff --git a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj index ff23533a16..9804e251e3 100644 --- a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj +++ b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj @@ -4,6 +4,7 @@ Exe $(TgsFrameworkVersion) + $(TgsCoreVersion) enable false diff --git a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj index 417e03092c..e1f04e898b 100644 --- a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj +++ b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -7,6 +7,7 @@ enable $(TgsFrameworkVersion) + $(TgsCoreVersion) false diff --git a/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs b/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs new file mode 100644 index 0000000000..10c9ab11b4 --- /dev/null +++ b/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs @@ -0,0 +1,38 @@ +using System; +using System.Reflection; + +using Tgstation.Server.Common.Extensions; + +namespace Tgstation.Server.Host.Watchdog +{ + /// + /// Settings for the bootstrapper feature. + /// + sealed class BootstrapSettings + { + /// + /// The current supported major version of . + /// + public const int FileMajorVersion = 1; + + /// + /// The token used to substitute . + /// + public const string VersionSubstitutionToken = "${version}"; + + /// + /// The version of the boostrapper file. + /// + public Version FileVersion { get; set; } = new Version(FileMajorVersion, 0, 0); + + /// + /// The of TGS last launched in the lib/Default directory. + /// + public Version TgsVersion { get; set; } = Assembly.GetEntryAssembly()!.GetName().Version!.Semver(); + + /// + /// The URL to format with to get the download URL. + /// + public string ServerUpdatePackageUrlFormatter { get; set; } = $"https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v{VersionSubstitutionToken}/ServerUpdatePackage.zip"; + } +} diff --git a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs index 4813383440..c139b9ac36 100644 --- a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs @@ -3,9 +3,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; +using System.IO.Compression; using System.Linq; +using System.Net.Http; using System.Reflection; using System.Runtime.InteropServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -119,6 +122,49 @@ namespace Tgstation.Server.Host.Watchdog else Directory.CreateDirectory(assemblyStoragePath); + var bootstrappable = args.Contains("--bootstrap", StringComparer.OrdinalIgnoreCase); + var homeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + var bootstrapperSettingsFile = Path.Combine(homeDirectory, ".tgstation-server-bootstrap.json"); + BootstrapSettings? bootstrapSettings = null; + if (!Directory.Exists(defaultAssemblyPath)) + { + if (!bootstrappable) + { + logger.LogCritical("Unable to locate host assembly directory!"); + return false; + } + + if (File.Exists(bootstrapperSettingsFile)) + { + logger.LogInformation("Loading bootstrap settings..."); + var bootstrapperSettingsJson = await File.ReadAllTextAsync(bootstrapperSettingsFile, cancellationToken); + bootstrapSettings = JsonSerializer.Deserialize(bootstrapperSettingsJson); + if (bootstrapSettings == null) + { + logger.LogCritical("Failed to deserialize {settingsFile}!", bootstrapperSettingsFile); + return false; + } + } + else + { + logger.LogInformation("Using default bootstrap settings..."); + bootstrapSettings = new BootstrapSettings(); // defaults + } + + if (bootstrapSettings.FileVersion.Major != BootstrapSettings.FileMajorVersion) + { + logger.LogCritical("Unable to parse bootstrapper file! Expected version: {expected}.X.X", BootstrapSettings.FileMajorVersion); + return false; + } + + string downloadUrl = bootstrapSettings.ServerUpdatePackageUrlFormatter.Replace(BootstrapSettings.VersionSubstitutionToken, bootstrapSettings.TgsVersion.ToString(), StringComparison.Ordinal); + logger.LogInformation("Bootstrapping from: {url}", downloadUrl); + using var httpClient = new HttpClient(); + await using var zipData = await httpClient.GetStreamAsync(new Uri(downloadUrl), cancellationToken); + using var archive = new ZipArchive(zipData, ZipArchiveMode.Read, true); + archive.ExtractToDirectory(defaultAssemblyPath); + } + var assemblyName = String.Join(".", nameof(Tgstation), nameof(Server), nameof(Host), "dll"); var assemblyPath = Path.Combine(defaultAssemblyPath, assemblyName); @@ -128,28 +174,59 @@ namespace Tgstation.Server.Host.Watchdog return false; } - if (!File.Exists(assemblyPath)) - { - logger.LogCritical("Unable to locate host assembly!"); - return false; - } - - var fileVersion = FileVersionInfo.GetVersionInfo(assemblyPath).FileVersion; - if (fileVersion == null) - { - logger.LogCritical("Failed to parse version info from {assemblyPath}!", assemblyPath); - return false; - } - - initialHostVersionTcs.SetResult( - Version.Parse( - fileVersion)); - var watchdogVersion = executingAssembly.GetName().Version?.Semver().ToString(); while (!cancellationToken.IsCancellationRequested) + { + if (!File.Exists(assemblyPath)) + { + logger.LogCritical("Unable to locate host assembly!"); + return false; + } + + var fileVersion = FileVersionInfo.GetVersionInfo(assemblyPath).FileVersion; + if (fileVersion == null) + { + logger.LogCritical("Failed to parse version info from {assemblyPath}!", assemblyPath); + return false; + } + + if (bootstrappable) + { + if (!Version.TryParse(fileVersion, out var bootstrappedVersion)) + { + logger.LogCritical("Failed to parse bootstrapped version prior to launch: {fileVersion}", fileVersion); + } + else + { + // save bootstrapper settings + var oldUrl = bootstrapSettings?.ServerUpdatePackageUrlFormatter; + bootstrapSettings = new BootstrapSettings + { + TgsVersion = bootstrappedVersion.Semver(), + }; + + bootstrapSettings.ServerUpdatePackageUrlFormatter = oldUrl ?? bootstrapSettings.ServerUpdatePackageUrlFormatter; + + Directory.CreateDirectory(homeDirectory); + await File.WriteAllTextAsync( + bootstrapperSettingsFile, + JsonSerializer.Serialize( + bootstrapSettings, + new JsonSerializerOptions + { + WriteIndented = true, + }), + cancellationToken); + } + } + using (logger.BeginScope("Host invocation")) { + initialHostVersionTcs.SetResult( + Version.Parse( + fileVersion)); + updateDirectory = Path.GetFullPath(Path.Combine(assemblyStoragePath, Guid.NewGuid().ToString())); logger.LogInformation("Update path set to {updateDirectory}", updateDirectory); using (var process = new Process()) @@ -358,6 +435,7 @@ namespace Tgstation.Server.Host.Watchdog } } } + } } catch (OperationCanceledException ex) { From 2b243f67b2758667586b3d4fd93bd1b1143a4bbf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 4 Nov 2024 21:15:32 -0500 Subject: [PATCH 02/25] Always store bootstrapped bins in home directory --- src/Tgstation.Server.Host.Watchdog/Watchdog.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs index c139b9ac36..09a985c9dd 100644 --- a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs @@ -92,7 +92,10 @@ namespace Tgstation.Server.Host.Watchdog return false; } - var assemblyStoragePath = Path.Combine(rootLocation, "lib"); // always always next to watchdog + var bootstrappable = args.Contains("--bootstrap", StringComparer.OrdinalIgnoreCase); + var homeDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".tgstation-server"); + + var assemblyStoragePath = Path.Combine(bootstrappable ? homeDirectory : rootLocation, "lib"); // always always next to watchdog var defaultAssemblyPath = Path.GetFullPath(Path.Combine(assemblyStoragePath, "Default")); @@ -122,9 +125,7 @@ namespace Tgstation.Server.Host.Watchdog else Directory.CreateDirectory(assemblyStoragePath); - var bootstrappable = args.Contains("--bootstrap", StringComparer.OrdinalIgnoreCase); - var homeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - var bootstrapperSettingsFile = Path.Combine(homeDirectory, ".tgstation-server-bootstrap.json"); + var bootstrapperSettingsFile = Path.Combine(homeDirectory, "bootstrap.json"); BootstrapSettings? bootstrapSettings = null; if (!Directory.Exists(defaultAssemblyPath)) { From 0442224e759782c4a35940c8bee76e2c7f195c6b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 17:21:24 -0500 Subject: [PATCH 03/25] Fix failed event script double setting a TaskCompletionSource --- build/Version.props | 2 +- src/Tgstation.Server.Host/Components/Engine/EngineManager.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/Version.props b/build/Version.props index 02bbc0e30a..41b4c978dd 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 6.12.0 + 6.11.4 5.4.0 10.12.0 0.5.0 diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index 970e3ab89d..9b1d5b61f6 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -477,9 +477,9 @@ namespace Tgstation.Server.Host.Components.Engine await InstallVersionFiles(progressReporter, version, customVersionStream, deploymentPipelineProcesses, cancellationToken); - ourTcs.SetResult(); - await eventConsumer.HandleEvent(EventType.EngineInstallComplete, new List { versionString }, deploymentPipelineProcesses, cancellationToken); + + ourTcs.SetResult(); } catch (Exception ex) { From 8bc92176154e357ee0299646390c9a9d13babc63 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 17:21:01 -0500 Subject: [PATCH 04/25] Fix update lockup with cancelled uploaded update packages --- src/Tgstation.Server.Host/Server.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 57a26f669b..640b0c8517 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -208,7 +208,23 @@ namespace Tgstation.Server.Host async Task RunUpdate() { - if (await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken)) + var updateExecutedSuccessfully = false; + try + { + updateExecutedSuccessfully = await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogDebug(ex, "Update cancelled!"); + UpdateInProgress = false; + } + catch (Exception ex) + { + logger.LogError(ex, "Update errored!"); + UpdateInProgress = false; + } + + if (updateExecutedSuccessfully) { logger.LogTrace("Update complete!"); await RestartImpl(newVersion, null, true, true); From f0d8dd5c59761ba2fddb08ac72545bd1e04d6baa Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 17:41:45 -0500 Subject: [PATCH 05/25] Fix issues with issue relocation during releases --- .../Tgstation.Server.ReleaseNotes/Program.cs | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 3cceaf2237..66e6869508 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -315,6 +315,39 @@ namespace Tgstation.Server.ReleaseNotes Description = "Next patch version" }); + + async ValueTask RelocateOpenIssues(Milestone originalMilestone, int moveToMilestoneNumber) + { + if (originalMilestone.OpenIssues + originalMilestone.ClosedIssues > 0) + { + var issuesInUnusedMilestone = await client.Search.SearchIssues(new SearchIssuesRequest + { + Milestone = originalMilestone.Title, + Repos = { { RepoOwner, RepoName } } + }); + + var issueUpdateTasks = new List(); + foreach (var I in issuesInUnusedMilestone.Items) + { + if (I.State.Value != ItemState.Closed) + issueUpdateTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate + { + Milestone = moveToMilestoneNumber + })); + + if (I.PullRequest != null && I.PullRequest.Merged) + { + Console.WriteLine($"Adding additional merged PR #{I.Number}..."); + var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); + noteTasks.Add(task); + allTasks.Add(task); + } + } + + await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); + } + } + if (version.Build == 0) { // close the patch milestone if it exists @@ -326,39 +359,10 @@ namespace Tgstation.Server.ReleaseNotes async ValueTask DeleteMilestone(Milestone milestoneToDelete, int moveToMilestoneNumber) { Console.WriteLine($"Moving {milestoneToDelete.OpenIssues} open issues and {milestoneToDelete.ClosedIssues} closed issues from unused patch milestone {milestoneToDelete.Title} to upcoming ones and deleting..."); - if (milestoneToDelete.OpenIssues + milestoneToDelete.ClosedIssues > 0) - { - var issuesInUnusedMilestone = await client.Search.SearchIssues(new SearchIssuesRequest - { - Milestone = milestoneToDelete.Title, - Repos = { { RepoOwner, RepoName } } - }); - - var issueUpdateTasks = new List(); - foreach (var I in issuesInUnusedMilestone.Items) - { - if (I.State.Value != ItemState.Closed) - issueUpdateTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate - { - Milestone = moveToMilestoneNumber - })); - - if (I.PullRequest != null) - { - Console.WriteLine($"Adding additional merged PR #{I.Number}..."); - var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); - noteTasks.Add(task); - allTasks.Add(task); - } - } - - await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); - } - + await RelocateOpenIssues(milestoneToDelete, moveToMilestoneNumber); allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); } - var unreleasedNextPatchMilestone = milestones.FirstOrDefault(x => x.Title.StartsWith($"v{highestReleaseVersion.Major}.{highestReleaseVersion.Minor}.")); if (unreleasedNextPatchMilestone != null) await DeleteMilestone(unreleasedNextPatchMilestone, nextPatchMilestone.Number); @@ -401,7 +405,11 @@ namespace Tgstation.Server.ReleaseNotes if (unreleasedNextMinorMilestone != null) await DeleteMilestone(unreleasedNextMinorMilestone, nextMinorMilestone.Number); } + else + await RelocateOpenIssues(milestone, nextMinorMilestone.Number); } + else + await RelocateOpenIssues(milestone, nextPatchMilestone.Number); } newNotes.Append(milestone.HtmlUrl); From 11e9fdb7e208282f4240f96be434addde382de76 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 18:06:36 -0500 Subject: [PATCH 06/25] Rename this method appropriately --- src/Tgstation.Server.Host/Components/InstanceManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 12ebb1300e..d0f91c4c46 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -603,7 +603,7 @@ namespace Tgstation.Server.Host.Components logger.LogInformation("{versionString}", assemblyInformationProvider.VersionString); console.SetTitle(assemblyInformationProvider.VersionString); - CheckSystemCompatibility(); + PreflightChecks(); // To let the web server startup immediately before we do any intense work await Task.Yield(); @@ -673,7 +673,7 @@ namespace Tgstation.Server.Host.Components /// /// Check we have a valid system and configuration. /// - void CheckSystemCompatibility() + void PreflightChecks() { logger.LogDebug("Running as user: {username}", Environment.UserName); From 0e55e8d80df719467517f32679488727dbc2810a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 18:14:57 -0500 Subject: [PATCH 07/25] Add support for global event scripts directories --- .../Components/StaticFiles/Configuration.cs | 34 +++++++++++++------ .../Configuration/GeneralConfiguration.cs | 11 ++++++ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f0edb6fc09..6e34b4f596 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -787,27 +787,41 @@ namespace Tgstation.Server.Host.Components.StaticFiles // always execute in serial using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken, logger)) { - var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); - var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); + var directories = generalConfiguration.AdditionalEventScriptsDirectories?.ToList() ?? new List(); + directories.Add(EventScriptsSubdirectory); - var scriptFiles = files - .Select(x => ioManager.GetFileName(x)) - .Where(x => scriptNames.Any( - scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) + var allScripts = new List(); + var tasks = directories.Select( + async scriptDirectory => + { + var files = await ioManager.GetFilesWithExtension(scriptDirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); + var resolvedScriptsDir = ioManager.ResolvePath(scriptDirectory); + + var scriptFiles = files + .Select(x => ioManager.ConcatPath(resolvedScriptsDir, ioManager.GetFileName(x))) + .Where(x => scriptNames.Any( + scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))); + + lock (allScripts) + allScripts.AddRange(scriptFiles); + }) .ToList(); - if (scriptFiles.Count == 0) + await ValueTaskExtensions.WhenAll(tasks); + if (allScripts.Count == 0) { logger.LogTrace("No event scripts starting with \"{scriptName}\" detected", String.Join("\" or \"", scriptNames)); return; } - foreach (var scriptFile in scriptFiles) + var resolvedInstanceScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); + + foreach (var scriptFile in allScripts.OrderBy(ioManager.GetFileName)) { logger.LogTrace("Running event script {scriptFile}...", scriptFile); await using (var script = await processExecutor.LaunchProcess( - ioManager.ConcatPath(resolvedScriptsDir, scriptFile), - resolvedScriptsDir, + scriptFile, + resolvedInstanceScriptsDir, String.Join( ' ', parameters.Select(arg => diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 1790316e37..5ea16ae64e 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -150,6 +153,11 @@ namespace Tgstation.Server.Host.Configuration /// public bool OpenDreamSuppressInstallOutput { get; set; } + /// + /// List of directories that have their contents merged with instance EventScripts directories when executing scripts. + /// + public List? AdditionalEventScriptsDirectories { get; set; } + /// /// Initializes a new instance of the class. /// @@ -190,6 +198,9 @@ namespace Tgstation.Server.Host.Configuration if (ByondTopicTimeout <= 1000) logger.LogWarning("The timeout for sending BYOND topics is very low ({ms}ms). Topic calls may fail to complete at all!", ByondTopicTimeout); + + if (AdditionalEventScriptsDirectories?.Any(path => !Path.IsPathRooted(path)) == true) + logger.LogWarning($"Config option \"{nameof(AdditionalEventScriptsDirectories)}\" contains non-rooted paths. These will be evaluated relative to each instances \"Configuration\" directory!"); } } } From 2e93ddb0dbd87c53b21ff70278d7c27d22e0e366 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 7 Nov 2024 18:24:22 -0500 Subject: [PATCH 08/25] Rebump version --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 41b4c978dd..02bbc0e30a 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 6.11.4 + 6.12.0 5.4.0 10.12.0 0.5.0 From 7f06abcd8e26819880180966c141b0900c00968c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 8 Nov 2024 17:49:05 -0500 Subject: [PATCH 09/25] Fix global event scripts --- .../Components/StaticFiles/Configuration.cs | 13 ++++++++++--- src/Tgstation.Server.Host/appsettings.yml | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 6e34b4f596..e1b37b50b3 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -794,13 +794,20 @@ namespace Tgstation.Server.Host.Components.StaticFiles var tasks = directories.Select( async scriptDirectory => { - var files = await ioManager.GetFilesWithExtension(scriptDirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); var resolvedScriptsDir = ioManager.ResolvePath(scriptDirectory); + logger.LogTrace("Checking for scripts in {directory}...", scriptDirectory); + var files = await ioManager.GetFilesWithExtension(scriptDirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); var scriptFiles = files - .Select(x => ioManager.ConcatPath(resolvedScriptsDir, ioManager.GetFileName(x))) + .Select(ioManager.GetFileName) .Where(x => scriptNames.Any( - scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))); + scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) + .Select(x => + { + var fullScriptPath = ioManager.ConcatPath(resolvedScriptsDir, x); + logger.LogTrace("Found matching script: {scriptPath}", fullScriptPath); + return fullScriptPath; + }); lock (allScripts) allScripts.AddRange(scriptFiles); diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index ac08b78c49..ef7e42f69f 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -19,6 +19,7 @@ General: OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. + AdditionalEventScriptsDirectories: # An array of directories that are considered to contain EventScripts alongside instance directories. Working directory will remain the instance EventScripts directory. Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes From 159677a2dd1c1e5444169358ef9235b69901ed15 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 8 Nov 2024 17:52:37 -0500 Subject: [PATCH 10/25] Do not build commits tagged with `[TGSRelease]` --- .github/workflows/ci-pipeline.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5717627962..5c5fa5e7fb 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -47,8 +47,17 @@ concurrency: cancel-in-progress: true jobs: + start-gate: + name: CI Start Gate + runs-on: ubuntu-latest + if: (!contains(github.event.head_commit.message, '[TGSRelease]')) + steps: + - name: GitHub Requires at Least One Step for a Job + run: exit 0 + build-releasenotes: name: Build ReleaseNotes for Other Jobs + needs: start-gate runs-on: ubuntu-latest steps: - name: Install Native Dependencies @@ -83,6 +92,7 @@ jobs: code-scanning: name: Run CodeQL + needs: start-gate runs-on: ubuntu-latest permissions: security-events: write @@ -134,6 +144,7 @@ jobs: dmapi-build: name: Build DMAPI + needs: start-gate strategy: fail-fast: false matrix: @@ -222,6 +233,7 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) + needs: start-gate strategy: fail-fast: false matrix: @@ -278,6 +290,7 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools runs-on: ubuntu-latest + needs: start-gate steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -412,6 +425,7 @@ jobs: docker-build: name: Build Docker Image runs-on: ubuntu-latest + needs: start-gate env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt steps: @@ -438,6 +452,7 @@ jobs: linux-unit-tests: name: Linux Tests + needs: start-gate strategy: fail-fast: false matrix: @@ -510,6 +525,7 @@ jobs: windows-unit-tests: name: Windows Tests + needs: start-gate strategy: fail-fast: false matrix: From f90d06cee9fb9f8a55f7c9b822ba6a1b60c7f4a8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 8 Nov 2024 18:11:55 -0500 Subject: [PATCH 11/25] Setup release attestations --- .github/workflows/ci-pipeline.yml | 74 ++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5c5fa5e7fb..16469163ce 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1731,6 +1731,12 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./swagger/tgs_api.json + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload OpenApi Spec uses: actions/upload-release-asset@v1 env: @@ -1811,6 +1817,12 @@ jobs: commitish: ${{ github.event.head_commit.id }} prerelease: ${{ env.TGS_GRAPHQL_PRERELEASE }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./schema/tgs_api.graphql + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload GraphQL Schema uses: actions/upload-release-asset@v1 env: @@ -1883,6 +1895,12 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./DMAPI.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload DMAPI Artifact uses: actions/upload-release-asset@v1 env: @@ -2148,6 +2166,12 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} + - name: Generate Server Console Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerConsole.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Server Console Artifact uses: actions/upload-release-asset@v1 env: @@ -2158,6 +2182,12 @@ jobs: asset_name: ServerConsole.zip asset_content_type: application/zip + - name: Generate Server Service Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerService.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Server Service Artifact uses: actions/upload-release-asset@v1 env: @@ -2168,6 +2198,12 @@ jobs: asset_name: ServerService.zip asset_content_type: application/zip + - name: Generate DMAPI Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./DMAPI.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload DMAPI Artifact uses: actions/upload-release-asset@v1 env: @@ -2178,6 +2214,12 @@ jobs: asset_name: DMAPI.zip asset_content_type: application/zip + - name: Generate REST API Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./swagger/tgs_api.json + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload REST API Artifact uses: actions/upload-release-asset@v1 env: @@ -2188,6 +2230,12 @@ jobs: asset_name: swagger.json asset_content_type: application/json + - name: Generate GraphQL API Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./schema/tgs-api.graphql + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload GraphQL API Artifact uses: actions/upload-release-asset@v1 env: @@ -2198,6 +2246,12 @@ jobs: asset_name: tgs-api.graphql asset_content_type: text/plain + - name: Generate Server Update Package Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerUpdatePackage.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Server Update Package Artifact uses: actions/upload-release-asset@v1 env: @@ -2208,7 +2262,13 @@ jobs: asset_name: ServerUpdatePackage.zip asset_content_type: application/zip - - name: Upload Debian Pacakaging Artifact + - name: Generate Debian Packaging Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./packaging-debian/tgstation-server-v${{ env.TGS_VERSION }}.debian.packaging.tar.xz + github-token: ${{ steps.app-token-generation.outputs.token }} + + - name: Upload Debian Packaging Artifact uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ steps.app-token-generation.outputs.token }} @@ -2218,6 +2278,12 @@ jobs: asset_name: tgstation-server-v${{ env.TGS_VERSION }}.debian.packaging.tar.xz asset_content_type: application/x-tar + - name: Generate MariaDB .msi Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/bin/Release/mariadb.msi + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload MariaDB .msi uses: actions/upload-release-asset@v1 env: @@ -2228,6 +2294,12 @@ jobs: asset_name: mariadb-${{ env.MARIADB_VERSION }}-winx64.msi asset_content_type: application/octet-stream + - name: Generate Installer .exe Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./build/package/winget/tgstation-server-installer.exe + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Installer .exe uses: actions/upload-release-asset@v1 env: From d40fcbf471781ca3099469bfc85085b1380cdc8f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 8 Nov 2024 20:51:15 -0500 Subject: [PATCH 12/25] Documentation comment cleanups --- src/Tgstation.Server.Host.Common/DotnetHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host.Common/DotnetHelper.cs b/src/Tgstation.Server.Host.Common/DotnetHelper.cs index 5945eac25d..c4edb2d3bf 100644 --- a/src/Tgstation.Server.Host.Common/DotnetHelper.cs +++ b/src/Tgstation.Server.Host.Common/DotnetHelper.cs @@ -14,10 +14,10 @@ namespace Tgstation.Server.Host.Common public static class DotnetHelper { /// - /// Gets the path to the dotnet executable. + /// Gets potential paths to the dotnet executable. /// /// If the current system is a Windows OS. - /// The path to the dotnet executable. + /// An of potential paths to the dotnet executable. public static IEnumerable GetPotentialDotnetPaths(bool isWindows) { var enviromentPath = Environment.GetEnvironmentVariable("PATH"); From e6d64d84496dbc0cc9ef5402935bbd9edc5d76f7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 8 Nov 2024 20:51:43 -0500 Subject: [PATCH 13/25] Fix gcore on nix --- .../System/PosixProcessFeatures.cs | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index f9894e9ac9..12a0164a1f 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -62,6 +64,39 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } + /// + /// Gets potential paths to the gcore executable. + /// + /// The potential paths to the gcore executable. + static IEnumerable GetPotentialGCorePaths() + { + var enviromentPath = Environment.GetEnvironmentVariable("PATH"); + IEnumerable enumerator; + if (enviromentPath == null) + enumerator = Enumerable.Empty(); + else + { + var paths = enviromentPath.Split(';'); + enumerator = paths + .Select(x => x.Split(':')) + .SelectMany(x => x); + } + + var exeName = "gcore"; + + enumerator = enumerator + .Concat(new List(2) + { + "/usr/bin", + "/usr/share/bin", + "/bin", + }); + + enumerator = enumerator.Select(x => Path.Combine(x, exeName)); + + return enumerator; + } + /// public void ResumeProcess(global::System.Diagnostics.Process process) { @@ -88,8 +123,15 @@ namespace Tgstation.Server.Host.System ArgumentNullException.ThrowIfNull(process); ArgumentNullException.ThrowIfNull(outputFile); - const string GCorePath = "/usr/bin/gcore"; - if (!await ioManager.FileExists(GCorePath, cancellationToken)) + string? gcorePath = null; + foreach (var path in GetPotentialGCorePaths()) + if (await ioManager.FileExists(path, cancellationToken)) + { + gcorePath = path; + break; + } + + if (gcorePath == null) throw new JobException(ErrorCode.MissingGCore); int pid; @@ -108,7 +150,7 @@ namespace Tgstation.Server.Host.System string? output; int exitCode; await using (var gcoreProc = await lazyLoadedProcessExecutor.Value.LaunchProcess( - GCorePath, + gcorePath, Environment.CurrentDirectory, $"{(!minidump ? "-a " : String.Empty)}-o {outputFile} {process.Id}", cancellationToken, From 05f48d60e34b93f96d72ef4e4d13a183de8f0a87 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 08:50:40 -0500 Subject: [PATCH 14/25] Create basic nix flake --- build/package/nix/ServerConsole.sha256 | 1 + build/package/nix/flake.nix | 13 +++ build/package/nix/package.nix | 121 ++++++++++++++++++++++++ build/package/nix/tgstation-server.nix | 124 +++++++++++++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 build/package/nix/ServerConsole.sha256 create mode 100644 build/package/nix/flake.nix create mode 100644 build/package/nix/package.nix create mode 100644 build/package/nix/tgstation-server.nix diff --git a/build/package/nix/ServerConsole.sha256 b/build/package/nix/ServerConsole.sha256 new file mode 100644 index 0000000000..ae3d91ae7d --- /dev/null +++ b/build/package/nix/ServerConsole.sha256 @@ -0,0 +1 @@ +sha256-mHlRHPSeZxyJPqN3KUmc0ftYNZgh81LauIu+fCSKPUI= diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix new file mode 100644 index 0000000000..f2bf8c14a9 --- /dev/null +++ b/build/package/nix/flake.nix @@ -0,0 +1,13 @@ +{ + description = "tgstation-server"; + + inputs = {}; + + outputs = { ... }: { + nixosModules = { + default = { ... }: { + imports = [ ./tgstation-server.nix ]; + }; + }; + }; +} diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix new file mode 100644 index 0000000000..1dae347413 --- /dev/null +++ b/build/package/nix/package.nix @@ -0,0 +1,121 @@ +{ + pkgs, + ... +}: + +let + inherit (pkgs) stdenv lib; + + versionParse = stdenv.mkDerivation { + pname = "tgstation-server-version-parse"; + version = "1.0.0"; + + meta = with pkgs.lib; { + description = "Version parser for tgstation-server"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/blob/gh-pages/changelog.yml"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + nativeBuildInputs = with pkgs; [ + xmlstarlet + ]; + + src = ./../..; + + installPhase = '' + mkdir -p $out + xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion ./Version.props > $out/tgs_version.txt + ''; + }; + + fixedOutput = stdenv.mkDerivation { + pname = "tgstation-server-release-server-console-zip"; + version = (builtins.readFile "${versionParse}/tgs_version.txt"); + + meta = with pkgs.lib; { + description = "Host watchdog binaries for tgstation-server"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v${version}"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + nativeBuildInputs = with pkgs; [ + curl + cacert + versionParse + ]; + + src = ./.; + + buildPhase = '' + curl -L https://file.house/b2eyKOJZ7ptxwqm8O24-9A==.zip -o ServerConsole.zip + ''; + + installPhase = '' + mkdir -p $out + mv ServerConsole.zip $out/ServerConsole.zip + ''; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = (builtins.readFile ./ServerConsole.sha256); + }; + rpath = lib.makeLibraryPath [ pkgs.stdenv_32bit.cc.cc.lib ]; +in +stdenv.mkDerivation { + pname = "tgstation-server"; + version = (builtins.readFile "${versionParse}/tgs_version.txt"); + + meta = with pkgs.lib; { + description = "A production scale tool for DreamMaker server management"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v${version}"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + buildInputs = with pkgs; [ + dotnetCorePackages.aspnetcore_8_0 + gdb + systemd + zlib + gcc_multi + glibc + patchelf + ]; + nativeBuildInputs = with pkgs; [ + makeWrapper + unzip + fixedOutput + versionParse + ]; + + src = ./.; + + installPhase = '' + mkdir -p $out/bin + unzip "${fixedOutput}/ServerConsole.zip" -d $out/bin + rm -rf $out/bin/lib + makeWrapper ${pkgs.dotnetCorePackages.aspnetcore_8_0}/dotnet $out/bin/tgstation-server --suffix PATH : ${ + lib.makeBinPath ( + with pkgs; + [ + patchelf + dotnetCorePackages.aspnetcore_8_0 + gdb + ] + ) + } --suffix LD_LIBRARY_PATH : ${ + lib.makeLibraryPath ( + with pkgs; + [ + systemd + zlib + ] + ) + } --add-flags "$out/bin/Tgstation.Server.Host.Console.dll --bootstrap" + ''; +} diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix new file mode 100644 index 0000000000..83076b44a2 --- /dev/null +++ b/build/package/nix/tgstation-server.nix @@ -0,0 +1,124 @@ +inputs@{ + config, + lib, + nixpkgs, + pkgs, + ... +}: + +let + pkgs-i686 = nixpkgs.legacyPackages.i686-linux; + + cfg = config.services.tgstation-server; + + package = import ./package.nix inputs; + + stdenv = pkgs-i686.stdenv_32bit; + + rpath = pkgs-i686.lib.makeLibraryPath [ + stdenv.cc.cc.lib + ]; + + byond-patcher = pkgs-i686.writeShellScriptBin "byond-patcher" '' + BYOND_PATH=$(realpath ../../Byond/$1/byond/bin/) + + patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ + --set-rpath "$BYOND_PATH:${rpath}" \ + $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} + ''; +in +{ + ##### interface. here we define the options that users of our service can specify + options = { + # the options for our service will be located under services.foo + services.tgstation-server = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable tgstation-server. + ''; + }; + + username = lib.mkOption { + type = lib.types.str; + default = "tgstation-server"; + description = '' + The name of the user used to execute tgstation-server. + ''; + }; + + groupname = lib.mkOption { + type = lib.types.str; + default = "tgstation-server"; + description = '' + The name of group the user used to execute tgstation-server will belong to. + ''; + }; + + home-directory = lib.mkOption { + type = lib.types.str; + default = "/home/tgstation-server"; + description = '' + The home directory of TGS. Should be persistent. + ''; + }; + + production-appsettings = lib.mkOption { + type = lib.types.lines; + default = ''''; + description = '' + The contents of appsettings.Production.yml in the /etc/tgstation-server.d directory. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + users.groups."${cfg.groupname}" = { }; + + users.users."${cfg.username}" = { + isSystemUser = true; + createHome = true; + group = cfg.groupname; + home = cfg.home-directory; + }; + + environment.etc = { + "tgstation-server.d/appsettings.yml" = { + text = (builtins.readFile "${package}/bin/appsettings.yml"); + group = cfg.groupname; + mode = "0644"; + }; + "tgstation-server.d/appsettings.Production.yml" = { + text = cfg.production-appsettings; + group = cfg.groupname; + mode = "0640"; + }; + "tgstation-server.d/EventScripts/EngineInstallComplete-050-PatchELFByond.sh" = { + source = "${byond-patcher}/bin/byond-patcher"; + group = cfg.groupname; + mode = "755"; + }; + }; + + systemd.services.tgstation-server = { + description = "tgstation-server"; + serviceConfig = { + User = cfg.username; + Type = "notify-reload"; + NotifyAccess = "all"; + WorkingDirectory = "${package}/bin"; + ExecStart = "${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts"; + Restart = "always"; + KillMode = "process"; + ReloadSignal = "SIGUSR2"; + AmbientCapabilities = "CAP_SYS_NICE CAP_SYS_PTRACE"; + WatchdogSec = "60"; + WatchdogSignal = "SIGTERM"; + LogsDirectory = "tgstation-server"; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; +} From 37fd094f719b7a001b9c9e589868da577da7737c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 10:09:18 -0500 Subject: [PATCH 15/25] Move byond patcher out of etc --- build/package/nix/tgstation-server.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index 83076b44a2..150a20fe94 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -19,7 +19,7 @@ let stdenv.cc.cc.lib ]; - byond-patcher = pkgs-i686.writeShellScriptBin "byond-patcher" '' + byond-patcher = pkgs-i686.writeShellScriptBin "EngineInstallComplete-050-TgsPatchELFByond.sh" '' BYOND_PATH=$(realpath ../../Byond/$1/byond/bin/) patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ @@ -95,11 +95,6 @@ in group = cfg.groupname; mode = "0640"; }; - "tgstation-server.d/EventScripts/EngineInstallComplete-050-PatchELFByond.sh" = { - source = "${byond-patcher}/bin/byond-patcher"; - group = cfg.groupname; - mode = "755"; - }; }; systemd.services.tgstation-server = { @@ -109,7 +104,7 @@ in Type = "notify-reload"; NotifyAccess = "all"; WorkingDirectory = "${package}/bin"; - ExecStart = "${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts"; + ExecStart = "${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts --General:AdditionalEventScriptsDirectories:1=${byond-patcher}/bin"; Restart = "always"; KillMode = "process"; ReloadSignal = "SIGUSR2"; From af6c5f3f823fae654208af8a160d0e24871ae323 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 10:36:22 -0500 Subject: [PATCH 16/25] Attempt at adding extra PATH specifications --- build/package/nix/tgstation-server.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index 150a20fe94..16714f012d 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -3,6 +3,7 @@ inputs@{ lib, nixpkgs, pkgs, + writeShellScriptBin, ... }: @@ -26,6 +27,11 @@ let --set-rpath "$BYOND_PATH:${rpath}" \ $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} ''; + + tgs-wrapper = pkgs.writeShellScriptBin "tgs-path-wrapper" '' + export PATH=$PATH:${cfg.extra-path} + exec ${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts --General:AdditionalEventScriptsDirectories:1=${byond-patcher}/bin + ''; in { ##### interface. here we define the options that users of our service can specify @@ -71,6 +77,14 @@ in The contents of appsettings.Production.yml in the /etc/tgstation-server.d directory. ''; }; + + extra-path = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Extra PATH entries to add to the TGS process + ''; + }; }; }; @@ -104,7 +118,7 @@ in Type = "notify-reload"; NotifyAccess = "all"; WorkingDirectory = "${package}/bin"; - ExecStart = "${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts --General:AdditionalEventScriptsDirectories:1=${byond-patcher}/bin"; + ExecStart = "${tgs-wrapper}/bin/tgs-path-wrapper"; Restart = "always"; KillMode = "process"; ReloadSignal = "SIGUSR2"; From 538e9cffeeb5149c72d1679debb08758fd024eba Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 11:17:11 -0500 Subject: [PATCH 17/25] Remove unnecessary patchelf --- build/package/nix/package.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 1dae347413..75c885c619 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -84,7 +84,6 @@ stdenv.mkDerivation { zlib gcc_multi glibc - patchelf ]; nativeBuildInputs = with pkgs; [ makeWrapper @@ -103,7 +102,6 @@ stdenv.mkDerivation { lib.makeBinPath ( with pkgs; [ - patchelf dotnetCorePackages.aspnetcore_8_0 gdb ] From bf5bfe65976cb847436cdd67d12a651438af920e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 11:17:19 -0500 Subject: [PATCH 18/25] Attempt to patch all .so's --- build/package/nix/tgstation-server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index 16714f012d..ee886e51e2 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -25,7 +25,7 @@ let patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ --set-rpath "$BYOND_PATH:${rpath}" \ - $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} + $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker,*.so} ''; tgs-wrapper = pkgs.writeShellScriptBin "tgs-path-wrapper" '' From bf30471dba555bc71c231e036ebc12113c99e86e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 11:19:07 -0500 Subject: [PATCH 19/25] Fix patchelf call --- build/package/nix/tgstation-server.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index ee886e51e2..6e2cce9c60 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -23,7 +23,7 @@ let byond-patcher = pkgs-i686.writeShellScriptBin "EngineInstallComplete-050-TgsPatchELFByond.sh" '' BYOND_PATH=$(realpath ../../Byond/$1/byond/bin/) - patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ + ${pkgs.patchelf}/bin/patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ --set-rpath "$BYOND_PATH:${rpath}" \ $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker,*.so} ''; From aa62320233392b6ef96445dfea1a5e729a568ead Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 11:21:23 -0500 Subject: [PATCH 20/25] Just patch dynamic libs specifically --- build/package/nix/tgstation-server.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index 6e2cce9c60..46412db74a 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -25,7 +25,9 @@ let ${pkgs.patchelf}/bin/patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ --set-rpath "$BYOND_PATH:${rpath}" \ - $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker,*.so} + $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} + + ${pkgs.patchelf}/bin/patchelf --set-rpath "$BYOND_PATH:${rpath}" $BYOND_PATH/*.so ''; tgs-wrapper = pkgs.writeShellScriptBin "tgs-path-wrapper" '' From c37e983cf82c7767fa8753c2ef3212961ff389ff Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 11:34:53 -0500 Subject: [PATCH 21/25] Test --- build/package/nix/tgstation-server.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix index 46412db74a..eb60c7c94c 100644 --- a/build/package/nix/tgstation-server.nix +++ b/build/package/nix/tgstation-server.nix @@ -26,8 +26,6 @@ let ${pkgs.patchelf}/bin/patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ --set-rpath "$BYOND_PATH:${rpath}" \ $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} - - ${pkgs.patchelf}/bin/patchelf --set-rpath "$BYOND_PATH:${rpath}" $BYOND_PATH/*.so ''; tgs-wrapper = pkgs.writeShellScriptBin "tgs-path-wrapper" '' From 57bec44ab66ccaa29282849b4ca54c3a380760e5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 15:09:14 -0500 Subject: [PATCH 22/25] Switch to proper versioned ServerConsole.zip and setup CD --- .github/workflows/ci-pipeline.yml | 60 ++++++++++++++++++++++++++++++- build/package/nix/package.nix | 2 +- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 16469163ce..339c0c3762 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -2172,7 +2172,13 @@ jobs: subject-path: ./ServerConsole.zip github-token: ${{ steps.app-token-generation.outputs.token }} - - name: Upload Server Console Artifact + - name: Upload Server Console Zip Artifact to Action + uses: actions/upload-artifact@v4 + with: + name: server-console-release + path: ./ServerConsole.zip + + - name: Upload Server Console Artifact to Release uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ steps.app-token-generation.outputs.token }} @@ -2310,6 +2316,58 @@ jobs: asset_name: tgstation-server-installer.exe asset_content_type: application/octet-stream + update-nix: + name: Update Nix SHA + needs: deploy-tgs + runs-on: ubuntu-latest + if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success' + steps: + - name: Install Native Packages # Name checked in rerunFlakyTests.js + run: | + sudo apt-get update + sudo apt-get install -y xmlstarlet + + - name: Setup Nix + uses: cachix/install-nix-action@v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Checkout + uses: actions/checkout@v4 + + - name: Parse TGS version + run: echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + + - name: Retrieve ServerConsole.zip Artifact + uses: actions/download-artifact@v4 + with: + name: server-console-release + path: server-console-release + + - name: Regenerate Nix Hash + run: | + nix hash path ./server-console-release > build/package/nix/ServerConsole.sha256 + cat build/package/nix/ServerConsole.sha256 + + - name: Commit + run: | + git config --global push.default simple + git config user.name "tgstation-server-ci[bot]" + git config user.email "161980869+tgstation-server-ci[bot]@users.noreply.github.com" + git add build/package/nix/ServerConsole.sha256 + git commit -m "Update nix SHA256 for [TGSRelease] v${{ env.TGS_VERSION }}" + + - name: Re-tag + run: | + git tag -d tgstation-server-v${{ env.TGS_VERSION }} + git tag tgstation-server-v${{ env.TGS_VERSION }} + + - name: Push Commit + run: git push + + - name: Force Push Tags + run: git push -f --tags + changelog-regen: name: Regenerate Changelog runs-on: ubuntu-latest diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 75c885c619..191e9f68ed 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -51,7 +51,7 @@ let src = ./.; buildPhase = '' - curl -L https://file.house/b2eyKOJZ7ptxwqm8O24-9A==.zip -o ServerConsole.zip + curl -L https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v${version}/ServerConsole.zip -o ServerConsole.zip ''; installPhase = '' From 4fe6b5ab4a0d3f76b6fb94b430e270048523d8bb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 15:20:11 -0500 Subject: [PATCH 23/25] Document new features --- README.md | 27 +++++++++++++++++++++++ src/Tgstation.Server.Host/appsettings.yml | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 24077a5a10..680f2f2a6b 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,31 @@ sudo dpkg --add-architecture i386 \ The service will execute as the newly created user: `tgstation-server`. You should, ideally, store your instances somewhere under `/home/tgstation-server`. +##### Nix Flake + +TGS supports being setup on Nix starting with version 6.12.0. Add the [flake](./build/package/nix/flake.nix) to your own system by adding the following code to your flake inputs. +```nix + tgstation-server = { + url = "github:tgstation/tgstation-server/tgstation-server-v${version}?dir=build/package/nix"; + }; +``` + +Where `version` is the latest major TGS version you wish to use. + +Note that changing this version does not change the core version of TGS used after the first launch. Instead, have TGS self-update via its API. + +For maximum game server uptime, do NOT modify this version unless you are doing a major TGS version update in which case it is a requirement. + +Configure TGS by setting up its service definition: +```nix + services.tgstation-server = { + enable = true; + production-appsettings = (builtins.readFile ./path/to/your/appsettings.Production.yml); + }; +``` + +Refer to [tgstation-server.nix](./build/package/nix/tgstation-server.nix) for the full list of available configuration options. + ##### Manual Setup The following dependencies are required. @@ -243,6 +268,8 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:SkipAddingByondFirewallException`: Set to `true` if you have Windows firewall disabled +- `General:AdditionalEventScriptsDirectories`: An array of directories that are considered to contain EventScripts alongside instance directories. Working directory for executed scripts will remain the instance EventScripts directory. + - `Session:HighPriorityLiveDreamDaemon`: Boolean controlling if live DreamDaemon instances get set to above normal priority processes. - `Session:LowPriorityDeploymentProcesses `: Boolean controlling if DreamMaker and API validation DreamDaemon instances get set to below normal priority processes. diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index ef7e42f69f..2978befbf9 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -19,7 +19,7 @@ General: OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. - AdditionalEventScriptsDirectories: # An array of directories that are considered to contain EventScripts alongside instance directories. Working directory will remain the instance EventScripts directory. + AdditionalEventScriptsDirectories: # An array of directories that are considered to contain EventScripts alongside instance directories. Working directory for exectued scripts will remain the instance EventScripts directory. Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes From 906d75e8ad0be04c69d06da9ba25ab2077d09d02 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 15:29:55 -0500 Subject: [PATCH 24/25] Fix bad engine installs not getting cleaned up --- .../Components/Engine/EngineManager.cs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index 9b1d5b61f6..113a0d249b 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -456,6 +456,7 @@ namespace Tgstation.Server.Host.Components.Engine } // okay up to us to install it then + string? installPath = null; try { if (customVersionStream != null) @@ -475,15 +476,26 @@ namespace Tgstation.Server.Host.Components.Engine var versionString = version.ToString(); await eventConsumer.HandleEvent(EventType.EngineInstallStart, new List { versionString }, deploymentPipelineProcesses, cancellationToken); - await InstallVersionFiles(progressReporter, version, customVersionStream, deploymentPipelineProcesses, cancellationToken); - + installPath = await InstallVersionFiles(progressReporter, version, customVersionStream, deploymentPipelineProcesses, cancellationToken); await eventConsumer.HandleEvent(EventType.EngineInstallComplete, new List { versionString }, deploymentPipelineProcesses, cancellationToken); ourTcs.SetResult(); } catch (Exception ex) { - if (ex is not OperationCanceledException) + if (installPath != null) + { + try + { + logger.LogDebug("Cleaning up failed installation at {path}...", installPath); + await ioManager.DeleteDirectory(installPath, cancellationToken); + } + catch (Exception ex2) + { + logger.LogError(ex2, "Error cleaning up failed installation!"); + } + } + else if (ex is not OperationCanceledException) await eventConsumer.HandleEvent(EventType.EngineInstallFail, new List { ex.Message }, deploymentPipelineProcesses, cancellationToken); lock (installedVersions) @@ -510,8 +522,8 @@ namespace Tgstation.Server.Host.Components.Engine /// Custom zip file to use. Will cause a number to be added. /// If processes should be launched as part of the deployment pipeline. /// The for the operation. - /// A representing the running operation. - async ValueTask InstallVersionFiles( + /// A resulting in the directory the engine was installed to. + async ValueTask InstallVersionFiles( JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, @@ -597,6 +609,8 @@ namespace Tgstation.Server.Host.Components.Engine await ioManager.DeleteDirectory(installFullPath, cancellationToken); throw; } + + return installFullPath; } /// From d1ee2e0e845951ee166a37cb1bd1a1c2a7784bda Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 9 Nov 2024 15:33:57 -0500 Subject: [PATCH 25/25] Fix `build-msi` job bypassing CI start gate --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 339c0c3762..b61a1e318e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1444,6 +1444,7 @@ jobs: build-msi: name: Build Windows Installer .exe runs-on: windows-latest + needs: start-gate env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: