Merge pull request #1485 from tgstation/MissingEvents [DMDeploy]

Add missing events to DMAPI

.github/workflows/ci-suite.yml

@@ -103,14 +103,14 @@ jobs:
     if: "!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success'"
     env:
       BYOND_MAJOR: 515
-      BYOND_MINOR: 1592
+      BYOND_MINOR: 1606
     runs-on: ubuntu-latest
     steps:
     - name: Install x86 libc Dependencies
       run: |
         sudo dpkg --add-architecture i386
         sudo apt-get update
-        sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386
+        sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386
 
     - name: Install BYOND
       if: steps.cache-byond.outputs.cache-hit != 'true'
@@ -315,7 +315,7 @@ jobs:
       matrix:
         watchdog-type: [ 'Basic', 'System' ]
         configuration: [ 'Debug', 'Release' ]
-    runs-on: windows-latest
+    runs-on: windows-2019
     steps:
     - name: Setup dotnet
       uses: actions/setup-dotnet@v2
@@ -436,7 +436,7 @@ jobs:
       run: |
         sudo dpkg --add-architecture i386
         sudo apt-get update
-        sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb
+        sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386
 
     - name: Install Node 12.X
       uses: actions/setup-node@v3

README.md

@@ -43,7 +43,6 @@ The following dependencies are required to run tgstation-server on Linux alongsi
 
 - libc6-i386
 - libstdc++6:i386
-- libssl1.0.0
 - gdb (for using gcore to create core dumps)
 - gcc-multilib (Only on 64-bit systems)
 

build/Dockerfile

@@ -65,13 +65,6 @@ RUN apt-get update \
 	&& apt-get install -y \
 		gcc-multilib \
 		gdb \
-		curl \
-	&& curl http://archive.ubuntu.com/ubuntu/pool/main/g/glibc/multiarch-support_2.27-3ubuntu1_amd64.deb --output multiarch-support_2.27.deb \
-	&& curl http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.0.0_1.0.1t-1+deb8u12_amd64.deb --output libssl1.0.0.deb \
-	&& dpkg -i multiarch-support_2.27.deb \
-	&& dpkg -i libssl1.0.0.deb \
-	&& rm multiarch-support_2.27.deb \
-	&& rm libssl1.0.0.deb \
 	&& rm -rf /var/lib/apt/lists/*
 
 EXPOSE 5000

build/Version.props

@@ -8,7 +8,7 @@
     <TgsApiVersion>9.10.2</TgsApiVersion>
     <TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>
     <TgsClientVersion>11.4.2</TgsClientVersion>
-    <TgsDmapiVersion>6.4.2</TgsDmapiVersion>
+    <TgsDmapiVersion>6.4.3</TgsDmapiVersion>
     <TgsInteropVersion>5.6.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.2.2</TgsHostWatchdogVersion>
     <TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>

src/DMAPI/tgs.dm

@@ -1,6 +1,6 @@
 // tgstation-server DMAPI
 
-#define TGS_DMAPI_VERSION "6.4.2"
+#define TGS_DMAPI_VERSION "6.4.3"
 
 // All functions and datums outside this document are subject to change with any version and should not be relied on.
 
@@ -104,6 +104,12 @@
 #define TGS_EVENT_WORLD_PRIME 21
 // DMAPI also doesnt implement this
 // #define TGS_EVENT_DREAM_DAEMON_LAUNCH 22
+/// After a single submodule update is performed. Parameters: Updated submodule name
+#define TGS_EVENT_REPO_SUBMODULE_UPDATE 23
+/// After CodeModifications are applied, before DreamMaker is run. Parameters: Game directory path, origin commit sha, byond version
+#define TGS_EVENT_PRE_DREAM_MAKER 24
+/// Whenever a deployment folder is deleted from disk. Parameters: Game directory path
+#define TGS_EVENT_DEPLOYMENT_CLEANUP 25
 
 // OTHER ENUMS
 

src/DMAPI/tgs/v3210/api.dm

@@ -179,7 +179,7 @@
 /datum/tgs_api/v3210/Revision()
 	if(!warned_revison)
 		var/datum/tgs_version/api_version = ApiVersion()
-		TGS_ERROR_LOG("Use of TgsRevision on [api_version.deprefixed_parameter] origin_commit only points to master!")
+		TGS_WARNING_LOG("Use of TgsRevision on [api_version.deprefixed_parameter] origin_commit only points to master!")
 		warned_revison = TRUE
 	var/datum/tgs_revision_information/ri = new
 	ri.commit = commit

src/Tgstation.Server.Host/Components/Byond/ByondInstallerBase.cs

@@ -97,7 +97,7 @@ namespace Tgstation.Server.Host.Components.Byond
 
 			Logger.LogTrace("Downloading BYOND version {major}.{minor}...", version.Major, version.Minor);
 			var url = String.Format(CultureInfo.InvariantCulture, ByondRevisionsUrlTemplate, version.Major, version.Minor);
-			return fileDownloader.DownloadFile(new Uri(url), cancellationToken);
+			return fileDownloader.DownloadFile(new Uri(url), null, cancellationToken);
 		}
 	}
 }

src/Tgstation.Server.Host/Core/Application.cs

@@ -435,7 +435,10 @@ namespace Tgstation.Server.Host.Core
 				});
 
 			// setup the HTTP request pipeline
-			// Final point where we wrap exceptions in a 500 (ErrorMessage) response
+			// Add additional logging context to the request
+			applicationBuilder.UseAdditionalRequestLoggingContext(swarmConfiguration);
+
+			// Wrap exceptions in a 500 (ErrorMessage) response
 			applicationBuilder.UseServerErrorHandling();
 
 			// Add the X-Powered-By response header

src/Tgstation.Server.Host/Core/ServerUpdater.cs

@@ -43,6 +43,11 @@ namespace Tgstation.Server.Host.Core
 		/// </summary>
 		readonly ILogger<ServerUpdater> logger;
 
+		/// <summary>
+		/// The <see cref="GeneralConfiguration"/> for the <see cref="ServerUpdater"/>.
+		/// </summary>
+		readonly GeneralConfiguration generalConfiguration;
+
 		/// <summary>
 		/// The <see cref="UpdatesConfiguration"/> for the <see cref="ServerUpdater"/>.
 		/// </summary>
@@ -61,6 +66,7 @@ namespace Tgstation.Server.Host.Core
 		/// <param name="fileDownloader">The value of <see cref="fileDownloader"/>.</param>
 		/// <param name="serverControl">The value of <see cref="serverControl"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		/// <param name="updatesConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="updatesConfiguration"/>.</param>
 		public ServerUpdater(
 			IGitHubClientFactory gitHubClientFactory,
@@ -68,6 +74,7 @@ namespace Tgstation.Server.Host.Core
 			IFileDownloader fileDownloader,
 			IServerControl serverControl,
 			ILogger<ServerUpdater> logger,
+			IOptions<GeneralConfiguration> generalConfigurationOptions,
 			IOptions<UpdatesConfiguration> updatesConfigurationOptions)
 		{
 			this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory));
@@ -75,6 +82,7 @@ namespace Tgstation.Server.Host.Core
 			this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader));
 			this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 			updatesConfiguration = updatesConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(updatesConfigurationOptions));
 		}
 
@@ -110,31 +118,33 @@ namespace Tgstation.Server.Host.Core
 				if (Version.TryParse(
 					release.TagName.Replace(
 						updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal),
-					out var version)
-					&& version == newVersion)
+					out var version))
 				{
-					var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault();
-					if (asset == default)
-						continue;
-
-					serverUpdateOperation = new ServerUpdateOperation
+					if (version == newVersion)
 					{
-						TargetVersion = version,
-						UpdateZipUrl = new Uri(asset.BrowserDownloadUrl),
-						SwarmService = swarmService,
-					};
+						var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault();
+						if (asset == default)
+							continue;
 
-					try
-					{
-						if (!serverControl.TryStartUpdate(this, version))
-							return ServerUpdateResult.UpdateInProgress;
+						serverUpdateOperation = new ServerUpdateOperation
+						{
+							TargetVersion = version,
+							UpdateZipUrl = new Uri(asset.BrowserDownloadUrl),
+							SwarmService = swarmService,
+						};
+
+						try
+						{
+							if (!serverControl.TryStartUpdate(this, version))
+								return ServerUpdateResult.UpdateInProgress;
+						}
+						finally
+						{
+							serverUpdateOperation = null;
+						}
+
+						return ServerUpdateResult.Started;
 					}
-					finally
-					{
-						serverUpdateOperation = null;
-					}
-
-					return ServerUpdateResult.Started;
 				}
 				else
 					logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName);
@@ -181,7 +191,11 @@ namespace Tgstation.Server.Host.Core
 				try
 				{
 					logger.LogTrace("Downloading zip package...");
-					updateZipData = await fileDownloader.DownloadFile(serverUpdateOperation.UpdateZipUrl, cancellationToken);
+					var bearerToken = generalConfiguration.GitHubAccessToken;
+					if (String.IsNullOrWhiteSpace(bearerToken))
+						bearerToken = null;
+
+					updateZipData = await fileDownloader.DownloadFile(serverUpdateOperation.UpdateZipUrl, bearerToken, cancellationToken);
 				}
 				catch (Exception ex)
 				{

src/Tgstation.Server.Host/IO/FileDownloader.cs

@@ -1,11 +1,13 @@
 using System;
 using System.IO;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Tgstation.Server.Api;
 using Tgstation.Server.Common;
 
 namespace Tgstation.Server.Host.IO
@@ -35,14 +37,20 @@ namespace Tgstation.Server.Host.IO
 		}
 
 		/// <inheritdoc />
-		public async Task<MemoryStream> DownloadFile(Uri url, CancellationToken cancellationToken)
+		public async Task<MemoryStream> DownloadFile(Uri url, string bearerToken, CancellationToken cancellationToken)
 		{
+			if (url == null)
+				throw new ArgumentNullException(nameof(url));
+
 			logger.LogDebug("Starting download of {url}...", url);
 			using var httpClient = httpClientFactory.CreateClient();
 			using var request = new HttpRequestMessage(
 				HttpMethod.Get,
 				url);
 
+			if (bearerToken != null)
+				request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, bearerToken);
+
 			var webRequestTask = httpClient.SendAsync(request, cancellationToken);
 			using var response = await webRequestTask;
 			response.EnsureSuccessStatusCode();

src/Tgstation.Server.Host/IO/IFileDownloader.cs

@@ -14,8 +14,9 @@ namespace Tgstation.Server.Host.IO
 		/// Downloads a file from <paramref name="url"/>.
 		/// </summary>
 		/// <param name="url">The URL to download.</param>
+		/// <param name="bearerToken">Optional <see cref="string"/> to use as the "Bearer" value in the optional "Authorization" header for the request.</param>
 		/// <param name="cancellationToken">A <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in a <see cref="MemoryStream"/> of the downloaded file.</returns>
-		Task<MemoryStream> DownloadFile(Uri url, CancellationToken cancellationToken);
+		Task<MemoryStream> DownloadFile(Uri url, string bearerToken, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -74,8 +74,7 @@
     <!-- Usage: GitLab interop -->
     <PackageReference Include="GitLabApiClient" Version="1.8.0" />
     <!-- Usage: git interop -->
-    <!-- Pinned: Later versions are known to show issues with TGS. Do not upgrade without thorough git testing -->
-    <PackageReference Include="LibGit2Sharp" Version="0.27.0-preview-0034" />
+    <PackageReference Include="LibGit2Sharp" Version="0.27.2" />
     <!-- Usage: JWT injection into HTTP pipeline -->
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.15" />
     <!-- Usage: Support ""legacy"" Newotonsoft.Json in HTTP pipeline. The rest of our codebase uses Newtonsoft. -->

tests/Tgstation.Server.Host.Tests/Components/Byond/TestPosixByondInstaller.cs

@@ -51,7 +51,7 @@ namespace Tgstation.Server.Host.Components.Byond.Tests
 			await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => installer.DownloadVersion(null, default));
 
 			var ourArray = Array.Empty<byte>();
-			mockFileDownloader.Setup(x => x.DownloadFile(It.Is<Uri>(uri => uri == new Uri("https://secure.byond.com/download/build/511/511.1385_byond_linux.zip")), default)).Returns(Task.FromResult(new MemoryStream(ourArray))).Verifiable();
+			mockFileDownloader.Setup(x => x.DownloadFile(It.Is<Uri>(uri => uri == new Uri("https://secure.byond.com/download/build/511/511.1385_byond_linux.zip")), null, default)).Returns(Task.FromResult(new MemoryStream(ourArray))).Verifiable();
 
 			var result = await installer.DownloadVersion(new Version(511, 1385), default);
 

tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs

@@ -14,6 +14,7 @@ using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host;
 using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Core;
+using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.Setup;
 using Tgstation.Server.Host.Utils;
 
@@ -189,7 +190,9 @@ namespace Tgstation.Server.Tests.Live
 			if (firstRun)
 				args[0] = string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", false);
 
-			using (swarmNodeId != null
+			var swarmMode = swarmNodeId != null;
+			ApplicationBuilderExtensions.LogSwarmIdentifier = swarmMode;
+			using (swarmMode
 				? LogContext.PushProperty(SerilogContextHelper.SwarmIdentifierContextProperty, swarmNodeId)
 				: null)
 				await RealServer.Run(cancellationToken);

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -44,6 +44,8 @@ namespace Tgstation.Server.Tests.Live
 		public static ushort DDPort { get; } = FreeTcpPort();
 		public static ushort DMPort { get; } = GetDMPort();
 
+		readonly Version TestUpdateVersion = new Version(5, 11, 0);
+
 		readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString()));
 
 		static void TerminateAllDDs()
@@ -120,13 +122,13 @@ namespace Tgstation.Server.Tests.Live
 			}
 
 			var connectionFactory = new DatabaseConnectionFactory();
-			using var connection = connectionFactory.CreateConnection(connectionString, databaseType);
 
 			using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1));
 			var cancellationToken = cts.Token;
 
 			while (true)
 			{
+				using var connection = connectionFactory.CreateConnection(connectionString, databaseType);
 				try
 				{
 					await connection.OpenAsync(cancellationToken);
@@ -252,11 +254,10 @@ namespace Tgstation.Server.Tests.Live
 					CheckInfo(controllerInfo);
 
 					// test update
-					var testUpdateVersion = new Version(4, 8, 1);
 					await controllerClient.Administration.Update(
 						new ServerUpdateRequest
 						{
-							NewVersion = testUpdateVersion
+							NewVersion = TestUpdateVersion
 						},
 						cancellationToken);
 					await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask);
@@ -270,7 +271,7 @@ namespace Tgstation.Server.Tests.Live
 						Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
 
 						var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
-						Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
+						Assert.AreEqual(TestUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
 						Directory.Delete(server.UpdatePath, true);
 					}
 
@@ -435,11 +436,10 @@ namespace Tgstation.Server.Tests.Live
 					await Assert.ThrowsExceptionAsync<ConflictException>(() => node1Client.Instances.GetId(controllerInstance, cancellationToken));
 
 					// test update
-					var testUpdateVersion = new Version(4, 8, 1);
 					await node1Client.Administration.Update(
 						new ServerUpdateRequest
 						{
-							NewVersion = testUpdateVersion
+							NewVersion = TestUpdateVersion
 						},
 						cancellationToken);
 					await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask);
@@ -453,7 +453,7 @@ namespace Tgstation.Server.Tests.Live
 						Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
 
 						var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
-						Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
+						Assert.AreEqual(TestUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
 						Directory.Delete(server.UpdatePath, true);
 					}
 
@@ -479,7 +479,7 @@ namespace Tgstation.Server.Tests.Live
 					await ApiAssert.ThrowsException<ApiConflictException>(() => controllerClient2.Administration.Update(
 						new ServerUpdateRequest
 						{
-							NewVersion = testUpdateVersion
+							NewVersion = TestUpdateVersion
 						},
 						cancellationToken), ErrorCode.SwarmIntegrityCheckFailed);
 
@@ -493,7 +493,7 @@ namespace Tgstation.Server.Tests.Live
 					await controllerClient2.Administration.Update(
 						new ServerUpdateRequest
 						{
-							NewVersion = testUpdateVersion
+							NewVersion = TestUpdateVersion
 						},
 						cancellationToken);
 

tools/Tgstation.Server.Migrator/Program.cs

@@ -386,7 +386,7 @@ try
 	using (var loggerFactory = LoggerFactory.Create(builder => { }))
 	{
 		var fileDownloader = new FileDownloader(httpClientFactory, loggerFactory.CreateLogger<FileDownloader>());
-		using var tgsFiveZipMemoryStream = await fileDownloader.DownloadFile(new Uri(serverServiceAsset.BrowserDownloadUrl), default);
+		using var tgsFiveZipMemoryStream = await fileDownloader.DownloadFile(new Uri(serverServiceAsset.BrowserDownloadUrl), null, default);
 		Console.WriteLine("Unzipping TGS5...");
 		await serverFactory.IOManager.ZipToDirectory(tgsInstallPath, tgsFiveZipMemoryStream, default);
 	}