From 2da9f7ca82a3ecd3faa9ed8ef32c4660eba99f72 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 17:23:04 -0500 Subject: [PATCH 02/15] Fix GitHubAccessToken never being used --- .../Core/GitHubClientFactory.cs | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs index 218a19bf4f..90423ac8d4 100644 --- a/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs @@ -1,5 +1,7 @@ using System; +using Microsoft.Extensions.Options; using Octokit; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Core @@ -12,35 +14,43 @@ namespace Tgstation.Server.Host.Core /// readonly IAssemblyInformationProvider assemblyInformationProvider; + /// + /// The for the . + /// + readonly GeneralConfiguration generalConfiguration; + /// /// Construct a /// - /// The value of - public GitHubClientFactory(IAssemblyInformationProvider assemblyInformationProvider) + /// The value of . + /// + public GitHubClientFactory(IAssemblyInformationProvider assemblyInformationProvider, IOptions generalConfigurationOptions) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } /// - /// Create a + /// Create a . /// + /// Optional access token to use as credentials. /// A new - GitHubClient CreateBaseClient() => new GitHubClient( - new ProductHeaderValue( - assemblyInformationProvider.ProductInfoHeaderValue.Product.Name, - assemblyInformationProvider.ProductInfoHeaderValue.Product.Version)); - - /// - public IGitHubClient CreateClient() => CreateBaseClient(); - - /// - public IGitHubClient CreateClient(string accessToken) + GitHubClient CreateClientImpl(string accessToken) { - if (accessToken == null) - throw new ArgumentNullException(nameof(accessToken)); - var result = CreateBaseClient(); - result.Credentials = new Credentials(accessToken); - return result; + var client = new GitHubClient( + new ProductHeaderValue( + assemblyInformationProvider.ProductInfoHeaderValue.Product.Name, + assemblyInformationProvider.ProductInfoHeaderValue.Product.Version)); + if (accessToken != null) + client.Credentials = new Credentials(accessToken); + + return client; } + + /// + public IGitHubClient CreateClient() => CreateClientImpl(generalConfiguration.GitHubAccessToken); + + /// + public IGitHubClient CreateClient(string accessToken) => CreateClientImpl(accessToken ?? throw new ArgumentNullException(nameof(accessToken))); } } From 47806398e7e33b3ebac1ba39d883c6183404af02 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 17:24:37 -0500 Subject: [PATCH 03/15] Version bump to 4.8.2 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index f63b1c518b..1eb3527d2a 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 4.8.1 + 4.8.2 2.3.0 8.3.0 9.2.0 From bc7b6b375003eece8b7efc68b95f907bd9ab4d19 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 19:36:55 -0500 Subject: [PATCH 04/15] Makes an annoying log trace --- src/Tgstation.Server.Host/Controllers/SwarmController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Controllers/SwarmController.cs b/src/Tgstation.Server.Host/Controllers/SwarmController.cs index 1c2409dd8a..473df47a38 100644 --- a/src/Tgstation.Server.Host/Controllers/SwarmController.cs +++ b/src/Tgstation.Server.Host/Controllers/SwarmController.cs @@ -235,7 +235,7 @@ namespace Tgstation.Server.Host.Controllers return; } - logger.LogDebug("Starting swarm request processing..."); + logger.LogTrace("Starting swarm request processing..."); await base.OnActionExecutionAsync(context, next).ConfigureAwait(false); } } From e2a26e96b2ed8f13f8ea55d91980efe2319777f5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 19:57:01 -0500 Subject: [PATCH 05/15] Documentation fixes --- src/Tgstation.Server.Host/Core/GitHubClientFactory.cs | 2 +- src/Tgstation.Server.Host/Core/IGitHubClientFactory.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs index 90423ac8d4..b77f48478c 100644 --- a/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Core/GitHubClientFactory.cs @@ -23,7 +23,7 @@ namespace Tgstation.Server.Host.Core /// Construct a /// /// The value of . - /// + /// The containing the value of . public GitHubClientFactory(IAssemblyInformationProvider assemblyInformationProvider, IOptions generalConfigurationOptions) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); diff --git a/src/Tgstation.Server.Host/Core/IGitHubClientFactory.cs b/src/Tgstation.Server.Host/Core/IGitHubClientFactory.cs index 68cf07d9e7..d91962bf50 100644 --- a/src/Tgstation.Server.Host/Core/IGitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Core/IGitHubClientFactory.cs @@ -8,7 +8,7 @@ namespace Tgstation.Server.Host.Core public interface IGitHubClientFactory { /// - /// Create a client with anonymous authentication or general authentica. Low rate limit. Attempts to use the server's token to bypass this. + /// Create a client. Low rate limit unless the server's GitHubAccessToken is set to bypass it. /// /// A new . IGitHubClient CreateClient(); From 9014e4f7cc303d5f10a1eaa2a5f4e32c370462d0 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 20:08:07 -0500 Subject: [PATCH 06/15] Fix swarm updates initiated from the controller failing --- .../Swarm/SwarmService.cs | 5 ++++ .../Tgstation.Server.Tests/IntegrationTest.cs | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index ed94445764..1b11dd58b1 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -535,6 +535,11 @@ namespace Tgstation.Server.Host.Swarm return false; } } + else + { + logger.LogTrace("No need to re-initiate update as it originated here on the swarm controller"); + updateCommitTcs = new TaskCompletionSource(); + } logger.LogDebug("Prepared for update to version {0}", version); } diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 83b09adc0f..51e2b04f6b 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -276,11 +276,36 @@ namespace Tgstation.Server.Tests var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath); Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver()); + Directory.Delete(server.UpdatePath, true); } CheckServerUpdated(controller); CheckServerUpdated(node1); CheckServerUpdated(node2); + + // regression: test it also works from the controller + serverTask = Task.WhenAll( + node1.Run(cancellationToken), + node2.Run(cancellationToken), + controller.Run(cancellationToken)); + + using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); + using var node1Client2 = await CreateAdminClient(node1.Url, cancellationToken); + using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); + + await controllerClient2.Administration.Update( + new Administration + { + NewVersion = testUpdateVersion + }, + cancellationToken); + + await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask); + Assert.IsTrue(serverTask.IsCompleted); + + CheckServerUpdated(controller); + CheckServerUpdated(node1); + CheckServerUpdated(node2); } finally { From f6b65c4cfa1a5390bf86e2781fd36935b3980e74 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 25 Jan 2021 20:13:44 -0500 Subject: [PATCH 07/15] Test build fix --- .../Core/TestGitHubClientFactory.cs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/Core/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Core/TestGitHubClientFactory.cs index 7a8c565f6b..6c3b29c467 100644 --- a/tests/Tgstation.Server.Host.Tests/Core/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Core/TestGitHubClientFactory.cs @@ -1,9 +1,11 @@ +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Octokit; using System; using System.Net.Http.Headers; using System.Threading.Tasks; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Core.Tests @@ -12,7 +14,11 @@ namespace Tgstation.Server.Host.Core.Tests public sealed class TestGitHubClientFactory { [TestMethod] - public void TestContruction() => Assert.ThrowsException(() => new GitHubClientFactory(null)); + public void TestContruction() + { + Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), null)); + Assert.ThrowsException(() => new GitHubClientFactory(null, null)); + } [TestMethod] public async Task TestCreateBasicClient() @@ -20,7 +26,12 @@ namespace Tgstation.Server.Host.Core.Tests var mockApp = new Mock(); mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable(); - var factory = new GitHubClientFactory(mockApp.Object); + var mockOptions = new Mock>(); + + var gc = new GeneralConfiguration(); + Assert.IsNull(gc.GitHubAccessToken); + mockOptions.SetupGet(x => x.Value).Returns(gc); + var factory = new GitHubClientFactory(mockApp.Object, mockOptions.Object); var client = factory.CreateClient(); Assert.IsNotNull(client); @@ -28,6 +39,12 @@ namespace Tgstation.Server.Host.Core.Tests Assert.AreEqual(AuthenticationType.Anonymous, credentials.AuthenticationType); + gc.GitHubAccessToken = "asdfasdfasdfasdfasdfasdf"; + client = factory.CreateClient(); + Assert.IsNotNull(client); + credentials = await client.Connection.CredentialStore.GetCredentials().ConfigureAwait(false); + + Assert.AreEqual(AuthenticationType.Oauth, credentials.AuthenticationType); mockApp.VerifyAll(); } @@ -38,7 +55,9 @@ namespace Tgstation.Server.Host.Core.Tests var mockApp = new Mock(); mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable(); - var factory = new GitHubClientFactory(mockApp.Object); + var mockOptions = new Mock>(); + mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var factory = new GitHubClientFactory(mockApp.Object, mockOptions.Object); Assert.ThrowsException(() => factory.CreateClient(null)); From 490061c05791763f67894c927d663cf87d5166c0 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 26 Jan 2021 12:41:38 -0500 Subject: [PATCH 08/15] Use 4.8.1 as the test update version --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 51e2b04f6b..7353aab914 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -257,7 +257,7 @@ namespace Tgstation.Server.Tests await Assert.ThrowsExceptionAsync(() => node1Client.Instances.GetId(controllerInstance, cancellationToken)); // test update - var testUpdateVersion = new Version(4, 6, 2); + var testUpdateVersion = new Version(4, 8, 1); await node1Client.Administration.Update( new Administration { From 6ac6a1b362f0413d35341dd8ca32b6884ee3b605 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 26 Jan 2021 14:31:20 -0500 Subject: [PATCH 09/15] Direct people to the discussions page --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7e467f3d3..f9cd540fdd 100644 --- a/README.md +++ b/README.md @@ -452,7 +452,7 @@ Should you end up with a lost database for some reason or want to reattach a det ## Troubleshooting -Feel free to ask for help at the coderbus discord in \#hosting-questions: https://discord.gg/Vh8TJp9. Cyberboss#0016 can answer most questions. +Feel free to ask for help [on the discussions page](https://github.com/tgstation/tgstation-server/discussions). ## Contributing From 849947adbaf35feb00971a2fbb404e78cfaf7d7c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 26 Jan 2021 16:28:58 -0500 Subject: [PATCH 10/15] Workaround for apt install bug --- .github/workflows/ci-suite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 1e33654d09..3358498148 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -31,7 +31,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y libc6-i386 libstdc++6:i386 + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 - name: Install BYOND if: steps.cache-byond.outputs.cache-hit != 'true' @@ -315,7 +315,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y libc6-i386 libstdc++6:i386 gdb + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb - name: Install Node 12.X uses: actions/setup-node@v1 From b506fb6d1caa80c42cd3807530647922328bc422 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 27 Jan 2021 13:50:26 -0500 Subject: [PATCH 11/15] 500 max_connections for Postgres integration tests Fixes #1095 [TGSDeploy] --- .github/workflows/ci-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 3358498148..dce4011d5f 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -272,7 +272,7 @@ jobs: ports: - 5432:5432 env: - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: cyberboss/postgres-max-connections # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready From 063ab88e108cd2790a291acb1e5d97b05d9d3fb7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 27 Jan 2021 17:05:56 -0500 Subject: [PATCH 12/15] Correct references to old webpanel repo --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- tools/ReleaseNotes/Program.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9008024d37..7f947ac1b4 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -29,7 +29,7 @@ - + diff --git a/tools/ReleaseNotes/Program.cs b/tools/ReleaseNotes/Program.cs index 8522b4066b..6fa66b6e81 100644 --- a/tools/ReleaseNotes/Program.cs +++ b/tools/ReleaseNotes/Program.cs @@ -263,7 +263,7 @@ namespace ReleaseNotes if (webControlVersion.Major == 0) postControlPanelMessage = true; - prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions.{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-control-panel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; + prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions.{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-webpanel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; break; case 3: prefix = "The /tg/station server suite"; From 1cd07f3006de2041c5264502629b5b85609edce6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 27 Jan 2021 17:13:38 -0500 Subject: [PATCH 13/15] Catch rate limit exception that was failing tests Progress on #1206 [TGSDeploy] --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 7353aab914..105499d95a 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -91,12 +91,12 @@ namespace Tgstation.Server.Tests var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath); Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver()); } - catch (RateLimitException) + catch (RateLimitException ex) { if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS4_TEST_GITHUB_TOKEN"))) throw; - Assert.Inconclusive("GitHub rate limit hit!"); + Assert.Inconclusive("GitHub rate limit hit: {0}", ex); } finally { @@ -307,6 +307,13 @@ namespace Tgstation.Server.Tests CheckServerUpdated(node1); CheckServerUpdated(node2); } + catch (RateLimitException ex) + { + if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS4_TEST_GITHUB_TOKEN"))) + throw; + + Assert.Inconclusive("GitHub rate limit hit: {0}", ex); + } finally { serverCts.Cancel(); From 654cfd846daae310dd6281a632fa6afefe535cf5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 27 Jan 2021 20:35:20 -0500 Subject: [PATCH 14/15] Another rate limit exception test failure [TGSDeploy] --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 105499d95a..26c192b618 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -504,6 +504,13 @@ namespace Tgstation.Server.Tests Assert.AreEqual(2, node2Info.SwarmServers.Count); Assert.IsNotNull(node2Info.SwarmServers.SingleOrDefault(x => x.Identifier == "controller")); } + catch (RateLimitException ex) + { + if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS4_TEST_GITHUB_TOKEN"))) + throw; + + Assert.Inconclusive("GitHub rate limit hit: {0}", ex); + } finally { serverCts.Cancel(); From 380b6e089eb24a7a39297a0cda1b5f9469a3d230 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 27 Jan 2021 20:57:05 -0500 Subject: [PATCH 15/15] Holy shit I'm dumb Fix POSTGRES_PASSWORD and image being swapped. [TGSDeploy] --- .github/workflows/ci-suite.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index dce4011d5f..6576a25841 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -268,12 +268,11 @@ jobs: needs: dmapi-build services: # We start all dbs here so we can just code the stuff once postgres: - image: postgres + image: cyberboss/postgres-max-connections # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. ports: - 5432:5432 env: - POSTGRES_PASSWORD: cyberboss/postgres-max-connections # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. - # Set health checks to wait until postgres has started + POSTGRES_PASSWORD: postgres options: >- --health-cmd pg_isready --health-interval 10s