diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5551c047e3..1efe65423b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -2,6 +2,7 @@ # Does CI on push/PR/cron. Deployments on push when triggered # - Validates Documentation # - Builds C# and DMAPI +# - Runs CodeQL Anaylsis # - Tests everything on massive matrix # - Packages # - Tests package installs/services/uninstalls @@ -19,7 +20,7 @@ name: 'CI Pipeline' on: schedule: - - cron: 0 23 * * * + - cron: 0 9 * * * push: branches: - dev @@ -136,11 +137,11 @@ jobs: exit 0 - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -182,11 +183,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -222,11 +223,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -286,11 +287,11 @@ jobs: if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -324,11 +325,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -375,11 +376,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -490,11 +491,11 @@ jobs: echo "TGS_TEST_DATABASE_TYPE=SqlServer" >> $GITHUB_ENV - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -531,11 +532,19 @@ jobs: fi - name: Store Live Tests Output + if: ${{ steps.live-tests.outputs.succeeded == 'YES' }} uses: actions/upload-artifact@v3 with: name: windows-integration-test-logs-${{ matrix.configuration }}-${{ matrix.watchdog-type }}-${{ matrix.database-type }} path: ./test_output.txt + - name: Store Errored Live Tests Output + if: ${{ steps.live-tests.outputs.succeeded != 'YES' }} + uses: actions/upload-artifact@v3 + with: + name: errored-windows-test-logs-${{ matrix.configuration }}-${{ matrix.watchdog-type }}-${{ matrix.database-type }} + path: ./test_output.txt + - name: Fail if Live Tests Failed if: ${{ steps.live-tests.outputs.succeeded != 'YES' }} run: exit 1 @@ -593,14 +602,14 @@ jobs: if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') services: # We start all dbs here so we can just code the stuff once mssql: - image: mcr.microsoft.com/mssql/server:2019-latest + image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} env: SA_PASSWORD: myPassword ACCEPT_EULA: 'Y' ports: - 1433:1433 postgres: - image: cyberboss/postgres-max-connections # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. + image: ${{ (matrix.database-type == 'PostgresSql') && 'cyberboss/postgres-max-connections' || '' }} # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. ports: - 5432:5432 env: @@ -611,7 +620,7 @@ jobs: --health-timeout 5s --health-retries 5 mariadb: - image: mariadb + image: ${{ (matrix.database-type == 'MariaDB') && 'mariadb' || '' }} ports: - 3306:3306 env: @@ -622,7 +631,7 @@ jobs: --health-timeout=2s --health-retries=3 mysql: - image: mysql:5.7.31 + image: ${{ (matrix.database-type == 'MySql') && 'mysql:5.7.31' || '' }} ports: - 3307:3306 env: @@ -687,11 +696,11 @@ jobs: run: echo "General__UseBasicWatchdog=true" >> $GITHUB_ENV - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -766,11 +775,11 @@ jobs: run: npm i -g ibm-openapi-validator@0.51.3 - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -791,11 +800,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1079,11 +1088,11 @@ jobs: echo "New dotnet path should be $DOTNET_PATH" - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1162,11 +1171,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1294,11 +1303,11 @@ jobs: echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1343,7 +1352,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1407,7 +1416,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1470,7 +1479,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1521,7 +1530,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1545,7 +1554,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1757,7 +1766,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1796,7 +1805,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Parse TGS version run: | @@ -1820,7 +1829,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Parse TGS version run: | @@ -1853,7 +1862,7 @@ jobs: run: winget install wingetcreate --version 1.2.8.0 --disable-interactivity --accept-source-agreements # Pinned due to breaking every other version - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build ReleaseNotes run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 9442ce5530..a7baad0e0e 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -1,8 +1,6 @@ name: 'Code Scanning' on: - schedule: - - cron: 0 23 * * 1 push: branches: - dev @@ -39,7 +37,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/README.md b/README.md index 8ca58e0560..d2830e93d9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This is a toolset to manage production BYOND servers. It includes the ability to ### Pre-Requisites -_Note: If you opt to use the Windows installer, all pre-requisites (including MariaDB) are provided out of the box._ +_Note: If you opt to use the Windows installer, all pre-requisites for running BYOND servers (including MariaDB) are provided out of the box. If you wish to use OpenDream you will need to install the required dotnet SDK manually._ tgstation-server needs a relational database to store it's data. @@ -101,9 +101,19 @@ If using the console version, run `./tgs.bat` in the root of the installation di Installing natively is the recommended way to run tgstation-server on Linux. -##### Ubuntu +##### Ubuntu/Debian Package -Install TGS and all it's dependencies via our apt repository, interactively configure it, and start the service with this one-liner: +You first need to add the appropriate Microsoft package repository for your distribution + +Refer to the Microsoft website for steps for + +- [Ubuntu](https://learn.microsoft.com/en-us/dotnet/core/install/linux-ubuntu#register-the-microsoft-package-repository) +- [Debian 12](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-12) +- [Debian 11](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-11) +- [Debian 10](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-10) +- [Other Distros](https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#manual-install) + +After that, install TGS and all it's dependencies via our apt repository, interactively configure it, and start the service with this one-liner: ```sh sudo dpkg --add-architecture i386 \ @@ -119,25 +129,11 @@ sudo dpkg --add-architecture i386 \ The service will execute as the newly created user: `tgstation-server`. -##### Debian - -The `aspnetcore-runtime-8.0` package isn't yet available on mainline Debian and must be [installed from Microsoft](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian) first. Use the following one-liner to add their packages repository. - -```sh -curl -L https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -o packages-microsoft-prod.deb \ -&& sudo dpkg -i packages-microsoft-prod.deb \ -&& rm packages-microsoft-prod.deb -``` - -After that, run the same command as the Ubuntu installation. - -_Support for more distros coming soon_ - -##### Manual +##### Manual Setup The following dependencies are required. -- aspnetcore-runtime-8.0 (Note, not all supported distros have this package, see the links above for official Microsoft installation instructions) +- aspnetcore-runtime-8.0 (See Prerequisites under the `Ubuntu/Debian Package` section) - libc6-i386 - libstdc++6:i386 - gcc-multilib (Only on 64-bit systems) @@ -185,6 +181,21 @@ Note that automatic configuration reloading is currently not supported in the co If using manual configuration, before starting your container make sure the aforementioned `appsettings.Production.yml` is setup properly. See below +#### OpenDream + +In order for TGS to use [OpenDream](https://github.com/OpenDreamProject/OpenDream), it requires the full .NET SDK to build whichever version your servers target. Whatever that is, it must be available using the `dotnet` command for whichever user runs TGS. + +OpenDream currently requires [.NET SDK 7.0](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) at the time of this writing. You must install this manually. + +On Linux, as long as OpenDream and TGS do not use the same .NET major version, you cannot achieve this with the package manager as they will conflict. The 7.0 SDK can be added to an 8.0 runtime installation via the following steps. + +1. Install `tgstation-server` using any of the above methods. +1. [Download the Linux SDK binaries](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) for your selected architecture. +1. Extract everything EXCEPT the `dotnet` executable, `LICENSE.txt``, and `ThirdPartyNotices.txt` in the `.tar.gz` on top of the existing installation directory `/usr/share/dotnet/` +1. Run `sudo chown -R root /usr/share/dotnet` + +You should now be able to run the `dotnet --list-sdks` command and see an entry for `7.0.XXX [/usr/share/dotnet/sdk]`. + ### Configuring The first time you run TGS you should be prompted with a configuration wizard which will guide you through setting up your `appsettings.Production.yml` diff --git a/build/NugetCommon.props b/build/NugetCommon.props index d1c341efd0..f07a00a656 100644 --- a/build/NugetCommon.props +++ b/build/NugetCommon.props @@ -25,7 +25,7 @@ - + diff --git a/build/analyzers.ruleset b/build/analyzers.ruleset index 034747c138..e61e800d00 100644 --- a/build/analyzers.ruleset +++ b/build/analyzers.ruleset @@ -1,4 +1,4 @@ - + @@ -88,7 +88,6 @@ - @@ -666,6 +665,12 @@ + + + + + + @@ -715,8 +720,8 @@ - - + + @@ -751,6 +756,8 @@ + + @@ -952,8 +959,8 @@ - - + + @@ -1026,8 +1033,6 @@ - - @@ -1043,4 +1048,4 @@ - + \ No newline at end of file diff --git a/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs b/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs new file mode 100644 index 0000000000..90c379f824 --- /dev/null +++ b/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs @@ -0,0 +1,25 @@ +using System; + +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Api.Extensions +{ + /// + /// Extension methods for the . + /// + public static class JobCodeExtensions + { + /// + /// If a given can be triggered by TGS startup. + /// + /// The . + /// if the can trigger before startup, otherwise. + public static bool IsServerStartupJob(this JobCode jobCode) + => jobCode switch + { + JobCode.Unknown or JobCode.Move or JobCode.RepositoryClone or JobCode.RepositoryUpdate or JobCode.RepositoryAutoUpdate or JobCode.RepositoryDelete or JobCode.EngineOfficialInstall or JobCode.EngineCustomInstall or JobCode.EngineDelete or JobCode.Deployment or JobCode.AutomaticDeployment or JobCode.WatchdogLaunch or JobCode.WatchdogRestart or JobCode.WatchdogDump => false, + JobCode.StartupWatchdogLaunch or JobCode.StartupWatchdogReattach or JobCode.ReconnectChatBot => true, + _ => throw new InvalidOperationException($"Invalid JobCode: {jobCode}"), + }; + } +} diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 21eff94c4b..876f1097d6 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -7,7 +7,6 @@ Client library for tgstation-server. json web api tgstation-server tgstation ss13 byond client http $(TGS_NUGET_RELEASE_NOTES_CLIENT) - NU5104 diff --git a/src/Tgstation.Server.Host.Service/Program.cs b/src/Tgstation.Server.Host.Service/Program.cs index c2f72b0901..4a9d39fcd2 100644 --- a/src/Tgstation.Server.Host.Service/Program.cs +++ b/src/Tgstation.Server.Host.Service/Program.cs @@ -179,7 +179,9 @@ namespace Tgstation.Server.Host.Service var exePath = Path.Combine(assemblyDirectory, $"{assemblyNameWithoutExtension}.exe"); var programDataDirectory = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.CommonApplicationData, + Environment.SpecialFolderOption.DoNotVerify), Server.Common.Constants.CanonicalPackageName); using var processInstaller = new ServiceProcessInstaller(); diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs b/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs index 369ccdeeeb..03c9ee2e54 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs @@ -16,7 +16,7 @@ namespace Tgstation.Server.Host.Components.Chat /// public bool Active { - get => active; + get => active && onDispose != null; set { if (active == value) diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index cf20d498a9..f02239969f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Engine protected override EngineType TargetEngineType => EngineType.Byond; /// - /// Bath to the system user's local BYOND folder. + /// Path to the system user's local BYOND folder. /// protected abstract string PathToUserFolder { get; } @@ -157,12 +157,6 @@ namespace Tgstation.Server.Host.Components.Engine ArgumentNullException.ThrowIfNull(fullDmbPath); var byondDir = PathToUserFolder; - if (String.IsNullOrWhiteSpace(byondDir)) - { - Logger.LogTrace("No relevant user BYOND directory to install a \"{fileName}\" in", TrustedDmbFileName); - return; - } - var cfgDir = IOManager.ConcatPath( byondDir, CfgDirectoryName); diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index bbd09ed1c1..b8ffc98334 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -230,18 +230,28 @@ namespace Tgstation.Server.Host.Components.Engine shortenedPath, $"run -c Release --project OpenDreamPackageTool -- --tgs -o {shortenedDeployPath}", null, - true, - true); + !GeneralConfiguration.OpenDreamSuppressInstallOutput, + !GeneralConfiguration.OpenDreamSuppressInstallOutput); using (cancellationToken.Register(() => buildProcess.Terminate())) buildExitCode = await buildProcess.Lifetime; - Logger.LogTrace("OD build complete, waiting for output..."); + string output; + + if (!GeneralConfiguration.OpenDreamSuppressInstallOutput) + { + var buildOutputTask = buildProcess.GetCombinedOutput(cancellationToken); + if (!buildOutputTask.IsCompleted) + Logger.LogTrace("OD build complete, waiting for output..."); + output = await buildOutputTask; + } + else + output = ""; Logger.LogDebug( "OpenDream build exited with code {exitCode}:{newLine}{output}", buildExitCode, Environment.NewLine, - await buildProcess.GetCombinedOutput(cancellationToken)); + output); }, sourcePath, cancellationToken); @@ -276,7 +286,9 @@ namespace Tgstation.Server.Host.Components.Engine cancellationToken))); } - await ValueTaskExtensions.WhenAll(MoveDirs(), MoveFiles()); + var dirsMoveTask = MoveDirs(); + var outputFilesMoveTask = MoveFiles(); + await ValueTaskExtensions.WhenAll(dirsMoveTask, outputFilesMoveTask); await IOManager.DeleteDirectory(sourcePath, cancellationToken); } diff --git a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs index cddace53c8..f3c9c561b1 100644 --- a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs @@ -65,7 +65,8 @@ namespace Tgstation.Server.Host.Components.Engine PathToUserFolder = IOManager.ResolvePath( IOManager.ConcatPath( Environment.GetFolderPath( - Environment.SpecialFolder.UserProfile), + Environment.SpecialFolder.UserProfile, + Environment.SpecialFolderOption.DoNotVerify), "./byond/cache")); } diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index 0ebf63ff9f..0f40d3f538 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -100,11 +100,12 @@ namespace Tgstation.Server.Host.Components.Engine this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); - var documentsDirectory = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); - if (String.IsNullOrWhiteSpace(documentsDirectory)) - PathToUserFolder = null; // happens with the service account - else - PathToUserFolder = IOManager.ResolvePath(IOManager.ConcatPath(documentsDirectory, "BYOND")); + var documentsDirectory = Environment.GetFolderPath( + Environment.SpecialFolder.MyDocuments, + Environment.SpecialFolderOption.DoNotVerify); + + PathToUserFolder = IOManager.ResolvePath( + IOManager.ConcatPath(documentsDirectory, "BYOND")); semaphore = new SemaphoreSlim(1); installedDirectX = false; diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs index 4f42884c7f..c97ca10418 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs @@ -56,15 +56,18 @@ namespace Tgstation.Server.Host.Components.Engine /// public override ValueTask Install(EngineVersion version, string installPath, CancellationToken cancellationToken) - => ValueTaskExtensions.WhenAll( - base.Install( - version, - installPath, - cancellationToken), - AddServerFirewallException( - version, - installPath, - cancellationToken)); + { + var installTask = base.Install( + version, + installPath, + cancellationToken); + var firewallTask = AddServerFirewallException( + version, + installPath, + cancellationToken); + + return ValueTaskExtensions.WhenAll(installTask, firewallTask); + } /// protected override async ValueTask HandleExtremelyLongPathOperation(Func shortenedPathOperation, string originalPath, CancellationToken cancellationToken) @@ -102,7 +105,7 @@ namespace Tgstation.Server.Host.Components.Engine // I really wish we could add the instance name here but // 1. It'd make IByondInstaller need to be transient per-instance and WindowsByondInstaller relys on being a singleton for its DX installer call // 2. The instance could be renamed, so it'd have to be an unfriendly ID anyway. - var ruleName = $"TGS DreamDaemon {version}"; + var ruleName = $"TGS OpenDream {version}"; exitCode = await WindowsFirewallHelper.AddFirewallException( ProcessExecutor, diff --git a/src/Tgstation.Server.Host/Components/Interop/Chunker.cs b/src/Tgstation.Server.Host/Components/Interop/Chunker.cs index d6a0c7c024..85f6de0d6c 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Chunker.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Chunker.cs @@ -57,15 +57,15 @@ namespace Tgstation.Server.Host.Components.Interop /// /// Process a given . /// - /// The of communication that was chunked. + /// The of communication that was chunked. /// The of expected. - /// The callback that receives the completed . + /// The callback that receives the completed . /// The callback that generates a for a given error. /// The . /// The for the operation. /// A resulting in the for the chunked request. - protected async ValueTask ProcessChunk( - Func> completionCallback, + protected async ValueTask ProcessChunk( + Func> completionCallback, Func chunkErrorCallback, ChunkData chunk, CancellationToken cancellationToken) @@ -128,11 +128,11 @@ namespace Tgstation.Server.Host.Components.Interop chunkSets.Remove(requestInfo.PayloadId.Value); } - TCommnication completedCommunication; + TCommunication completedCommunication; var fullCommunicationJson = String.Concat(payloads); try { - completedCommunication = JsonConvert.DeserializeObject(fullCommunicationJson, DMApiConstants.SerializerSettings); + completedCommunication = JsonConvert.DeserializeObject(fullCommunicationJson, DMApiConstants.SerializerSettings); } catch (Exception ex) { diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index a3b81d2bad..125ceded30 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -247,20 +247,16 @@ namespace Tgstation.Server.Host.Components.Repository libGitRepo, refSpecList, remote, - new FetchOptions - { - Prune = true, - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = TransferProgressHandler( - progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), - cancellationToken), - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }, + new FetchOptions().Hydrate( + logger, + progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken), logMessage); } - catch (UserCancelledException) + catch (UserCancelledException ex) { + logger.LogTrace(ex, "Suppressing fetch cancel exception"); } catch (LibGit2SharpException ex) { @@ -439,14 +435,12 @@ namespace Tgstation.Server.Host.Components.Repository var fetchOptions = new FetchOptions { Prune = true, - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), TagFetchMode = TagFetchMode.All, - }; - - if (progressReporter != null) - fetchOptions.OnTransferProgress = TransferProgressHandler(progressReporter.CreateSection("Fetch Origin", 1.0), cancellationToken); + }.Hydrate( + logger, + progressReporter?.CreateSection("Fetch Origin", 1.0), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken); commands.Fetch( libGitRepo, @@ -1041,19 +1035,18 @@ namespace Tgstation.Server.Host.Components.Repository var submoduleUpdateOptions = new SubmoduleUpdateOptions { Init = true, - OnProgress = output => !cancellationToken.IsCancellationRequested, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), + OnCheckoutNotify = (_, _) => !cancellationToken.IsCancellationRequested, }; + submoduleUpdateOptions.FetchOptions.Hydrate( + logger, + progressReporter?.CreateSection($"Fetch submodule {submodule.Name}", factor), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken); + if (progressReporter != null) - { - submoduleUpdateOptions.OnTransferProgress = TransferProgressHandler( - progressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor), - cancellationToken); submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler( progressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)); - } logger.LogDebug("Updating submodule {submoduleName}...", submodule.Name); Task RawSubModuleUpdate() => Task.Factory.StartNew( @@ -1131,37 +1124,6 @@ namespace Tgstation.Server.Host.Components.Repository progressReporter.ReportProgress(percentage); }; - - /// - /// Generate a from a given and . - /// - /// The of the operation. - /// The for the operation. - /// A new based on . - TransferProgressHandler TransferProgressHandler(JobProgressReporter progressReporter, CancellationToken cancellationToken) => (transferProgress) => - { - double? percentage; - var totalObjectsToProcess = transferProgress.TotalObjects * 2; - var processedObjects = transferProgress.IndexedObjects + transferProgress.ReceivedObjects; - if (totalObjectsToProcess < processedObjects || totalObjectsToProcess == 0) - percentage = null; - else - { - percentage = (double)processedObjects / totalObjectsToProcess; - if (percentage < 0) - percentage = null; - } - - if (percentage == null) - logger.LogDebug( - "Bad transfer progress values (Please tell Cyberboss)! Indexed: {indexed}, Received: {received}, Total: {total}", - transferProgress.IndexedObjects, - transferProgress.ReceivedObjects, - transferProgress.TotalObjects); - - progressReporter.ReportProgress(percentage); - return !cancellationToken.IsCancellationRequested; - }; } #pragma warning restore CA1506 } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 2940e0d134..8ce9b60318 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Utils; @@ -127,8 +128,6 @@ namespace Tgstation.Server.Host.Components.Repository CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(url); - - logger.LogInformation("Begin clone {url} (Branch: {initialBranch})", url, initialBranch); lock (semaphore) { if (CloneInProgress) @@ -136,12 +135,14 @@ namespace Tgstation.Server.Host.Components.Repository CloneInProgress = true; } + var repositoryPath = ioManager.ResolvePath(); + logger.LogInformation("Begin clone {url} to {path} (Branch: {initialBranch})", url, repositoryPath, initialBranch); + try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) { logger.LogTrace("Semaphore acquired for clone"); - var repositoryPath = ioManager.ResolvePath(); if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try { @@ -149,20 +150,7 @@ namespace Tgstation.Server.Host.Components.Repository var checkoutProgressReporter = progressReporter?.CreateSection(null, 0.25f); var cloneOptions = new CloneOptions { - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => - { - if (cloneProgressReporter != null) - { - var percentage = ((double)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2); - cloneProgressReporter.ReportProgress(percentage); - } - - return !cancellationToken.IsCancellationRequested; - }, RecurseSubmodules = recurseSubmodules, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, OnCheckoutProgress = (path, completed, remaining) => { if (checkoutProgressReporter == null) @@ -172,9 +160,14 @@ namespace Tgstation.Server.Host.Components.Repository checkoutProgressReporter.ReportProgress(percentage); }, BranchName = initialBranch, - CredentialsProvider = repositoryFactory.GenerateCredentialsHandler(username, password), }; + cloneOptions.FetchOptions.Hydrate( + logger, + cloneProgressReporter, + repositoryFactory.GenerateCredentialsHandler(username, password), + cancellationToken); + await repositoryFactory.Clone( url, cloneOptions, diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index d3feab14a6..2e57410eab 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -22,6 +21,7 @@ using Tgstation.Server.Host.Components.Engine; using Tgstation.Server.Host.Components.Interop; using Tgstation.Server.Host.Components.Interop.Bridge; using Tgstation.Server.Host.Components.Interop.Topic; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -108,6 +108,11 @@ namespace Tgstation.Server.Host.Components.Session /// public ReattachInformation ReattachInformation { get; } + /// + /// The used to prevent concurrent calls into /world/Topic(). + /// + public FifoSemaphore TopicSendSemaphore { get; } + /// /// The for the . /// @@ -148,11 +153,6 @@ namespace Tgstation.Server.Host.Components.Session /// readonly TaskCompletionSource initialBridgeRequestTcs; - /// - /// The used to prevent concurrent calls into /world/Topic(). - /// - readonly FifoSemaphore topicSendSemaphore; - /// /// The metadata. /// @@ -285,7 +285,7 @@ namespace Tgstation.Server.Host.Components.Session initialBridgeRequestTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); reattachTopicCts = new CancellationTokenSource(); - topicSendSemaphore = new FifoSemaphore(); + TopicSendSemaphore = new FifoSemaphore(); synchronizationLock = new object(); if (apiValidationSession || DMApiAvailable) @@ -337,9 +337,8 @@ namespace Tgstation.Server.Host.Components.Session Logger.LogTrace("Disposing..."); - // yield then acquire the topic semaphore to prevent new calls from starting - await Task.Yield(); - (await topicSendSemaphore.Lock(CancellationToken.None)).Dispose(); // DCT: None available + reattachTopicCts.Cancel(); + var semaphoreLockTask = TopicSendSemaphore.Lock(CancellationToken.None); // DCT: None available if (!released) { @@ -363,7 +362,8 @@ namespace Tgstation.Server.Host.Components.Session if (!released) await Lifetime; // finish the async callback - topicSendSemaphore.Dispose(); + (await semaphoreLockTask).Dispose(); + TopicSendSemaphore.Dispose(); } /// @@ -884,35 +884,15 @@ namespace Tgstation.Server.Host.Components.Session } var targetPort = ReattachInformation.Port; - Byond.TopicSender.TopicResponse byondResponse = null; - var firstSend = true; - - using (await topicSendSemaphore.Lock(cancellationToken)) - { - const int PrioritySendAttempts = 5; - var endpoint = new IPEndPoint(IPAddress.Loopback, targetPort); - for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) - try - { - firstSend = false; - - Logger.LogTrace("Begin topic request"); - byondResponse = await byondTopicSender.SendTopic( - endpoint, - queryString, - cancellationToken); - - Logger.LogTrace("End topic request"); - break; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); - - if (priority && i > 0) - await asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); - } - } + Byond.TopicSender.TopicResponse byondResponse; + using (await TopicSendSemaphore.Lock(cancellationToken)) + byondResponse = await byondTopicSender.SendWithOptionalPriority( + asyncDelayer, + Logger, + queryString, + targetPort, + priority, + cancellationToken); if (byondResponse == null) { diff --git a/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs b/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs index 40605e5de8..29768e7604 100644 --- a/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs @@ -1,6 +1,7 @@ using System; using Byond.TopicSender; + using Microsoft.Extensions.Logging; namespace Tgstation.Server.Host.Components.Session diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f42b4cc7e2..5e871c2551 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -71,7 +71,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// Map of s to the filename of the event scripts they trigger. /// - static readonly IReadOnlyDictionary> EventTypeScriptFileNameMap = new Dictionary>( + public static IReadOnlyDictionary> EventTypeScriptFileNameMap { get; } = new Dictionary>( Enum.GetValues(typeof(EventType)) .Cast() .Select( @@ -298,8 +298,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles string GetFileSha() { var content = synchronousIOManager.ReadFile(path); - using var sha1 = SHA1.Create(); - return String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + return String.Join(String.Empty, SHA1.HashData(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); } var originalSha = GetFileSha(); @@ -406,12 +405,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles var fileName = ioManager.GetFileName(file); // need to normalize - bool ignored; - if (platformIdentifier.IsWindows) - ignored = ignoreFiles.Any(y => fileName.ToUpperInvariant() == y.ToUpperInvariant()); - else - ignored = ignoreFiles.Any(y => fileName == y); - + var fileComparison = platformIdentifier.IsWindows + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + var ignored = ignoreFiles.Any(y => fileName.Equals(y, fileComparison)); if (ignored) { logger.LogTrace("Ignoring static file {fileName}...", fileName); @@ -442,12 +439,14 @@ namespace Tgstation.Server.Host.Components.StaticFiles using (var reader = new StringReader(ignoreFileText)) { cancellationToken.ThrowIfCancellationRequested(); - var line = await reader.ReadLineAsync(); + var line = await reader.ReadLineAsync(cancellationToken); if (!String.IsNullOrEmpty(line)) ignoreFiles.Add(line); } - await ValueTaskExtensions.WhenAll(SymlinkBase(true), SymlinkBase(false)); + var filesSymlinkTask = SymlinkBase(true); + var dirsSymlinkTask = SymlinkBase(false); + await ValueTaskExtensions.WhenAll(filesSymlinkTask, dirsSymlinkTask); } } @@ -615,7 +614,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) .ToList(); - if (!scriptFiles.Any()) + if (scriptFiles.Count == 0) { logger.LogTrace("No event scripts starting with \"{scriptName}\" detected", String.Join("\" or \"", scriptNames)); return; @@ -710,19 +709,19 @@ namespace Tgstation.Server.Host.Components.StaticFiles return; await ioManager.CreateDirectory(CodeModificationsSubdirectory, cancellationToken); - await ValueTaskExtensions.WhenAll( - ioManager.WriteAllBytes( - ioManager.ConcatPath( - CodeModificationsSubdirectory, - CodeModificationsHeadFile), - Encoding.UTF8.GetBytes(DefaultHeadInclude), - cancellationToken), - ioManager.WriteAllBytes( - ioManager.ConcatPath( - CodeModificationsSubdirectory, - CodeModificationsTailFile), - Encoding.UTF8.GetBytes(DefaultTailInclude), - cancellationToken)); + var headWriteTask = ioManager.WriteAllBytes( + ioManager.ConcatPath( + CodeModificationsSubdirectory, + CodeModificationsHeadFile), + Encoding.UTF8.GetBytes(DefaultHeadInclude), + cancellationToken); + var tailWriteTask = ioManager.WriteAllBytes( + ioManager.ConcatPath( + CodeModificationsSubdirectory, + CodeModificationsTailFile), + Encoding.UTF8.GetBytes(DefaultTailInclude), + cancellationToken); + await ValueTaskExtensions.WhenAll(headWriteTask, tailWriteTask); } return Task.WhenAll( diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 33140b9e57..4495a99fe3 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -395,6 +395,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { + Logger.LogTrace("Making new provider {id} active...", newProvider.CompileJob.Id); await newProvider.MakeActive(cancellationToken); } finally diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index a54cde8e83..7600dc9a36 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -62,8 +62,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The new . May be modified. /// The for the operation. - /// A representing the running operation. - ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken); + /// A resulting in if a reboot is required, otherwise. + ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken); /// /// Restarts the watchdog. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 33b5ca0720..4fb3e1677f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -267,18 +267,20 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) + public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) { bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters); ActiveLaunchParameters = launchParameters; - if (match || Status == WatchdogStatus.Offline) - return; + if (match || Status == WatchdogStatus.Offline || Status == WatchdogStatus.DelayedRestart) + return false; var oldTcs = Interlocked.Exchange(ref activeParametersUpdated, new TaskCompletionSource()); oldTcs.SetResult(); } + + return true; } /// @@ -444,20 +446,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public async ValueTask CreateDump(CancellationToken cancellationToken) { - const string DumpDirectory = "ProcessDumps"; - await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); - - var dumpFileName = diagnosticsIOManager.ResolvePath( - diagnosticsIOManager.ConcatPath( - DumpDirectory, - $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); - - var session = GetActiveController(); - if (session?.Lifetime.IsCompleted != false) - throw new JobException(ErrorCode.GameServerOffline); - - Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); - await session.CreateDump(dumpFileName, cancellationToken); + using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) + await CreateDumpNoLock(cancellationToken); } /// @@ -758,8 +748,9 @@ namespace Tgstation.Server.Host.Components.Watchdog try { var sessionEventTask = relayToSession ? ((IEventConsumer)this).HandleEvent(eventType, parameters, false, cancellationToken) : ValueTask.CompletedTask; + var eventConsumerTask = eventConsumer.HandleEvent(eventType, parameters, false, cancellationToken); await ValueTaskExtensions.WhenAll( - eventConsumer.HandleEvent(eventType, parameters, false, cancellationToken), + eventConsumerTask, sessionEventTask); } catch (JobException ex) @@ -1142,7 +1133,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogDebug("DumpOnHealthCheckRestart enabled."); try { - await CreateDump(cancellationToken); + await CreateDumpNoLock(cancellationToken); } catch (JobException ex) { @@ -1192,5 +1183,34 @@ namespace Tgstation.Server.Host.Components.Watchdog .Where(nullableChannelId => nullableChannelId.HasValue) .Select(nullableChannelId => nullableChannelId.Value)); } + + /// + /// Attempt to create a process dump for the game server. Requires a lock on . + /// + /// The for the operation. + /// A representing the running operation. + async ValueTask CreateDumpNoLock(CancellationToken cancellationToken) + { + const string DumpDirectory = "ProcessDumps"; + var dumpFileNameTemplate = diagnosticsIOManager.ResolvePath( + diagnosticsIOManager.ConcatPath( + DumpDirectory, + $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); + + var dumpFileName = dumpFileNameTemplate; + var iteration = 0; + while (await diagnosticsIOManager.FileExists(dumpFileName, cancellationToken)) + dumpFileName = $"{dumpFileNameTemplate} ({++iteration})"; + + if (iteration == 0) + await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); + + var session = GetActiveController(); + if (session?.Lifetime.IsCompleted != false) + throw new JobException(ErrorCode.GameServerOffline); + + Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); + await session.CreateDump(dumpFileName, cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs index aa798cfed3..c317fc1b97 100644 --- a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs @@ -62,7 +62,9 @@ namespace Tgstation.Server.Host.Configuration return platformIdentifier.IsWindows ? ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.CommonApplicationData, + Environment.SpecialFolderOption.DoNotVerify), assemblyInformationProvider.VersionPrefix, "logs") : ioManager.ConcatPath( diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index f33ef79b11..8e2fe59579 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -145,6 +145,11 @@ namespace Tgstation.Server.Host.Configuration /// public string OpenDreamGitTagPrefix { get; set; } = DefaultOpenDreamGitTagPrefix; + /// + /// If the dotnet output of creating an OpenDream installation should be suppressed. Known to cause issues in CI. + /// + public bool OpenDreamSuppressInstallOutput { get; set; } + /// /// Initializes a new instance of the class. /// diff --git a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs index 08d8f74dad..d69947d4ba 100644 --- a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs +++ b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; @@ -15,6 +14,7 @@ using Microsoft.Net.Http.Headers; using Tgstation.Server.Api; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; namespace Tgstation.Server.Host.Controllers { @@ -136,22 +136,11 @@ namespace Tgstation.Server.Host.Controllers if (Request.Headers.ContainsKey(FetchChannelVaryHeader)) return GetChannelJson(); - var fileInfo = hostEnvironment.WebRootFileProvider.GetFileInfo(appRoute); - if (fileInfo.Exists) - { - logger.LogTrace("Serving static file \"{filename}\"...", appRoute); - var contentTypeProvider = new FileExtensionContentTypeProvider(); - if (!contentTypeProvider.TryGetContentType(fileInfo.Name, out var contentType)) - contentType = MediaTypeNames.Application.Octet; - else if (contentType == MediaTypeNames.Application.Json) - Response.Headers.Add( - HeaderNames.CacheControl, - new StringValues(new[] { "public", "max-age=31536000", "immutable" })); + var foundFile = this.TryServeFile(hostEnvironment, logger, appRoute); + if (foundFile != null) + return foundFile; - return File(appRoute, contentType); - } - else - logger.LogTrace("Requested static file \"{filename}\" does not exist! Redirecting to index...", appRoute); + logger.LogTrace("Requested static file \"{filename}\" does not exist! Redirecting to index...", appRoute); return File("index.html", MediaTypeNames.Text.Html); } diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 5a3ff51f0e..ca84dac467 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -106,7 +106,7 @@ namespace Tgstation.Server.Host.Controllers [TgsAuthorize(DreamDaemonRights.ReadMetadata | DreamDaemonRights.ReadRevision)] [ProducesResponseType(typeof(DreamDaemonResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 410)] - public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, cancellationToken); + public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, false, cancellationToken); /// /// Stops the Watchdog if it's running. @@ -231,7 +231,7 @@ namespace Tgstation.Server.Host.Controllers // run this second because current may be modified by it // slight race condition with request cancellation, but I CANNOT be assed right now - await watchdog.ChangeSettings(current, cancellationToken); + var rebootRequired = await watchdog.ChangeSettings(current, cancellationToken); var rebootState = watchdog.RebootState; var oldSoftRestart = rebootState == RebootState.Restart; @@ -243,7 +243,7 @@ namespace Tgstation.Server.Host.Controllers else if ((oldSoftRestart && model.SoftRestart == false) || (oldSoftShutdown && model.SoftShutdown == false)) await watchdog.ResetRebootState(cancellationToken); - return await ReadImpl(current, cancellationToken); + return await ReadImpl(current, rebootRequired, cancellationToken); }); } #pragma warning restore CA1506 @@ -305,9 +305,10 @@ namespace Tgstation.Server.Host.Controllers /// Implementation of . /// /// The to operate on if any. + /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. - ValueTask ReadImpl(DreamDaemonSettings settings, CancellationToken cancellationToken) + ValueTask ReadImpl(DreamDaemonSettings settings, bool knownForcedReboot, CancellationToken cancellationToken) => WithComponentInstance(async instance => { var dd = instance.Watchdog; @@ -360,6 +361,10 @@ namespace Tgstation.Server.Host.Controllers result.Visibility = settings.Visibility.Value; result.SoftRestart = rstate == RebootState.Restart; result.SoftShutdown = rstate == RebootState.Shutdown; + + if (rstate == RebootState.Normal && knownForcedReboot) + result.SoftRestart = true; + result.StartupTimeout = settings.StartupTimeout.Value; result.HealthCheckSeconds = settings.HealthCheckSeconds.Value; result.DumpOnHealthCheckRestart = settings.DumpOnHealthCheckRestart.Value; diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index 850785d612..3ccca028fe 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -2,10 +2,13 @@ using System.Collections.Generic; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -19,14 +22,14 @@ namespace Tgstation.Server.Host.Controllers public sealed class RootController : Controller { /// - /// The route to the TGS logo .svg in the on Windows. + /// The name of the TGS logo .svg in the on Windows. /// - public const string ProjectLogoSvgRouteWindows = "/0176d5d8b7d307f158e0.svg"; + const string LogoSvgWindowsName = "0176d5d8b7d307f158e0"; /// - /// The route to the TGS logo .svg in the on Linux. + /// The name of the TGS logo .svg in the on Linux. /// - public const string ProjectLogoSvgRouteLinux = "/b5616c99bf2052a6bbd7.svg"; + const string LogoSvgLinuxName = "b5616c99bf2052a6bbd7"; /// /// The for the . @@ -38,6 +41,16 @@ namespace Tgstation.Server.Host.Controllers /// readonly IPlatformIdentifier platformIdentifier; + /// + /// THe for the . + /// + readonly IWebHostEnvironment hostEnvironment; + + /// + /// The for the . + /// + readonly ILogger logger; + /// /// The for the . /// @@ -53,16 +66,22 @@ namespace Tgstation.Server.Host.Controllers /// /// The value of . /// The value of . + /// The value of . + /// The value of . /// The containing the value of . /// The containing the value of . public RootController( IAssemblyInformationProvider assemblyInformationProvider, IPlatformIdentifier platformIdentifier, + IWebHostEnvironment hostEnvironment, + ILogger logger, IOptions generalConfigurationOptions, IOptions controlPanelConfigurationOptions) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.hostEnvironment = hostEnvironment ?? throw new ArgumentNullException(nameof(hostEnvironment)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); controlPanelConfiguration = controlPanelConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(controlPanelConfigurationOptions)); } @@ -98,13 +117,24 @@ namespace Tgstation.Server.Host.Controllers var model = new { Links = links, - Svg = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- - ? ProjectLogoSvgRouteWindows - : ProjectLogoSvgRouteLinux, Title = assemblyInformationProvider.VersionString, }; return View(model); } + + /// + /// Retrieve the logo .svg for the webpanel. + /// + /// The appropriate . + [HttpGet("logo.svg")] + public IActionResult GetLogo() + { + var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- + ? LogoSvgWindowsName + : LogoSvgLinuxName; + + return (IActionResult)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg") ?? NotFound(); + } } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 3ce52f39be..10f2728465 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -367,7 +367,9 @@ namespace Tgstation.Server.Host.Core // only global repo manager should be for the OD repo var openDreamRepositoryDirectory = ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify), assemblyInformationProvider.VersionPrefix, "OpenDreamRepository"); services.AddSingleton( diff --git a/src/Tgstation.Server.Host/Database/DatabaseContext.cs b/src/Tgstation.Server.Host/Database/DatabaseContext.cs index cca6313b53..3dd14560f6 100644 --- a/src/Tgstation.Server.Host/Database/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Database/DatabaseContext.cs @@ -245,11 +245,8 @@ namespace Tgstation.Server.Host.Database const string ConfigureMethodName = nameof(SqlServerDatabaseContext.ConfigureWith); var configureFunction = typeof(TDatabaseContext).GetMethod( ConfigureMethodName, - BindingFlags.Public | BindingFlags.Static); - - if (configureFunction == null) - throw new InvalidOperationException($"Context type {typeof(TDatabaseContext).FullName} missing static {ConfigureMethodName} function!"); - + BindingFlags.Public | BindingFlags.Static) + ?? throw new InvalidOperationException($"Context type {typeof(TDatabaseContext).FullName} missing static {ConfigureMethodName} function!"); return (optionsBuilder, config) => configureFunction.Invoke(null, new object[] { optionsBuilder, config }); } diff --git a/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs b/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs index 9ab3b1cc87..2d1d8119a0 100644 --- a/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs @@ -1,7 +1,13 @@ using System; using System.Net; +using System.Net.Mime; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; @@ -30,5 +36,38 @@ namespace Tgstation.Server.Host.Extensions /// A with the given . public static ObjectResult StatusCode(this ControllerBase controller, HttpStatusCode statusCode, object errorMessage) => controller?.StatusCode((int)statusCode, errorMessage) ?? throw new ArgumentNullException(nameof(controller)); + + /// + /// Try to serve a given file . + /// + /// The . + /// The . + /// The . + /// The path to the file in the 'wwwroot'. + /// A if the file was found. otherwise. + public static VirtualFileResult TryServeFile(this ControllerBase controller, IWebHostEnvironment hostEnvironment, ILogger logger, string path) + { + ArgumentNullException.ThrowIfNull(controller); + ArgumentNullException.ThrowIfNull(hostEnvironment); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(path); + + var fileInfo = hostEnvironment.WebRootFileProvider.GetFileInfo(path); + if (fileInfo.Exists) + { + logger.LogTrace("Serving static file \"{filename}\"...", path); + var contentTypeProvider = new FileExtensionContentTypeProvider(); + if (!contentTypeProvider.TryGetContentType(fileInfo.Name, out var contentType)) + contentType = MediaTypeNames.Application.Octet; + else if (contentType == MediaTypeNames.Application.Json) + controller.Response.Headers.Add( + HeaderNames.CacheControl, + new StringValues(new[] { "public", "max-age=31536000", "immutable" })); + + return controller.File(path, contentType); + } + + return null; + } } } diff --git a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs new file mode 100644 index 0000000000..e1b47d644a --- /dev/null +++ b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs @@ -0,0 +1,91 @@ +using System; +using System.Threading; + +using LibGit2Sharp; +using LibGit2Sharp.Handlers; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Jobs; + +namespace Tgstation.Server.Host.Extensions +{ + /// + /// Extension methods for . + /// + static class FetchOptionsExtensions + { + /// + /// Hydrate a given set of . + /// + /// The to hydrate. + /// The for the operation. + /// The optional . + /// The optional . + /// The for the operation. + /// The hydrated . + public static FetchOptions Hydrate( + this FetchOptions fetchOptions, + ILogger logger, + JobProgressReporter progressReporter, + CredentialsHandler credentialsHandler, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(fetchOptions); + ArgumentNullException.ThrowIfNull(logger); + + fetchOptions.OnProgress = _ => !cancellationToken.IsCancellationRequested; + fetchOptions.OnTransferProgress = transferProgress => + { + if (progressReporter != null) + { + var percentage = ((double)transferProgress.IndexedObjects + transferProgress.ReceivedObjects) / (transferProgress.TotalObjects * 2); + progressReporter.ReportProgress(percentage); + } + + return !cancellationToken.IsCancellationRequested; + }; + fetchOptions.OnUpdateTips = (_, _, _) => !cancellationToken.IsCancellationRequested; + fetchOptions.CredentialsProvider = credentialsHandler; + fetchOptions.RepositoryOperationStarting = _ => !cancellationToken.IsCancellationRequested; + fetchOptions.OnTransferProgress = TransferProgressHandler( + logger, + progressReporter, + cancellationToken); + + return fetchOptions; + } + + /// + /// Generate a from a given and . + /// + /// The for the operation. + /// The optional of the operation. + /// The for the operation. + /// A new based on . + static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter progressReporter, CancellationToken cancellationToken) => transferProgress => + { + double? percentage; + var totalObjectsToProcess = transferProgress.TotalObjects * 2; + var processedObjects = transferProgress.IndexedObjects + transferProgress.ReceivedObjects; + if (totalObjectsToProcess < processedObjects || totalObjectsToProcess == 0) + percentage = null; + else + { + percentage = (double)processedObjects / totalObjectsToProcess; + if (percentage < 0) + percentage = null; + } + + if (percentage == null) + logger.LogDebug( + "Bad transfer progress values (Please tell Cyberboss)! Indexed: {indexed}, Received: {received}, Total: {total}", + transferProgress.IndexedObjects, + transferProgress.ReceivedObjects, + transferProgress.TotalObjects); + + progressReporter?.ReportProgress(percentage); + return !cancellationToken.IsCancellationRequested; + }; + } +} diff --git a/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs b/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs new file mode 100644 index 0000000000..07cd38bd1f --- /dev/null +++ b/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs @@ -0,0 +1,68 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +using Byond.TopicSender; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Extensions +{ + /// + /// Extension methods for . + /// + static class TopicClientExtensions + { + /// + /// Send a with optional repeated priority. + /// + /// The to send with. + /// The to use for delayed retries if an error occurs. + /// The to write to. + /// The to send. + /// The local port to send the topic to. + /// If priority retries should be used. + /// The for the operation. + /// A resulting in the on success, on failure. + public static async ValueTask SendWithOptionalPriority( + this ITopicClient topicClient, + IAsyncDelayer delayer, + ILogger logger, + string queryString, + ushort port, + bool priority, + CancellationToken cancellationToken) + { + const int PrioritySendAttempts = 5; + var endpoint = new IPEndPoint(IPAddress.Loopback, port); + var firstSend = true; + + for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) + try + { + firstSend = false; + + logger.LogTrace("Begin topic request"); + var byondResponse = await topicClient.SendTopic( + endpoint, + queryString, + cancellationToken); + + logger.LogTrace("End topic request"); + return byondResponse; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); + + if (priority && i > 0) + await delayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + + return null; + } + } +} diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 87387c60a9..abdb26170c 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Serilog.Context; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Common.Extensions; @@ -452,7 +453,12 @@ namespace Tgstation.Server.Host.Jobs } } - var instanceCoreProvider = await activationTcs.Task.WaitAsync(cancellationToken); + var activationTask = activationTcs.Task; + + Debug.Assert(activationTask.IsCompleted || job.JobCode.Value.IsServerStartupJob(), "Non-server startup job registered before activation!"); + + var instanceCoreProvider = await activationTask.WaitAsync(cancellationToken); + QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); diff --git a/src/Tgstation.Server.Host/Properties/launchSettings.json b/src/Tgstation.Server.Host/Properties/launchSettings.json index 52ba379ea3..b93f29050d 100644 --- a/src/Tgstation.Server.Host/Properties/launchSettings.json +++ b/src/Tgstation.Server.Host/Properties/launchSettings.json @@ -7,9 +7,15 @@ "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Docker": { - "commandName": "Docker", - "publishAllPorts": true + "WSL": { + "commandName": "WSL2", + "distributionName": "Ubuntu", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development", + "ASPNETCORE_ENVIRONMENT": "Development", + "Database__ConnectionString": "Data Source=192.168.2.16,1433;Initial Catalog=TGS_Linux;User Id=tgs_debug;Password=asdf;Encrypt=False;Application Name=tgstation-server", + "General__ValidInstancePaths__0": "/home/dominion/tgs_debug_pen" + } } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index b9671572b2..79f30c22b9 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -600,6 +600,8 @@ namespace Tgstation.Server.Host.Setup }; CreateTestConnection(csb.ConnectionString); + + csb.Mode = SqliteOpenMode.ReadWriteCreate; databaseConfiguration.ConnectionString = csb.ConnectionString; } diff --git a/src/Tgstation.Server.Host/System/IProcess.cs b/src/Tgstation.Server.Host/System/IProcess.cs index 8df102b2b2..dc9a1d5eac 100644 --- a/src/Tgstation.Server.Host/System/IProcess.cs +++ b/src/Tgstation.Server.Host/System/IProcess.cs @@ -23,13 +23,12 @@ namespace Tgstation.Server.Host.System /// Get the stderr and stdout output of the . /// /// The for the operation. - /// A resulting in the stderr and stdout output of the . + /// A resulting in the stderr and stdout output of the . /// /// To guarantee that all data is received from the when redirecting streams to a file /// the result of this function must be ed before is called. - /// May call internally if the process has exited. /// - ValueTask GetCombinedOutput(CancellationToken cancellationToken); + Task GetCombinedOutput(CancellationToken cancellationToken); /// /// Asycnhronously terminates the process. diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index da67c8f0e8..266e220708 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -7,7 +7,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -28,11 +27,6 @@ namespace Tgstation.Server.Host.System /// readonly IProcessFeatures processFeatures; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -68,7 +62,6 @@ namespace Tgstation.Server.Host.System /// Initializes a new instance of the class. /// /// The value of . - /// The value of . /// The value of . /// The override value of . /// The value of . @@ -76,7 +69,6 @@ namespace Tgstation.Server.Host.System /// If was NOT just created. public Process( IProcessFeatures processFeatures, - IAsyncDelayer asyncDelayer, global::System.Diagnostics.Process handle, CancellationTokenSource readerCts, Task readTask, @@ -92,7 +84,6 @@ namespace Tgstation.Server.Host.System cancellationTokenSource = readerCts ?? new CancellationTokenSource(); this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.readTask = readTask; @@ -128,7 +119,7 @@ namespace Tgstation.Server.Host.System /// public async ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref disposed, 1) == 1) + if (Interlocked.Exchange(ref disposed, 1) != 0) return; logger.LogTrace("Disposing PID {pid}...", Id); @@ -144,30 +135,18 @@ namespace Tgstation.Server.Host.System } /// - public async ValueTask GetCombinedOutput(CancellationToken cancellationToken) + public Task GetCombinedOutput(CancellationToken cancellationToken) { if (readTask == null) throw new InvalidOperationException("Output/Error stream reading was not enabled!"); - // workaround for https://github.com/dotnet/runtime/issues/28583 (?) - if (handle.HasExited) - { - handle.WaitForExit(); - await Task.WhenAny(readTask, asyncDelayer.Delay(TimeSpan.FromSeconds(30), cancellationToken)); - - if (!readTask.IsCompleted) - { - logger.LogWarning("Detected process output read hang on PID {pid}! Closing handle as a workaround...", Id); - await DisposeAsync(); - } - } - - return await readTask.WaitAsync(cancellationToken); + return readTask.WaitAsync(cancellationToken); } /// public void Terminate() { + CheckDisposed(); if (handle.HasExited) { logger.LogTrace("PID {pid} already exited", Id); @@ -190,6 +169,7 @@ namespace Tgstation.Server.Host.System /// public void AdjustPriority(bool higher) { + CheckDisposed(); var targetPriority = higher ? ProcessPriorityClass.AboveNormal : ProcessPriorityClass.BelowNormal; try { @@ -205,6 +185,7 @@ namespace Tgstation.Server.Host.System /// public void Suspend() { + CheckDisposed(); try { processFeatures.SuspendProcess(handle); @@ -220,6 +201,7 @@ namespace Tgstation.Server.Host.System /// public void Resume() { + CheckDisposed(); try { processFeatures.ResumeProcess(handle); @@ -235,6 +217,7 @@ namespace Tgstation.Server.Host.System /// public string GetExecutingUsername() { + CheckDisposed(); var result = processFeatures.GetExecutingUsername(handle); logger.LogTrace("PID {pid} Username: {username}", Id, result); return result; @@ -244,6 +227,7 @@ namespace Tgstation.Server.Host.System public ValueTask CreateDump(string outputFile, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(outputFile); + CheckDisposed(); logger.LogTrace("Dumping PID {pid} to {dumpFilePath}...", Id, outputFile); return processFeatures.CreateDump(handle, outputFile, cancellationToken); @@ -255,18 +239,29 @@ namespace Tgstation.Server.Host.System /// A resulting in the or if the process was detached. async Task WrapLifetimeTask() { + bool hasExited; try { await handle.WaitForExitAsync(cancellationTokenSource.Token); - var exitCode = handle.ExitCode; - logger.LogTrace("PID {pid} exited with code {exitCode}", Id, exitCode); - return exitCode; + hasExited = true; } catch (OperationCanceledException ex) { logger.LogTrace(ex, "Process lifetime task cancelled!"); - return null; + hasExited = handle.HasExited; } + + if (!hasExited) + return null; + + var exitCode = handle.ExitCode; + logger.LogTrace("PID {pid} exited with code {exitCode}", Id, exitCode); + return exitCode; } + + /// + /// Throws an if a method of the was called after . + /// + void CheckDisposed() => ObjectDisposedException.ThrowIf(disposed != 0, this); } } diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index f6218f5cd9..8df021f795 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,13 +1,14 @@ using System; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -24,11 +25,6 @@ namespace Tgstation.Server.Host.System /// readonly IProcessFeatures processFeatures; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -65,19 +61,16 @@ namespace Tgstation.Server.Host.System /// Initializes a new instance of the class. /// /// The value of . - /// The value of . /// The value of . /// The value of . /// The value of . public ProcessExecutor( IProcessFeatures processFeatures, - IAsyncDelayer asyncDelayer, IIOManager ioManager, ILogger logger, ILoggerFactory loggerFactory) { this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); @@ -122,16 +115,19 @@ namespace Tgstation.Server.Host.System ArgumentNullException.ThrowIfNull(workingDirectory); ArgumentNullException.ThrowIfNull(arguments); - if (!noShellExecute && readStandardHandles) - throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!"); - - logger.LogDebug( - noShellExecute - ? "Launching process in {workingDirectory}: {exe} {arguments}" - : "Shell launching process in {workingDirectory}: {exe} {arguments}", + if (noShellExecute) + logger.LogDebug( + "Launching process in {workingDirectory}: {exe} {arguments}", workingDirectory, fileName, arguments); + else + logger.LogDebug( + "Shell launching process in {workingDirectory}: {exe} {arguments}", + workingDirectory, + fileName, + arguments); + var handle = new global::System.Diagnostics.Process(); try { @@ -145,17 +141,15 @@ namespace Tgstation.Server.Host.System CancellationTokenSource disposeCts = null; try { - TaskCompletionSource processStartTcs = null; + TaskCompletionSource processStartTcs = null; if (readStandardHandles) { - processStartTcs = new TaskCompletionSource(); - handle.StartInfo.RedirectStandardOutput = true; - handle.StartInfo.RedirectStandardError = true; - + processStartTcs = new TaskCompletionSource(); disposeCts = new CancellationTokenSource(); readTask = ConsumeReaders(handle, processStartTcs.Task, fileRedirect, disposeCts.Token); } + int pid; try { ExclusiveProcessLaunchLock.EnterReadLock(); @@ -168,7 +162,8 @@ namespace Tgstation.Server.Host.System ExclusiveProcessLaunchLock.ExitReadLock(); } - processStartTcs?.SetResult(); + pid = handle.Id; + processStartTcs?.SetResult(pid); } catch (Exception ex) { @@ -178,7 +173,6 @@ namespace Tgstation.Server.Host.System var process = new Process( processFeatures, - asyncDelayer, handle, disposeCts, readTask, @@ -225,76 +219,102 @@ namespace Tgstation.Server.Host.System /// Consume the stdout/stderr streams into a . /// /// The . - /// The that completes when starts. + /// The resulting in the of the started process. /// The optional path to redirect the streams to. - /// The that triggers when the is disposed. + /// The for the operation. /// A resulting in the program's output/error text if is , otherwise. - async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startTask, string fileRedirect, CancellationToken disposeToken) + async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startupAndPid, string fileRedirect, CancellationToken cancellationToken) { - await startTask; + handle.StartInfo.RedirectStandardOutput = true; + handle.StartInfo.RedirectStandardError = true; - var pid = handle.Id; - logger.LogTrace("Starting read for PID {pid}...", pid); - - // once we obtain these handles we're responsible for them - using var stdOutHandle = handle.StandardOutput; - using var stdErrHandle = handle.StandardError; - Task outputReadTask = null, errorReadTask = null; - bool outputOpen = true, errorOpen = true; - async Task GetNextLine() - { - if (outputOpen && outputReadTask == null) - outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); - - if (errorOpen && errorReadTask == null) - errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); - - var completedTask = await Task.WhenAny(outputReadTask ?? errorReadTask, errorReadTask ?? outputReadTask); - var line = await completedTask.WaitAsync(disposeToken); - if (completedTask == outputReadTask) - { - outputReadTask = null; - if (line == null) - outputOpen = false; - } - else - { - errorReadTask = null; - if (line == null) - errorOpen = false; - } - - if (line == null && (errorOpen || outputOpen)) - return await GetNextLine(); - - return line; - } - - await using var fileStream = fileRedirect != null ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; + bool writingToFile; + await using var fileStream = (writingToFile = fileRedirect != null) ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; await using var writer = fileStream != null ? new StreamWriter(fileStream) : null; - string text; var stringBuilder = fileStream == null ? new StringBuilder() : null; - try - { - while ((text = await GetNextLine()) != null) + + var dataChannel = Channel.CreateUnbounded( + new UnboundedChannelOptions { - if (fileStream != null) - { - await writer.WriteLineAsync(text.AsMemory(), disposeToken); - await writer.FlushAsync(disposeToken); - } - else - stringBuilder.AppendLine(text); + AllowSynchronousContinuations = !writingToFile, + SingleReader = true, + SingleWriter = false, + }); + + var handlesOpen = 2; + async void DataReceivedHandler(object sender, DataReceivedEventArgs eventArgs) + { + var line = eventArgs.Data; + if (line == null) + { + var handlesRemaining = Interlocked.Decrement(ref handlesOpen); + if (handlesRemaining == 0) + dataChannel.Writer.Complete(); + + return; } - logger.LogTrace("Finished read for PID {pid}", pid); + try + { + await dataChannel.Writer.WriteAsync(line, cancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogWarning(ex, "Handle channel write interrupted!"); + } } - catch (OperationCanceledException ex) + + handle.OutputDataReceived += DataReceivedHandler; + handle.ErrorDataReceived += DataReceivedHandler; + + async ValueTask OutputWriter() { - logger.LogWarning(ex, "PID {pid} stream reading interrupted!", pid); - if (fileStream != null) - await writer.WriteLineAsync("-- Process detached, log truncated. This is likely due a to TGS restart --"); + var enumerable = dataChannel.Reader.ReadAllAsync(cancellationToken); + if (writingToFile) + { + var enumerator = enumerable.GetAsyncEnumerator(cancellationToken); + var nextEnumeration = enumerator.MoveNextAsync(); + while (await nextEnumeration) + { + var text = enumerator.Current; + nextEnumeration = enumerator.MoveNextAsync(); + await writer.WriteLineAsync(text.AsMemory(), cancellationToken); + + if (!nextEnumeration.IsCompleted) + await writer.FlushAsync(cancellationToken); + } + } + else + await foreach (var text in enumerable) + stringBuilder.AppendLine(text); + } + + var pid = await startupAndPid; + logger.LogTrace("Starting read for PID {pid}...", pid); + + using (cancellationToken.Register(() => dataChannel.Writer.TryComplete())) + { + handle.BeginOutputReadLine(); + using (cancellationToken.Register(handle.CancelOutputRead)) + { + handle.BeginErrorReadLine(); + using (cancellationToken.Register(handle.CancelErrorRead)) + { + try + { + await OutputWriter(); + + logger.LogTrace("Finished read for PID {pid}", pid); + } + catch (OperationCanceledException ex) + { + logger.LogWarning(ex, "PID {pid} stream reading interrupted!", pid); + if (fileStream != null) + await writer.WriteLineAsync("-- Process detached, log truncated. This is likely due a to TGS restart --"); + } + } + } } return stringBuilder?.ToString(); @@ -312,7 +332,6 @@ namespace Tgstation.Server.Host.System var pid = handle.Id; return new Process( processFeatures, - asyncDelayer, handle, null, null, diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9890be89d0..a47c87fbd2 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -63,7 +63,7 @@ - + @@ -73,7 +73,7 @@ - + @@ -95,7 +95,7 @@ - + @@ -107,7 +107,7 @@ - + @@ -125,7 +125,7 @@ - + diff --git a/src/Tgstation.Server.Host/Views/Root/Index.cshtml b/src/Tgstation.Server.Host/Views/Root/Index.cshtml index 209a6ad5ac..bc23093e87 100644 --- a/src/Tgstation.Server.Host/Views/Root/Index.cshtml +++ b/src/Tgstation.Server.Host/Views/Root/Index.cshtml @@ -1,5 +1,4 @@ @{ - var svgPath = Model.Svg; var title = Model.Title; @@ -13,7 +12,7 @@ - + @{ if (Model.Links != null) foreach (KeyValuePair kvp in Model.Links) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 60624d3617..56f87f2169 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -18,6 +18,7 @@ General: DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository + OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 0bff755db9..90164c7e80 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -6,9 +6,11 @@ log << "Initial value of sleep_offline: [sleep_offline]" sleep_offline = FALSE - // Intentionally slow down startup for testing purposes - for(var/i in 1 to 10000000) - dab() + if(params["slow_start"]) + // Intentionally slow down startup for health check testing purposes + for(var/i in 1 to 10000000) + dab() + TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE) var/sec = TgsSecurityLevel() @@ -189,7 +191,7 @@ var/run_bridge_test var/its_sad = data["im_out_of_memes"] if(its_sad) TestLegacyBridge() - return "yeah gimmie a sec" + return "all gucci" TgsChatBroadcast(new /datum/tgs_message_content("Received non-tgs topic: `[T]`")) @@ -252,9 +254,9 @@ var/received_health_check = FALSE /proc/RebootAsync() set waitfor = FALSE - world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 3 seconds")); + world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 1 seconds")); world.log << "About to sleep. sleep_offline: [world.sleep_offline]" - sleep(30) + sleep(10) world.log << "Done sleep, calling Reboot" world.Reboot() @@ -362,10 +364,6 @@ var/suppress_bridge_spam = FALSE api.access_identifier = old_ai /proc/TestLegacyBridge() - set waitfor = FALSE - - sleep(10) - var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) if(api.interop_version.suite != 5) FailTest("Legacy bridge test not required anymore?") diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs index bbf369efce..04e7cdb66b 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs @@ -1,4 +1,4 @@ -using LibGit2Sharp; +using LibGit2Sharp; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -39,12 +39,11 @@ namespace Tgstation.Server.Host.Components.Repository.Tests try { var factory = CreateFactory(); + var cloneOpts = new CloneOptions(); + cloneOpts.FetchOptions.CredentialsProvider = factory.GenerateCredentialsHandler(null, null); await factory.Clone( new Uri("https://github.com/Cyberboss/Test"), - new CloneOptions - { - CredentialsProvider = factory.GenerateCredentialsHandler(null, null) - }, + cloneOpts, tempDir, default); diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index 407145730c..083657a075 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -54,13 +54,12 @@ namespace Tgstation.Server.Host.System.Tests builder.SetMinimumLevel(LogLevel.Trace); }); - IProcessExecutor processExecutor = null; + ProcessExecutor processExecutor = null; processExecutor = new ProcessExecutor( new PosixProcessFeatures( new Lazy(() => processExecutor), new DefaultIOManager(), loggerFactory.CreateLogger()), - new AsyncDelayer(), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index bd7d01bbbd..de540b606d 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -94,7 +94,9 @@ namespace Tgstation.Server.Tests // actions is supposed to cache BYOND for us var dir = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + Environment.GetFolderPath( + Environment.SpecialFolder.UserProfile, + Environment.SpecialFolderOption.DoNotVerify), "byond-zips-cache", windows ? "windows" : "linux"); path = Path.Combine( diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 2edb4686ed..14cc51dd4d 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -44,6 +44,8 @@ namespace Tgstation.Server.Tests.Live ulong channelIdAllocator; + public static Task MessageGuard = Task.CompletedTask; + static IAsyncDelayer CreateMockDelayer() { // at time of writing, this is used exclusively for the reconnection interval which works in minutes @@ -219,6 +221,8 @@ namespace Tgstation.Server.Tests.Live var delay = random.Next(0, 10000); await Task.Delay(delay, cancellationToken); + await MessageGuard; + // %5 chance to disconnect randomly if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95) connected = false; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs index 0963b72b4f..abb699815e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index 8b5fab58c9..8cc7d0b6fb 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; @@ -138,10 +139,10 @@ namespace Tgstation.Server.Tests.Live.Instance var updatedDD = await dreamDaemonClient.Update(new DreamDaemonRequest { - StartupTimeout = 15, + StartupTimeout = 30, Port = ddPort }, cancellationToken); - Assert.AreEqual(15U, updatedDD.StartupTimeout); + Assert.AreEqual(30U, updatedDD.StartupTimeout); Assert.AreEqual(ddPort, updatedDD.Port); async Task CompileAfterByondInstall() diff --git a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs index 83e7f322c9..57c4255130 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs @@ -140,7 +140,7 @@ namespace Tgstation.Server.Tests.Live.Instance case EngineType.Byond: return 30; case EngineType.OpenDream: - return 300; + return 500; default: throw new InvalidOperationException($"Unknown engine type: {testVersion.Engine.Value}"); } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 255a2e8a83..a50d4a6504 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -45,11 +45,11 @@ namespace Tgstation.Server.Tests.Live.Instance CancellationToken cancellationToken) { var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); - var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); - var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); + await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); + await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); - var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); + await using var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); var byondTask = engineTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); @@ -66,15 +66,15 @@ namespace Tgstation.Server.Tests.Live.Instance await configTest.SetupDMApiTests(true, cancellationToken); await byondTask; - await new WatchdogTest( + await using var wdt = new WatchdogTest( testVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, - usingBasicWatchdog) - .Run(cancellationToken); + usingBasicWatchdog); + await wdt.Run(cancellationToken); } public static async ValueTask DownloadEngineVersion( @@ -85,7 +85,9 @@ namespace Tgstation.Server.Tests.Live.Instance { var ioManager = new DefaultIOManager(); var odRepoDir = ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify), new AssemblyInformationProvider().VersionPrefix, "OpenDreamRepository"); var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); @@ -197,7 +199,7 @@ namespace Tgstation.Server.Tests.Live.Instance ReconnectionInterval = 1, }, cancellationToken); - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); EngineInstallResponse installJob2; await using (var stableBytesMs = await TestingUtils.ExtractMemoryStreamFromInstallationData( @@ -219,7 +221,6 @@ namespace Tgstation.Server.Tests.Live.Instance await chatRequest; await Task.Yield(); - await Task.WhenAll( jrt.WaitForJob(installJob2.InstallJob, EngineTest.EngineInstallationTimeout(compatVersion) + 30, false, null, cancellationToken), jrt.WaitForJob(cloneRequest.Result.ActiveJob, 60, false, null, cancellationToken), @@ -257,7 +258,8 @@ namespace Tgstation.Server.Tests.Live.Instance await configSetupTask; - await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); + await using var wdt = new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog); + await wdt.Run(cancellationToken); await instanceManagerClient.Update(new InstanceUpdateRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index d9456c22bc..84985b6a9b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.SignalR.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; @@ -66,7 +67,7 @@ namespace Tgstation.Server.Tests.Live.Instance } catch(Exception ex) { - finishTcs.SetException(ex); + finishTcs.TrySetException(ex); } return Task.CompletedTask; @@ -84,7 +85,7 @@ namespace Tgstation.Server.Tests.Live.Instance } } - public async Task Run(CancellationToken cancellationToken) + public async Task Run(CancellationToken cancellationToken) { var neverReceiver = new ShouldNeverReceiveUpdates() { @@ -98,16 +99,33 @@ namespace Tgstation.Server.Tests.Live.Instance }, }; - await using (permedConn = (HubConnection)await permedUser.SubscribeToJobUpdates( + permedConn = (HubConnection)await permedUser.SubscribeToJobUpdates( this, null, null, - cancellationToken)) - await using (permlessConn = (HubConnection)await permlessUser.SubscribeToJobUpdates( - neverReceiver, - null, - null, - cancellationToken)) + cancellationToken); + + try + { + permlessConn = (HubConnection)await permlessUser.SubscribeToJobUpdates( + neverReceiver, + null, + null, + cancellationToken); + } + catch + { + await permedConn.DisposeAsync(); + throw; + } + + return FinishAsync(cancellationToken); + } + + async Task FinishAsync(CancellationToken cancellationToken) + { + await using (permedConn) + await using (permlessConn) { Console.WriteLine($"Initial conn1: {permedConn.ConnectionId}"); Console.WriteLine($"Initial conn2: {permlessConn.ConnectionId}"); @@ -138,7 +156,10 @@ namespace Tgstation.Server.Tests.Live.Instance Online = true, }, cancellationToken); - var jobs = await permedUser.Instances.CreateClient(instance).Jobs.List(null, cancellationToken); + var jobs = await permedUser.Instances.CreateClient(instance).Jobs.List(new PaginationSettings + { + PageSize = 100 + }, cancellationToken); if (wasOffline) await permedUser.Instances.Update(new InstanceUpdateRequest { @@ -153,6 +174,13 @@ namespace Tgstation.Server.Tests.Live.Instance .Select(CheckInstance); var allJobs = (await ValueTaskExtensions.WhenAll(allJobsTask, allInstances.Count)).SelectMany(x => x).ToList(); + + var groups = allJobs.GroupBy(x => x.Id.Value).ToList(); + var uniqueAllJobs = groups.Select(x => x.First()).ToList(); + + static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); + Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count, $"Duplicated Jobs:{Environment.NewLine}{JobListFormatter(groups.Where(x => x.Count() > 1).SelectMany(x => x))}"); + var missableMissedJobs = 0; foreach (var job in allJobs) { @@ -185,18 +213,24 @@ namespace Tgstation.Server.Tests.Live.Instance } else { - var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot - || job.JobCode == JobCode.StartupWatchdogLaunch - || job.JobCode == JobCode.StartupWatchdogReattach; - Assert.IsTrue(wasMissableJob); + var wasMissableJob = job.JobCode.Value.IsServerStartupJob(); + Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: #{job.Id.Value} - {job.JobCode} - {job.Description}"); ++missableMissedJobs; } } + var jobsSeenByHubButNotInAllJobs = seenJobs.Values.Where(x => !allJobs.Any(y => y.Id.Value == x.Id.Value)).ToList(); + // some instances may be detached, but our cache remains var accountedJobs = allJobs.Count - missableMissedJobs; - var accountedSeenJobs = seenJobs.Where(x => allInstances.Any(i => i.Id.Value == x.Value.InstanceId)).ToList(); - Assert.AreEqual(accountedJobs, accountedSeenJobs.Count, $"Mismatch in seen jobs:{Environment.NewLine}{String.Join(Environment.NewLine, allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)).Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}"))}"); + var errorMessage = $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(jobsSeenByHubButNotInAllJobs)}{Environment.NewLine}Current Instances: {String.Join(", ", allInstances.Select(i => i.Id.Value))}"; + Assert.AreEqual( + accountedJobs, + seenJobs.Count - jobsSeenByHubButNotInAllJobs.Count, + errorMessage); + Assert.IsTrue( + jobsSeenByHubButNotInAllJobs.All(job => job.JobCode.Value == JobCode.Move), + errorMessage); Assert.IsTrue(accountedJobs <= seenJobs.Count); Assert.AreNotEqual(0, permlessSeenJobs.Count); Assert.IsTrue(permlessSeenJobs.Count < seenJobs.Count); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs index bfe8c98d82..c6501e6766 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -13,16 +14,72 @@ using Tgstation.Server.Client.Components; namespace Tgstation.Server.Tests.Live.Instance { - class JobsRequiredTest + class JobsRequiredTest : IAsyncDisposable { protected IJobsClient JobsClient { get; } readonly IApiClient apiClient; + IAsyncDisposable hubConnection; + readonly Task hubConnectionTask; + readonly CancellationTokenSource cancellationTokenSource; + + readonly ConcurrentDictionary> registry; + public JobsRequiredTest(IJobsClient jobsClient) { JobsClient = jobsClient ?? throw new ArgumentNullException(nameof(jobsClient)); apiClient = (IApiClient)jobsClient.GetType().GetProperty("ApiClient", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(jobsClient); + registry = new ConcurrentDictionary>(); + + cancellationTokenSource = new CancellationTokenSource(); + hubConnectionTask = CreateHubConnection(); + } + + async Task CreateHubConnection() + { + var receiver = new JobReceiver + { + Callback = job => Register(job), + }; + + hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationTokenSource.Token); + } + + public async ValueTask DisposeAsync() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + try + { + await hubConnectionTask; + } + catch (OperationCanceledException) + { + } + + if (hubConnection != null) + await hubConnection.DisposeAsync(); + } + + Task Register(JobResponse updatedJob) + { + var tcs = registry.AddOrUpdate(updatedJob.Id.Value, + _ => + { + var tcs = new TaskCompletionSource(); + if (updatedJob.StoppedAt.HasValue) + tcs.SetResult(updatedJob); + return tcs; + }, + (_, oldTcs) => + { + if (updatedJob.StoppedAt.HasValue) + oldTcs.TrySetResult(updatedJob); + return oldTcs; + }); + + return tcs.Task; } class JobReceiver : IJobsHub @@ -42,47 +99,21 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(originalJob.JobCode); var job = originalJob; + var registryTask = Register(job); + await Task.WhenAny( + registryTask, + Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); + + if (!registryTask.IsCompleted) + // one last get in case SignalR dropped the ball + job = await JobsClient.GetId(job, cancellationToken); + else + job = await registryTask; + if (!job.StoppedAt.HasValue) { - var tcs = new TaskCompletionSource(); - var receiver = new JobReceiver - { - Callback = updatedJob => - { - if (updatedJob.Id != job.Id) - return; - - job = updatedJob; - if (updatedJob.StoppedAt.HasValue) - tcs.TrySetResult(); - }, - }; - - JobResponse firstCheck; - await using (var hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationToken)) - { - // initial GET after connecting - firstCheck = await JobsClient.GetId(job, cancellationToken); - if (!firstCheck.StoppedAt.HasValue) - { - firstCheck = null; - await Task.WhenAny( - tcs.Task, - Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); - } - } - - if (firstCheck != null) - job = firstCheck; - else if (!job.StoppedAt.HasValue) - // one last get in case SignalR dropped the ball - job = await JobsClient.GetId(job, cancellationToken); - - if (!job.StoppedAt.HasValue) - { - await JobsClient.Cancel(job, cancellationToken); - Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); - } + await JobsClient.Cancel(job, cancellationToken); + Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); } if (expectFailure.HasValue && expectFailure.Value ^ job.ExceptionDetails != null) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bcde5714ff..c2fa31ba31 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -1,6 +1,5 @@ using Byond.TopicSender; -using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -19,6 +18,7 @@ using System.Linq; using System.Net; using System.Net.Sockets; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using System.Threading; @@ -68,6 +68,7 @@ namespace Tgstation.Server.Tests.Live.Instance readonly bool watchdogRestartsProcess; bool ranTimeoutTest = false; + const string BaseAdditionalParameters = "expect_chat_channels=1&expect_static_files=1"; public WatchdogTest(EngineVersion testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool watchdogRestartsProcess) : base(instanceClient.Jobs) @@ -88,8 +89,20 @@ namespace Tgstation.Server.Tests.Live.Instance DisconnectTimeout = TimeSpan.FromSeconds(30) }, loggerFactory.CreateLogger($"WatchdogTest.TopicClient.{instanceClient.Metadata.Name}")); } - public async Task Run(CancellationToken cancellationToken) + { + try + { + await RunInt(cancellationToken); + } + catch + { + System.Console.WriteLine($"WATCHDOG TEST FAILING INSTANCE ID {instanceClient.Metadata.Id.Value}"); + throw; + } + } + + async Task RunInt(CancellationToken cancellationToken) { System.Console.WriteLine($"TEST: START WATCHDOG TESTS {instanceClient.Metadata.Name}"); @@ -120,12 +133,12 @@ namespace Tgstation.Server.Tests.Live.Instance // Increase startup timeout, disable heartbeats, enable map threads because we've tested without for years instanceClient.DreamDaemon.Update(new DreamDaemonRequest { - StartupTimeout = 15, + StartupTimeout = 30, HealthCheckSeconds = 0, Port = ddPort, MapThreads = 2, LogOutput = false, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1" + AdditionalParameters = BaseAdditionalParameters }, cancellationToken).AsTask(), CheckByondVersions(), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -152,9 +165,21 @@ namespace Tgstation.Server.Tests.Live.Instance await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); + // no chatty bullshit while we test health checks + var tcs = new TaskCompletionSource(); + var oldTask = Interlocked.Exchange(ref DummyChatProvider.MessageGuard, tcs.Task); + await RunHealthCheckTest(true, cancellationToken); await RunHealthCheckTest(false, cancellationToken); + async void Cleanup() + { + await oldTask; + tcs.SetResult(); + } + + Cleanup(); + await InteropTestsForLongRunningDme(cancellationToken); await instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -176,7 +201,6 @@ namespace Tgstation.Server.Tests.Live.Instance var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest { SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1", }, cancellationToken); var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); await ddUpdateTask; @@ -197,7 +221,9 @@ namespace Tgstation.Server.Tests.Live.Instance // reimplement TellWorldToReboot because it expects a new deployment and we don't care System.Console.WriteLine("TEST: Hack world reboot topic..."); - var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", FindTopicPort(), cancellationToken); + var result = await SendTestTopic( + "tgs_integration_test_special_tactics=1", + cancellationToken); Assert.AreEqual("ack", result.StringData); using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -223,13 +249,38 @@ namespace Tgstation.Server.Tests.Live.Instance await RunTest(false); } + ValueTask SendTestTopic(string queryString, CancellationToken cancellationToken) + => SendTestTopic(queryString, topicClient, instanceManager.GetInstanceReference(instanceClient.Metadata), FindTopicPort(), cancellationToken); + + public static async ValueTask SendTestTopic(string queryString, ITopicClient topicClient, IInstanceReference instanceReference, ushort topicPort, CancellationToken cancellationToken) + { + using (instanceReference) + { + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + + var watchdog = instanceReference?.Watchdog; + var session = (SessionController)watchdog?.GetType().GetMethod("GetActiveController", BindingFlags.Instance | BindingFlags.NonPublic)?.Invoke(watchdog, null); + + using (session != null + ? await session.TopicSendSemaphore.Lock(cancellationToken) + : null) + return await topicClient.SendWithOptionalPriority( + new AsyncDelayer(), + loggerFactory.CreateLogger(), + queryString, + topicPort, + true, + cancellationToken); + } + } + async ValueTask BroadcastTest(CancellationToken cancellationToken) { - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics_broadcast=1", - FindTopicPort(), - cancellationToken); + var topicRequestResult = await SendTestTopic("tgs_integration_test_tactics_broadcast=1", cancellationToken); Assert.IsNotNull(topicRequestResult); Assert.AreEqual("!!NULL!!", topicRequestResult.StringData); @@ -240,10 +291,8 @@ namespace Tgstation.Server.Tests.Live.Instance BroadcastMessage = TestBroadcastMessage, }, cancellationToken); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics_broadcast=1", - FindTopicPort(), + topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics_broadcast=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -294,34 +343,34 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask RegressionTest1550(CancellationToken cancellationToken) { + // Previous test, StartAndLeaveRunning, has SoftRestart set. We don't want that. + var restartJob = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob, 10, false, null, cancellationToken); + // we need to cycle deployments twice because TGS holds the initial deployment var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); Assert.IsNotNull(currentStatus.StagedCompileJob); - ValidateSessionId(currentStatus, false); + ValidateSessionId(currentStatus, true); var expectedStaged = currentStatus.StagedCompileJob; Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); - Assert.IsFalse(currentStatus.SoftShutdown.Value); + Assert.IsFalse(currentStatus.SoftShutdown); currentStatus = await TellWorldToReboot(true, cancellationToken); - ValidateSessionId(currentStatus, watchdogRestartsProcess); + ValidateSessionId(currentStatus, true); Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"shadow_wizard_money_gang=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "shadow_wizard_money_gang=1", cancellationToken); Assert.IsNotNull(topicRequestResult); Assert.AreEqual("we love casting spells", topicRequestResult.StringData); currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); + Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); ValidateSessionId(currentStatus, false); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); @@ -443,10 +492,8 @@ namespace Tgstation.Server.Tests.Live.Instance async Task SendChatOverloadCommand(CancellationToken cancellationToken) { // for the code coverage really... - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics5=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics5=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -511,11 +558,8 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsTrue(job.ErrorCode == ErrorCode.GameServerOffline || job.ErrorCode == ErrorCode.GCoreFailure, $"{job.ErrorCode}: {job.ExceptionDetails}"); - await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); - - var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - ValidateSessionId(ddStatus, true); - Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value); + var restartJob2 = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob2, 20, false, null, cancellationToken); } async Task TestDMApiFreeDeploy(CancellationToken cancellationToken) @@ -537,7 +581,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); @@ -608,11 +652,11 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); - await GracefulWatchdogShutdown(60, cancellationToken); + await GracefulWatchdogShutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); @@ -662,7 +706,9 @@ namespace Tgstation.Server.Tests.Live.Instance var pid = proc.Id; var foundLivePath = false; var allPaths = new List(); - foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) + + Assert.IsFalse(proc.HasExited); + foreach (var fd in Directory.GetFiles($"/proc/{pid}/fd")) { var sb = new StringBuilder(UInt16.MaxValue); if (Syscall.readlink(fd, sb) == -1) @@ -691,26 +737,21 @@ namespace Tgstation.Server.Tests.Live.Instance { System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST"); - // Check reverse mapping - var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - DumpOnHealthCheckRestart = !checkDump, - }, cancellationToken); - // enable health checks - status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { HealthCheckSeconds = 1, DumpOnHealthCheckRestart = checkDump, }, cancellationToken); Assert.AreEqual(checkDump, status.DumpOnHealthCheckRestart); + Assert.AreEqual(1U, status.HealthCheckSeconds.Value); var startJob = await StartDD(cancellationToken); await WaitForJob(startJob, 40, false, null, cancellationToken); - CheckDDPriority(); + await CheckDDPriority(); // lock on to DD and pause it so it can't health check var ddProcs = TestLiveServer.GetEngineServerProcessesOnPort(testVersion.Engine.Value, ddPort).Where(x => !x.HasExited).ToList(); @@ -723,7 +764,6 @@ namespace Tgstation.Server.Tests.Live.Instance RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures(new Lazy(() => executor), Mock.Of(), Mock.Of>()), - Mock.Of(), Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); @@ -731,20 +771,18 @@ namespace Tgstation.Server.Tests.Live.Instance .GetProcess(ddProc.Id); // Ensure it's responding to health checks - await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime); + await Task.WhenAny(Task.Delay(6000, cancellationToken), ourProcessHandler.Lifetime); Assert.IsFalse(ddProc.HasExited); // check DD agrees - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics8=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics8=1", cancellationToken); Assert.IsNotNull(topicRequestResult); Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType); Assert.IsNotNull(topicRequestResult.StringData); - Assert.AreEqual(topicRequestResult.StringData, "received health check"); + Assert.AreEqual("received health check", topicRequestResult.StringData); var ddStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -752,14 +790,16 @@ namespace Tgstation.Server.Tests.Live.Instance }, cancellationToken); ValidateSessionId(ddStatus, true); + global::System.Console.WriteLine($"WATCHDOG TEST {instanceClient.Metadata.Id}: COMMENCE PROCESS SUSPEND FOR HEALTH CHECK DEATH PID {ourProcessHandler.Id}."); ourProcessHandler.Suspend(); + global::System.Console.WriteLine($"WATCHDOG TEST {instanceClient.Metadata.Id}: FINISH PROCESS SUSPEND FOR HEALTH CHECK DEATH. WAITING FOR LIFETIME {ourProcessHandler.Id}."); - await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); + await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(4), cancellationToken)); + Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); - var timeout = 60; + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value); if (ddStatus.Status.Value == WatchdogStatus.Offline) @@ -770,6 +810,8 @@ namespace Tgstation.Server.Tests.Live.Instance if (--timeout == 0) Assert.Fail("DreamDaemon didn't shutdown within the timeout!"); + + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); } while (timeout > 0); @@ -816,7 +858,6 @@ namespace Tgstation.Server.Tests.Live.Instance throw; } } - await Task.Delay(TimeSpan.FromSeconds(3), cts.Token); return await instanceClient.DreamDaemon.Start(cancellationToken); } @@ -846,7 +887,9 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine("TEST: Sending Bridge tests topic..."); - var bridgeTestTopicResult = await topicClient.SendTopic(IPAddress.Loopback, $"tgs_integration_test_tactics2={accessIdentifier}", FindTopicPort(), cancellationToken); + var bridgeTestTopicResult = await SendTestTopic( + $"tgs_integration_test_tactics2={accessIdentifier}", + cancellationToken); Assert.AreEqual("ack2", bridgeTestTopicResult.StringData); await bridgeTestsTcs.Task.WaitAsync(cancellationToken); @@ -855,7 +898,7 @@ namespace Tgstation.Server.Tests.Live.Instance BridgeController.LogContent = true; // Time for DD to revert the bridge access identifier change - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); } async Task ValidateTopicLimits(CancellationToken cancellationToken) @@ -896,10 +939,8 @@ namespace Tgstation.Server.Tests.Live.Instance try { System.Console.WriteLine($"Topic send limit test S:{currentSize}..."); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, + topicRequestResult = await SendTestTopic( $"tgs_integration_test_tactics3={topicClient.SanitizeString(JsonConvert.SerializeObject(topic, DMApiConstants.SerializerSettings))}", - FindTopicPort(), cancellationToken); } catch (ArgumentOutOfRangeException) @@ -911,7 +952,11 @@ namespace Tgstation.Server.Tests.Live.Instance || topicRequestResult.StringData != "pass") { if (topicRequestResult != null) + { + Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType, $"String data is: {topicRequestResult.StringData ?? "<>"}"); Assert.AreEqual("fail", topicRequestResult.StringData); + } + if (currentSize == lastSize + 1) break; baseSize = lastSize; @@ -937,10 +982,8 @@ namespace Tgstation.Server.Tests.Live.Instance { var currentSize = baseSize + (int)Math.Pow(2, nextPow); System.Console.WriteLine($"Topic recieve limit test S:{currentSize}..."); - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, + var topicRequestResult = await SendTestTopic( $"tgs_integration_test_tactics4={topicClient.SanitizeString(currentSize.ToString())}", - FindTopicPort(), cancellationToken); if (topicRequestResult.ResponseType != TopicResponseType.StringResponse @@ -1068,8 +1111,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text); } - void CheckDDPriority() + async ValueTask CheckDDPriority() { + await Task.Yield(); var allProcesses = TestLiveServer.GetEngineServerProcessesOnPort(testVersion.Engine.Value, ddPort).Where(x => !x.HasExited).ToList(); if (allProcesses.Count == 0) Assert.Fail("Expected engine server to be running here"); @@ -1109,7 +1153,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -1157,7 +1201,7 @@ namespace Tgstation.Server.Tests.Live.Instance ValidateSessionId(daemonStatus, true); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.AreEqual(true, daemonStatus.SoftRestart); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -1195,7 +1239,7 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 70, false, null, cancellationToken); - CheckDDPriority(); + await CheckDDPriority(); var byondInstallJobTask = instanceClient.Engine.SetActiveVersion( new EngineVersionRequest @@ -1249,22 +1293,28 @@ namespace Tgstation.Server.Tests.Live.Instance public async Task StartAndLeaveRunning(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG STARTING ENDLESS"); - await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); - var startJob = await StartDD(cancellationToken); await WaitForJob(startJob, 40, false, null, cancellationToken); - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = "slow_start=1", + }, + cancellationToken); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); - CheckDDPriority(); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); + await CheckDDPriority(); Assert.AreEqual(ddPort, daemonStatus.CurrentPort); // Try killing the DD process to ensure it gets set to the restoring state bool firstTime = true; do { + if(!firstTime) + Assert.IsFalse(daemonStatus.SoftRestart); + ValidateSessionId(daemonStatus, true); KillDD(firstTime); firstTime = false; @@ -1272,7 +1322,7 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online); - Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status); // Kill it again do @@ -1281,13 +1331,18 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring); - Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status); await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = String.Empty, + }, + cancellationToken); ValidateSessionId(daemonStatus, true); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1304,17 +1359,17 @@ namespace Tgstation.Server.Tests.Live.Instance return ddProc != null; } - public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken) - => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring ? true : testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); - public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken) + public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0) + => TellWorldToReboot2(instanceClient, instanceManager, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); + public static async Task TellWorldToReboot2(IInstanceClient instanceClient, IInstanceManager instanceManager, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); - var initialCompileJob = daemonStatus.ActiveCompileJob; + var initialSession = daemonStatus.ActiveCompileJob; - System.Console.WriteLine("TEST: Sending world reboot topic..."); + System.Console.WriteLine($"TEST: Sending world reboot topic @ {path}#L{source}"); - var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", topicPort, cancellationToken); + var result = await SendTestTopic("tgs_integration_test_special_tactics=1", topicClient, instanceManager.GetInstanceReference(instanceClient.Metadata), topicPort, cancellationToken); Assert.AreEqual("ack", result.StringData); using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -1328,11 +1383,9 @@ namespace Tgstation.Server.Tests.Live.Instance await Task.Delay(TimeSpan.FromSeconds(1), tempToken); daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); } - while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id); + while (initialSession.Id == daemonStatus.ActiveCompileJob.Id); } - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - if (waitForOnlineIfRestoring && daemonStatus.Status == WatchdogStatus.Restoring) { do @@ -1353,7 +1406,7 @@ namespace Tgstation.Server.Tests.Live.Instance ApiValidationSecurityLevel = deploymentSecurity, ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}", RequireDMApiValidation = requireApi, - Timeout = TimeSpan.FromMilliseconds(1), + Timeout = !ranTimeoutTest ? TimeSpan.FromMilliseconds(1) : TimeSpan.FromMinutes(5), }, cancellationToken); JobResponse compileJobJob; @@ -1366,29 +1419,33 @@ namespace Tgstation.Server.Tests.Live.Instance compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken); + + await instanceClient.DreamMaker.Update(new DreamMakerRequest + { + Timeout = TimeSpan.FromMinutes(5), + }, cancellationToken); ranTimeoutTest = true; } - await instanceClient.DreamMaker.Update(new DreamMakerRequest - { - Timeout = TimeSpan.FromMinutes(5), - }, cancellationToken); - compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, false, null, cancellationToken); + // annoying but, with signalR instant job updates, this running task can get queued before the task that processes the watchdog's monitor activation + for (var i = 0; i < 10; ++i) + await Task.Yield(); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; + Assert.IsNotNull(targetJob); if (requireApi) - { - var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; - Assert.IsNotNull(targetJob); Assert.IsNotNull(targetJob.DMApiVersion); - } + else + Assert.IsNull(targetJob.DMApiVersion); return ddInfo; } - async Task GracefulWatchdogShutdown(uint timeout, CancellationToken cancellationToken) + async Task GracefulWatchdogShutdown(CancellationToken cancellationToken) { await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -1398,9 +1455,10 @@ namespace Tgstation.Server.Tests.Live.Instance var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline); + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); if (ddStatus.Status.Value == WatchdogStatus.Offline) break; @@ -1446,9 +1504,12 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) { - var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", FindTopicPort(), cancellationToken); - Assert.AreEqual("yeah gimmie a sec", result.StringData); - await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); + System.Console.WriteLine("TEST: TestLegacyBridgeEndpoint"); + var result = await SendTestTopic( + "im_out_of_memes=1", + cancellationToken); + Assert.IsNotNull(result); + Assert.AreEqual("all gucci", result.StringData); await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); } } diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 6fce2c452c..645edac147 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -153,6 +153,8 @@ namespace Tgstation.Server.Tests.Live $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", $"General:SkipAddingByondFirewallException={!TestingUtils.RunningInGitHubActions}", $"General:OpenDreamGitUrl={OpenDreamUrl}", + $"Security:TokenExpiryMinutes=120", // timeouts are useless for us + $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", }; swarmArgs = new List(); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 14b26d8ec8..6edee102ea 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -28,6 +28,7 @@ using Newtonsoft.Json; using Npgsql; using Tgstation.Server.Api; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Api.Models.Response; @@ -53,12 +54,12 @@ namespace Tgstation.Server.Tests.Live { public static readonly Version TestUpdateVersion = new(5, 11, 0); - static readonly ushort mainDDPort = FreeTcpPort(); - static readonly ushort mainDMPort = FreeTcpPort(mainDDPort); - static readonly ushort compatDMPort = FreeTcpPort(mainDDPort, mainDMPort); - static readonly ushort compatDDPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort); - static readonly ushort odDMPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort, compatDDPort); - static readonly ushort odDDPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort, compatDDPort, odDMPort); + static readonly Lazy odDMPort = new Lazy(() => FreeTcpPort()); + static readonly Lazy odDDPort = new Lazy(() => FreeTcpPort(odDMPort.Value)); + static readonly Lazy compatDMPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value)); + static readonly Lazy compatDDPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value)); + static readonly Lazy mainDDPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value, compatDDPort.Value)); + static readonly Lazy mainDMPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value, compatDDPort.Value, mainDDPort.Value)); readonly ServerClientFactory clientFactory = new (new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString())); @@ -146,8 +147,12 @@ namespace Tgstation.Server.Tests.Live static ushort FreeTcpPort(params ushort[] usedPorts) { + var portList = new ushort[] { 42069, 42070, 42071, 42072, 42073, 42074 }; + return portList.First(x => !usedPorts.Contains(x)); + /* ushort result; var listeners = new List(); + try { do @@ -166,7 +171,7 @@ namespace Tgstation.Server.Tests.Live result = (ushort)((IPEndPoint)l.LocalEndpoint).Port; } - while (usedPorts.Contains(result)); + while (usedPorts.Contains(result) || result < 10000); } finally { @@ -176,6 +181,7 @@ namespace Tgstation.Server.Tests.Live } } return result; + */ } [ClassInitialize] @@ -1049,7 +1055,7 @@ namespace Tgstation.Server.Tests.Live var ioManager = new Host.IO.DefaultIOManager(); var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); - var jobsTest = new JobsRequiredTest(instanceClient.Jobs); + await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs); var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows ? new Host.IO.WindowsPostWriteHandler() : new Host.IO.PosixPostWriteHandler(loggerFactory.CreateLogger())); @@ -1080,7 +1086,6 @@ namespace Tgstation.Server.Tests.Live new PlatformIdentifier().IsWindows ? new WindowsProcessFeatures(loggerFactory.CreateLogger()) : new PosixProcessFeatures(new Lazy(() => processExecutor), ioManager, loggerFactory.CreateLogger()), - new AsyncDelayer(), ioManager, loggerFactory.CreateLogger(), loggerFactory); @@ -1213,7 +1218,7 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public Task TestOpenDreamExclusiveTgsOperation() { - if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_OD_EXCLUSIVE"))) + if (Environment.GetEnvironmentVariable("TGS_TEST_OD_EXCLUSIVE") != "true") Assert.Inconclusive("This test is covered by TestStandardTgsOperation"); return TestStandardTgsOperation(true); @@ -1395,11 +1400,11 @@ namespace Tgstation.Server.Tests.Live InstanceResponse odInstance, compatInstance; if (!openDreamOnly) { + jobsHubTestTask = FailFast(await jobsHubTest.Run(cancellationToken)); // returns Task var rootTest = FailFast(RawRequestTests.Run(clientFactory, firstAdminClient, cancellationToken)); var adminTest = FailFast(new AdministrationTest(firstAdminClient.Administration).Run(cancellationToken)); var usersTest = FailFast(new UsersTest(firstAdminClient).Run(cancellationToken)); - jobsHubTestTask = FailFast(jobsHubTest.Run(cancellationToken)); var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory); var compatInstanceTask = instanceManagerTest.CreateTestInstance("CompatTestsInstance", cancellationToken); var odInstanceTask = instanceManagerTest.CreateTestInstance("OdTestsInstance", cancellationToken); @@ -1457,8 +1462,8 @@ namespace Tgstation.Server.Tests.Live await edgeODVersionTask, server.OpenDreamUrl, firstAdminClient.Instances.CreateClient(odInstance), - odDMPort, - odDDPort, + odDMPort.Value, + odDDPort.Value, server.HighPriorityDreamDaemon, server.UsingBasicWatchdog, cancellationToken); @@ -1484,8 +1489,8 @@ namespace Tgstation.Server.Tests.Live }, server.OpenDreamUrl, firstAdminClient.Instances.CreateClient(compatInstance), - compatDMPort, - compatDDPort, + compatDMPort.Value, + compatDDPort.Value, server.HighPriorityDreamDaemon, server.UsingBasicWatchdog, cancellationToken)); @@ -1497,8 +1502,8 @@ namespace Tgstation.Server.Tests.Live instanceTest .RunTests( instanceClient, - mainDMPort, - mainDDPort, + mainDMPort.Value, + mainDDPort.Value, server.HighPriorityDreamDaemon, server.LowPriorityDeployments, server.UsingBasicWatchdog, @@ -1532,10 +1537,11 @@ namespace Tgstation.Server.Tests.Live // test the reattach message queueing // for the code coverage really... - var topicRequestResult = await WatchdogTest.StaticTopicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics6=1", - mainDDPort, + var topicRequestResult = await WatchdogTest.SendTestTopic( + "tgs_integration_test_tactics6=1", + WatchdogTest.StaticTopicClient, + null, + mainDDPort.Value, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1590,7 +1596,7 @@ namespace Tgstation.Server.Tests.Live .ToList(); } - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); @@ -1608,10 +1614,11 @@ namespace Tgstation.Server.Tests.Live var chatReadTask = instanceClient.ChatBots.List(null, cancellationToken); // Check the DMAPI got the channels again https://github.com/tgstation/tgstation-server/issues/1490 - topicRequestResult = await WatchdogTest.StaticTopicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics7=1", - mainDDPort, + topicRequestResult = await WatchdogTest.SendTestTopic( + "tgs_integration_test_tactics7=1", + WatchdogTest.StaticTopicClient, + GetInstanceManager().GetInstanceReference(instanceClient.Metadata), + mainDDPort.Value, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1622,7 +1629,13 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(connectedChannelCount, topicRequestResult.FloatData.Value); - dd = await WatchdogTest.TellWorldToReboot2(instanceClient, WatchdogTest.StaticTopicClient, mainDDPort, true, cancellationToken); + dd = await WatchdogTest.TellWorldToReboot2( + instanceClient, + GetInstanceManager(), + WatchdogTest.StaticTopicClient, + mainDDPort.Value, + true, + cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); // if this assert fails, you likely have to crack open the debugger and read test_fail_reason.txt manually Assert.IsNull(dd.StagedCompileJob); @@ -1649,22 +1662,17 @@ namespace Tgstation.Server.Tests.Live { var jobs = await instanceClient.Jobs.ListActive(null, cancellationToken); if (jobs.Count == 0) - { - var entities = await instanceClient.Jobs.List(null, cancellationToken); - var getTasks = entities - .Select(e => instanceClient.Jobs.GetId(e, cancellationToken)) - .ToList(); - - jobs = (await ValueTaskExtensions.WhenAll(getTasks)) + jobs = (await instanceClient.Jobs.List(null, cancellationToken)) .Where(x => x.StartedAt.Value > preStartupTime) - .ToList(); - } + .ToList(); + else + jobs = jobs.Where(x => x.JobCode.Value.IsServerStartupJob()).ToList(); - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); - await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken); + await jrt.WaitForJob(job, 140, job.JobCode == JobCode.ReconnectChatBot ? null : false, null, cancellationToken); } } @@ -1675,16 +1683,16 @@ namespace Tgstation.Server.Tests.Live var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { + await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - await jobsHubTest.WaitForReconnect(cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1718,22 +1726,24 @@ namespace Tgstation.Server.Tests.Live serverTask = server.Run(cancellationToken).AsTask(); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { + await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - await jobsHubTest.WaitForReconnect(cancellationToken); var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value); Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); - currentDD = await wdt.TellWorldToReboot(true, cancellationToken); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); + currentDD = await wdt.TellWorldToReboot(false, cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken); - await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); + await using var repoTestObj = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + var repoTest = repoTestObj.RunPostTest(cancellationToken); + await using var chatTestObj = new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance); + await chatTestObj.RunPostTest(cancellationToken); await repoTest; await DummyChatProvider.RandomDisconnections(false, cancellationToken); diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index 1075b698d3..98b7c82f44 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -24,7 +24,6 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - Mock.Of(), new DefaultIOManager(), Mock.Of>(), loggerFactory); @@ -45,40 +44,47 @@ namespace Tgstation.Server.Tests [TestMethod] public async Task TestScriptExecutionWithFileOutput() { - using var loggerFactory = LoggerFactory.Create(x => { }); + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - Mock.Of(), new DefaultIOManager(), - Mock.Of>(), + loggerFactory.CreateLogger(), loggerFactory); - var tempFile = Path.GetTempFileName(); - File.Delete(tempFile); - try - { - await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, tempFile, true, true)) - { - using var cts = new CancellationTokenSource(); - cts.CancelAfter(3000); - var exitCode = await process.Lifetime.WaitAsync(cts.Token); - - await process.GetCombinedOutput(cts.Token); - - Assert.AreEqual(0, exitCode); - } - - Assert.IsTrue(File.Exists(tempFile)); - var result = File.ReadAllText(tempFile).Trim(); - - // no guarantees about order - Assert.IsTrue(result.Contains("Hello World!")); - Assert.IsTrue(result.Contains("Hello Error!")); - } - finally + // run on a loop to spot the hang + for (var i = 0; i < 1000; ++i) { + var tempFile = Path.GetTempFileName(); File.Delete(tempFile); + try + { + await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, tempFile, true, true)) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(3000); + var exitCode = await process.Lifetime.WaitAsync(cts.Token); + + await process.GetCombinedOutput(cts.Token); + + Assert.AreEqual(0, exitCode); + } + + Assert.IsTrue(File.Exists(tempFile), $"Could not find temp file: {tempFile}"); + var result = File.ReadAllText(tempFile).Trim(); + + // no guarantees about order + Assert.IsTrue(result.Contains("Hello World!"), $"Result: {result}"); + Assert.IsTrue(result.Contains("Hello Error!"), $"Result: {result}"); + } + finally + { + File.Delete(tempFile); + } } } } diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 03b998d438..34dad9e006 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -208,7 +208,6 @@ namespace Tgstation.Server.Tests new Lazy(() => null), Mock.Of(), loggerFactory.CreateLogger()), - Mock.Of(), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); @@ -416,11 +415,13 @@ namespace Tgstation.Server.Tests if (!Directory.Exists(directory)) Assert.Inconclusive("Webpanel not built?"); - var logo = new PlatformIdentifier().IsWindows - ? RootController.ProjectLogoSvgRouteWindows - : RootController.ProjectLogoSvgRouteLinux; + static string GetConstField(string name) => (string)typeof(RootController).GetField(name, BindingFlags.NonPublic | BindingFlags.Static).GetValue(null); - var path = $"../../../../../src/Tgstation.Server.Host/wwwroot{logo}"; + var logo = new PlatformIdentifier().IsWindows + ? GetConstField("LogoSvgWindowsName") + : GetConstField("LogoSvgLinuxName"); + + var path = $"../../../../../src/Tgstation.Server.Host/wwwroot/{logo}.svg"; Assert.IsTrue(File.Exists(path)); var content = await File.ReadAllBytesAsync(path); @@ -462,8 +463,8 @@ namespace Tgstation.Server.Tests EngineVersion engineVersion, Stream byondBytes, ByondInstallerBase byondInstaller, - IIOManager ioManager, - IProcessExecutor processExecutor, + DefaultIOManager ioManager, + ProcessExecutor processExecutor, string tempPath) { using (byondBytes)