From 53336f2a7568a5a2cddc69e98649cc050060943e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 13:45:05 -0400 Subject: [PATCH 01/23] Add VS docker debug support --- src/Tgstation.Server.Host/Dockerfile | 24 +++++++++++++++++++ .../Properties/launchSettings.json | 10 +++++--- .../Tgstation.Server.Host.csproj | 3 +++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 src/Tgstation.Server.Host/Dockerfile diff --git a/src/Tgstation.Server.Host/Dockerfile b/src/Tgstation.Server.Host/Dockerfile new file mode 100644 index 0000000000..3e1b6d9178 --- /dev/null +++ b/src/Tgstation.Server.Host/Dockerfile @@ -0,0 +1,24 @@ +#See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. + +# THIS SHOULD NOT BE USED TO CREATE THE PRODUCTION BUILD IT'S FOR DEBUGGING ONLY + +FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-buster-slim AS base +WORKDIR /app +EXPOSE 80 + +FROM mcr.microsoft.com/dotnet/core/sdk:3.1-buster AS build +WORKDIR /src +COPY ["src/Tgstation.Server.Host/Tgstation.Server.Host.csproj", "src/Tgstation.Server.Host/"] +COPY ["src/Tgstation.Server.Api/Tgstation.Server.Api.csproj", "src/Tgstation.Server.Api/"] +RUN dotnet restore "src/Tgstation.Server.Host/Tgstation.Server.Host.csproj" +COPY . . +WORKDIR "/src/src/Tgstation.Server.Host" +RUN dotnet build "Tgstation.Server.Host.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "Tgstation.Server.Host.csproj" -c Release -o /app/publish + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "Tgstation.Server.Host.dll"] diff --git a/src/Tgstation.Server.Host/Properties/launchSettings.json b/src/Tgstation.Server.Host/Properties/launchSettings.json index 3665dcfc22..8ed172fae4 100644 --- a/src/Tgstation.Server.Host/Properties/launchSettings.json +++ b/src/Tgstation.Server.Host/Properties/launchSettings.json @@ -2,11 +2,15 @@ "profiles": { "Tgstation.Server.Host": { "commandName": "Project", - "launchBrowser": false, "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development", - "DOTNET_ENVIRONMENT": "Development" + "DOTNET_ENVIRONMENT": "Development", + "ASPNETCORE_ENVIRONMENT": "Development" } + }, + "Docker": { + "commandName": "Docker", + "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", + "publishAllPorts": true } } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 243f4f4c32..bfcd9dc733 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -26,6 +26,8 @@ ClientApp/node_modules ClientApp/node_modules/.install-stamp + Linux + ..\.. @@ -68,6 +70,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + From e8fd2592d6fb485c504cca45147ec3852f027719 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:00:12 -0400 Subject: [PATCH 02/23] Minor fixes - Move ProgramShutdownTokenSource to the System namespace. - Fix a bad doc XML cref. --- src/Tgstation.Server.Host/Program.cs | 3 ++- .../{ => System}/ProgramShutdownTokenSource.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename src/Tgstation.Server.Host/{ => System}/ProgramShutdownTokenSource.cs (97%) diff --git a/src/Tgstation.Server.Host/Program.cs b/src/Tgstation.Server.Host/Program.cs index efd41636a4..ded8ef0744 100644 --- a/src/Tgstation.Server.Host/Program.cs +++ b/src/Tgstation.Server.Host/Program.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.System; namespace Tgstation.Server.Host { @@ -29,7 +30,7 @@ namespace Tgstation.Server.Host /// Entrypoint for the /// /// The command line arguments - /// The + /// The . public static async Task Main(string[] args) { // first arg is 100% always the update path, starting it otherwise is solely for debugging purposes diff --git a/src/Tgstation.Server.Host/ProgramShutdownTokenSource.cs b/src/Tgstation.Server.Host/System/ProgramShutdownTokenSource.cs similarity index 97% rename from src/Tgstation.Server.Host/ProgramShutdownTokenSource.cs rename to src/Tgstation.Server.Host/System/ProgramShutdownTokenSource.cs index 957b9f14b0..3ca60dbab9 100644 --- a/src/Tgstation.Server.Host/ProgramShutdownTokenSource.cs +++ b/src/Tgstation.Server.Host/System/ProgramShutdownTokenSource.cs @@ -1,7 +1,7 @@ using System; using System.Threading; -namespace Tgstation.Server.Host +namespace Tgstation.Server.Host.System { /// /// Contains a that triggers when the operating system requests the program shuts down. From 1d1854f8423645899b6ad3df6d059887cf3893b5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:26:51 -0400 Subject: [PATCH 03/23] Add more logging to docker startup --- build/tgs.docker.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/build/tgs.docker.sh b/build/tgs.docker.sh index f9904700dd..ac505e765b 100755 --- a/build/tgs.docker.sh +++ b/build/tgs.docker.sh @@ -1,9 +1,20 @@ #!/bin/sh +$SCRIPT_VERSION="1.1.0" + +echo "tgstation-server 4 container startup script v$SCRIPT_VERSION" +echo "PWD: $PWD" + PROD_CONFIG=/config_data/appsettings.Production.json +HOST_CONFIG=/app/appsettings.Production.json + if [ ! -f $PROD_CONFIG ]; then + echo "$PROD_CONFIG not detected! Creating empty..." echo "{}" > $PROD_CONFIG fi -ln -s $PROD_CONFIG /app/appsettings.Production.json +echo "Linking $PROD_CONFIG to $HOST_CONFIG" +ln -s $PROD_CONFIG $HOST_CONFIG + +echo "Executing console runner..." exec dotnet Tgstation.Server.Host.Console.dll From e1a10bc0dd89ba2e6f87b92845638e8d3b1d8c6a Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:29:55 -0400 Subject: [PATCH 04/23] Add a more informative throw when kestrel configuration is missing --- src/Tgstation.Server.Host/Core/ServerPortProivder.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index 244836e5af..88602efa95 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -21,11 +21,14 @@ namespace Tgstation.Server.Host.Core var httpEndpoint = configuration .GetSection("Kestrel") - .GetSection("Endpoints") + .GetSection("EndPoints") .GetSection("Http") .GetSection("Url") .Value; + if (httpEndpoint == null) + throw new InvalidOperationException("Missing required configuration option for Kestrel:EndPoints:Http:Url!"); + HttpApiPort = GetPortFromAddress(httpEndpoint); } From 4bd8bc3acb98933dcf58bffdc904c70ed55c1f72 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:38:23 -0400 Subject: [PATCH 05/23] Hopefully fix appveyor taking so long to build PRs - Always use dot to generate graphs. --- build/BuildDox.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/BuildDox.ps1 b/build/BuildDox.ps1 index 4d79b96727..4e0b4dcf13 100644 --- a/build/BuildDox.ps1 +++ b/build/BuildDox.ps1 @@ -11,11 +11,10 @@ if($publish_dox){ echo "Cloning https://git@$github_url..." git clone -b gh-pages --single-branch "https://git@$github_url" "$doxdir" 2>$null rm -r "$doxdir\*" - Add-Content "$bf\docs\Doxyfile" "`nPROJECT_NUMBER = $version`nINPUT = $bf`nOUTPUT_DIRECTORY = $doxdir`nPROJECT_LOGO = $bf/build/tgs.ico" -}else{ - Add-Content "$bf\docs\Doxyfile" "`nPROJECT_NUMBER = $version`nINPUT = $bf`nOUTPUT_DIRECTORY = $doxdir`nPROJECT_LOGO = $bf/build/tgs.ico`nHAVE_DOT=YES" } +Add-Content "$bf\docs\Doxyfile" "`nPROJECT_NUMBER = $version`nINPUT = $bf`nOUTPUT_DIRECTORY = $doxdir`nPROJECT_LOGO = $bf/build/tgs.ico`nHAVE_DOT=YES" + &"C:\Program Files\doxygen\bin\doxygen.exe" "$bf\docs\Doxyfile" if($publish_dox){ From a8d0f4c948ca805e12c74b33d1fdb5a79c2775f7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:41:00 -0400 Subject: [PATCH 06/23] Fix docker script error --- build/tgs.docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tgs.docker.sh b/build/tgs.docker.sh index ac505e765b..d08153c37f 100755 --- a/build/tgs.docker.sh +++ b/build/tgs.docker.sh @@ -1,6 +1,6 @@ #!/bin/sh -$SCRIPT_VERSION="1.1.0" +SCRIPT_VERSION="1.1.0" echo "tgstation-server 4 container startup script v$SCRIPT_VERSION" echo "PWD: $PWD" From fd1624ea2819f6fb5bf029a55ad5c3c638bd342c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:41:15 -0400 Subject: [PATCH 07/23] Improve ServerPortProvider code --- .../Core/ServerPortProivder.cs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index 88602efa95..5ddcd4c3a0 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -29,22 +29,14 @@ namespace Tgstation.Server.Host.Core if (httpEndpoint == null) throw new InvalidOperationException("Missing required configuration option for Kestrel:EndPoints:Http:Url!"); - HttpApiPort = GetPortFromAddress(httpEndpoint); - } - - /// - /// Convert a given to its port. - /// - /// The address . - /// The parsed port. - static ushort GetPortFromAddress(string address) - { - var splits = address.Split(":", StringSplitOptions.RemoveEmptyEntries); + var splits = httpEndpoint.Split(":", StringSplitOptions.RemoveEmptyEntries); var portString = splits.Last(); portString = portString.TrimEnd('/'); + if (!UInt16.TryParse(portString, out var result)) - throw new InvalidOperationException("Failed to parse HTTP API port!"); - return result; + throw new InvalidOperationException($"Failed to parse HTTP EndPoint port: {httpEndpoint}"); + + HttpApiPort = result; } } } From 05777646c714606ad50014e67e256966274a12d9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:51:03 -0400 Subject: [PATCH 08/23] Improve wording of an exception --- src/Tgstation.Server.Host/Core/ServerPortProivder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index 5ddcd4c3a0..a71dc72a29 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -27,7 +27,7 @@ namespace Tgstation.Server.Host.Core .Value; if (httpEndpoint == null) - throw new InvalidOperationException("Missing required configuration option for Kestrel:EndPoints:Http:Url!"); + throw new InvalidOperationException("Missing required configuration option Kestrel:EndPoints:Http:Url!"); var splits = httpEndpoint.Split(":", StringSplitOptions.RemoveEmptyEntries); var portString = splits.Last(); From 82a40c9bc01c91def65cfc360e2d17ea6811afd9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:55:34 -0400 Subject: [PATCH 09/23] Early out if ln fails. Make it verbose --- build/tgs.docker.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build/tgs.docker.sh b/build/tgs.docker.sh index d08153c37f..e253a94150 100755 --- a/build/tgs.docker.sh +++ b/build/tgs.docker.sh @@ -14,7 +14,12 @@ if [ ! -f $PROD_CONFIG ]; then fi echo "Linking $PROD_CONFIG to $HOST_CONFIG" -ln -s $PROD_CONFIG $HOST_CONFIG +ln -v -s $PROD_CONFIG $HOST_CONFIG + +if [ ! -f "$HOST_CONFIG" ]; then + echo "ln failed to create symlink!" + exit 1 +exit echo "Executing console runner..." exec dotnet Tgstation.Server.Host.Console.dll From 8b66f243e3a530735f2c7c65e2857762a5dde5a6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:58:16 -0400 Subject: [PATCH 10/23] Add missing fi --- build/tgs.docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tgs.docker.sh b/build/tgs.docker.sh index e253a94150..8ee520aab3 100755 --- a/build/tgs.docker.sh +++ b/build/tgs.docker.sh @@ -19,7 +19,7 @@ ln -v -s $PROD_CONFIG $HOST_CONFIG if [ ! -f "$HOST_CONFIG" ]; then echo "ln failed to create symlink!" exit 1 -exit +fi echo "Executing console runner..." exec dotnet Tgstation.Server.Host.Console.dll From 44f8ae022e9f702783e06a15cacdbb9944887fb2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 14:59:02 -0400 Subject: [PATCH 11/23] Echo not necessary with -v --- build/tgs.docker.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build/tgs.docker.sh b/build/tgs.docker.sh index 8ee520aab3..4445b92a53 100755 --- a/build/tgs.docker.sh +++ b/build/tgs.docker.sh @@ -13,7 +13,6 @@ if [ ! -f $PROD_CONFIG ]; then echo "{}" > $PROD_CONFIG fi -echo "Linking $PROD_CONFIG to $HOST_CONFIG" ln -v -s $PROD_CONFIG $HOST_CONFIG if [ ! -f "$HOST_CONFIG" ]; then From 6db68e3d3718afd8b60389c1708d5fc179440de1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 15:17:37 -0400 Subject: [PATCH 12/23] Debugging docker uses Debug config --- src/Tgstation.Server.Host/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Dockerfile b/src/Tgstation.Server.Host/Dockerfile index 3e1b6d9178..3138161af8 100644 --- a/src/Tgstation.Server.Host/Dockerfile +++ b/src/Tgstation.Server.Host/Dockerfile @@ -13,10 +13,10 @@ COPY ["src/Tgstation.Server.Api/Tgstation.Server.Api.csproj", "src/Tgstation.Ser RUN dotnet restore "src/Tgstation.Server.Host/Tgstation.Server.Host.csproj" COPY . . WORKDIR "/src/src/Tgstation.Server.Host" -RUN dotnet build "Tgstation.Server.Host.csproj" -c Release -o /app/build +RUN dotnet build "Tgstation.Server.Host.csproj" -c Debug -o /app/build FROM build AS publish -RUN dotnet publish "Tgstation.Server.Host.csproj" -c Release -o /app/publish +RUN dotnet publish "Tgstation.Server.Host.csproj" -c Debug -o /app/publish FROM base AS final WORKDIR /app From 7a9901d0eae46484b6875f92822fd13826e4f622 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 15:17:51 -0400 Subject: [PATCH 13/23] Log configuration at application startup --- src/Tgstation.Server.Host/Core/Application.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 47fc4e4d30..c75fa87853 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -336,6 +336,11 @@ namespace Tgstation.Server.Host.Core if (logger == null) throw new ArgumentNullException(nameof(logger)); + // Log the active configuration. + logger.LogTrace("Active Config:"); + foreach (var section in Configuration.GetChildren()) + logger.LogTrace("{0}: {1}", section.Key, section.Value); + logger.LogDebug("Content Root: {0}", hostingEnvironment.ContentRootPath); logger.LogTrace("Web Root: {0}", hostingEnvironment.WebRootPath); From 08c0e55680631caf65afb5623125d8dcbcb40f62 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 15:19:23 -0400 Subject: [PATCH 14/23] Rename local debugger configuration --- src/Tgstation.Server.Host/Properties/launchSettings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Properties/launchSettings.json b/src/Tgstation.Server.Host/Properties/launchSettings.json index 8ed172fae4..e39c1a9307 100644 --- a/src/Tgstation.Server.Host/Properties/launchSettings.json +++ b/src/Tgstation.Server.Host/Properties/launchSettings.json @@ -1,6 +1,6 @@ { "profiles": { - "Tgstation.Server.Host": { + "Local": { "commandName": "Project", "environmentVariables": { "DOTNET_ENVIRONMENT": "Development", From 6c2e53fc5ca4551a0546add00c148ec7f8c033ff Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 15:28:47 -0400 Subject: [PATCH 15/23] Delete this --- src/Tgstation.Server.Host/Core/Application.cs | 5 ----- .../Core/ServerPortProivder.cs | 22 ++++++++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index c75fa87853..47fc4e4d30 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -336,11 +336,6 @@ namespace Tgstation.Server.Host.Core if (logger == null) throw new ArgumentNullException(nameof(logger)); - // Log the active configuration. - logger.LogTrace("Active Config:"); - foreach (var section in Configuration.GetChildren()) - logger.LogTrace("{0}: {1}", section.Key, section.Value); - logger.LogDebug("Content Root: {0}", hostingEnvironment.ContentRootPath); logger.LogTrace("Web Root: {0}", hostingEnvironment.WebRootPath); diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index a71dc72a29..ba4e5ef469 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using System; using System.Linq; @@ -14,11 +15,30 @@ namespace Tgstation.Server.Host.Core /// Initializes a new instance of the . /// /// The to use. - public ServerPortProivder(IConfiguration configuration) + /// The to use. + public ServerPortProivder(IConfiguration configuration, ILogger logger) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); + // Log the active configuration. + logger.LogTrace("Active Config:"); + + void LogSection(IConfigurationSection section, string prefix) + { + prefix = $"{prefix}{section.Key}:"; + if (section.Value != null) + logger.LogTrace("{0} {1}", prefix, section.Value); + else + { + foreach (var child in section.GetChildren()) + LogSection(child, prefix); + } + } + + foreach (var section in configuration.GetChildren()) + LogSection(section, String.Empty); + var httpEndpoint = configuration .GetSection("Kestrel") .GetSection("EndPoints") From cdd910220d95cee7e747f0f55b727520e0829b61 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 15:59:46 -0400 Subject: [PATCH 16/23] ADDDKDJSF --- src/Tgstation.Server.Host/Core/ServerPortProivder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index ba4e5ef469..02de421de9 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -22,13 +22,13 @@ namespace Tgstation.Server.Host.Core throw new ArgumentNullException(nameof(configuration)); // Log the active configuration. - logger.LogTrace("Active Config:"); + logger.LogInformation("Active Config:"); void LogSection(IConfigurationSection section, string prefix) { prefix = $"{prefix}{section.Key}:"; if (section.Value != null) - logger.LogTrace("{0} {1}", prefix, section.Value); + logger.LogInformation("{0} {1}", prefix, section.Value); else { foreach (var child in section.GetChildren()) From 181ca95cb10cde8fd3f40197dd27fdccbae79458 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 16:05:38 -0400 Subject: [PATCH 17/23] Okay remove this --- .../Core/ServerPortProivder.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index 02de421de9..ee81825d91 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -21,24 +21,6 @@ namespace Tgstation.Server.Host.Core if (configuration == null) throw new ArgumentNullException(nameof(configuration)); - // Log the active configuration. - logger.LogInformation("Active Config:"); - - void LogSection(IConfigurationSection section, string prefix) - { - prefix = $"{prefix}{section.Key}:"; - if (section.Value != null) - logger.LogInformation("{0} {1}", prefix, section.Value); - else - { - foreach (var child in section.GetChildren()) - LogSection(child, prefix); - } - } - - foreach (var section in configuration.GetChildren()) - LogSection(section, String.Empty); - var httpEndpoint = configuration .GetSection("Kestrel") .GetSection("EndPoints") From b9b0a3f380c4d50694c60d251b8fdc97862508af Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 18:03:21 -0400 Subject: [PATCH 18/23] Fix using the wrong base path for hosting - Move the call to IServiceCollection.RemoveEventLogging - Fix warnings in ServerPortProvider --- src/Tgstation.Server.Host/Core/ServerPortProivder.cs | 4 +--- src/Tgstation.Server.Host/ServerFactory.cs | 6 ++++-- src/Tgstation.Server.Host/Setup/SetupApplication.cs | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs index ee81825d91..a71dc72a29 100644 --- a/src/Tgstation.Server.Host/Core/ServerPortProivder.cs +++ b/src/Tgstation.Server.Host/Core/ServerPortProivder.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using System; using System.Linq; @@ -15,8 +14,7 @@ namespace Tgstation.Server.Host.Core /// Initializes a new instance of the . /// /// The to use. - /// The to use. - public ServerPortProivder(IConfiguration configuration, ILogger logger) + public ServerPortProivder(IConfiguration configuration) { if (configuration == null) throw new ArgumentNullException(nameof(configuration)); diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index ebb98be2a5..5075e72412 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -43,7 +43,9 @@ namespace Tgstation.Server.Host throw new ArgumentNullException(nameof(args)); IHostBuilder CreateDefaultBuilder() => Microsoft.Extensions.Hosting.Host.CreateDefaultBuilder(args) - .ConfigureAppConfiguration((context, configurationBuilder) => configurationBuilder.SetBasePath(IOManager.ResolvePath())) + .ConfigureAppConfiguration((context, configuration) => configuration + .SetBasePath( + IOManager.ResolvePath())) .ConfigureServices(services => services.RemoveEventLogging()); var setupWizardHostBuilder = CreateDefaultBuilder() @@ -56,7 +58,7 @@ namespace Tgstation.Server.Host await setupHost.RunAsync(cancellationToken).ConfigureAwait(false); } - var hostBuilder = Microsoft.Extensions.Hosting.Host.CreateDefaultBuilder(args) + var hostBuilder = CreateDefaultBuilder() .ConfigureWebHostDefaults(webHostBuilder => webHostBuilder .UseApplication(postSetupServices) diff --git a/src/Tgstation.Server.Host/Setup/SetupApplication.cs b/src/Tgstation.Server.Host/Setup/SetupApplication.cs index 9ec5c003d9..a6ae4e07bc 100644 --- a/src/Tgstation.Server.Host/Setup/SetupApplication.cs +++ b/src/Tgstation.Server.Host/Setup/SetupApplication.cs @@ -58,6 +58,8 @@ namespace Tgstation.Server.Host.Setup if (services == null) throw new ArgumentNullException(nameof(services)); + services.RemoveEventLogging(); + services.AddSingleton(IOManager); services.AddSingleton(AssemblyInformationProvider); From b2bfa1f63ee17dcd5d7a13d0c3c127b38f49232a Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 18:11:57 -0400 Subject: [PATCH 19/23] Typos in the readme --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4d3d0add96..334f29e022 100644 --- a/README.md +++ b/README.md @@ -361,15 +361,17 @@ Contact project maintainers to get your client added to this list Note that tgstation-server is NOT a backup solution, the onus is on the server runners. -The `Repository` folder should, by the nature of git, not need to be backed up or should be done so on the remote server if necessary. +The `Repository` folder should be backed up on the remote server, do not rely on the instance copy to store changes. -The `BYOND` and `Game` folders should never be backed up due. +The `BYOND` and `Game` folders should never be backed up due to being intertwined with instance data. -The `Configuration` folder should and database be fully backed up +The `Configuration` folder should be fully backed up. -To restore an installation from backups, first restore the instance `Configuration` folders in their new homes. Then restore the database, modifying the `Path` column in the `Instances` table where necessary to point to the new instances. Then start the server pointed at the new database. +The database should be fully backed up. -Should you end up with a lost database for some reason or want to reattach a detached instance you can reattach an existing folder by creating an empty file named `TGS4_ALLOW_INSTANCE_ATTACH` inside it (This is automatically created when detaching instances). Then create a new instance with that path, this will bypass the empty folder check. Note that this will not restore things such as user permissions, server config options, or deployment metadata. Those must be reconfigured manually +To restore an installation from backups, first restore the instance `Configuration` folder in its new home. Then restore the database, modifying the `Path` column in the `Instances` table where necessary to point to the new instances. Then start the server pointed at the new database. + +Should you end up with a lost database for some reason or want to reattach a detached instance you can reattach an existing folder by creating an empty file named `TGS4_ALLOW_INSTANCE_ATTACH` inside it (This is automatically created when detaching instances). Then create a new instance with that path, this will bypass the empty folder check. Note that this will not restore things such as user permissions, server config options, or deployment metadata. Those must be reconfigured manually. ## Troubleshooting From 23aca07cc9fc358e97c4166a75ec91c2be9573e6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 18:32:08 -0400 Subject: [PATCH 20/23] Code cleanup for libgit2 interop --- .../Components/InstanceFactory.cs | 15 +- .../Components/Repository/ILibGit2Commands.cs | 29 ++++ ...actory.cs => ILibGit2RepositoryFactory.cs} | 2 +- .../Components/Repository/LibGit2Commands.cs | 34 ++++ ...Factory.cs => LibGit2RepositoryFactory.cs} | 34 ++-- .../Components/Repository/Repository.cs | 146 ++++++++++-------- .../Repository/RepositoryManager.cs | 26 +++- src/Tgstation.Server.Host/Core/Application.cs | 3 +- .../Repository/TestRepositoryFactory.cs | 10 +- 9 files changed, 212 insertions(+), 87 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Repository/ILibGit2Commands.cs rename src/Tgstation.Server.Host/Components/Repository/{IRepositoryFactory.cs => ILibGit2RepositoryFactory.cs} (96%) create mode 100644 src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs rename src/Tgstation.Server.Host/Components/Repository/{RepositoryFactory.cs => LibGit2RepositoryFactory.cs} (73%) diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 9c57799c91..a04d0c7747 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -108,9 +108,14 @@ namespace Tgstation.Server.Host.Components readonly IPlatformIdentifier platformIdentifier; /// - /// The for the . + /// The for the . /// - readonly IRepositoryFactory repositoryFactory; + readonly ILibGit2RepositoryFactory repositoryFactory; + + /// + /// The for the . + /// + readonly ILibGit2Commands repositoryCommands; /// /// The for the . @@ -138,6 +143,7 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of . + /// The value of . /// The value of . public InstanceFactory( IIOManager ioManager, @@ -157,7 +163,8 @@ namespace Tgstation.Server.Host.Components INetworkPromptReaper networkPromptReaper, IGitHubClientFactory gitHubClientFactory, IPlatformIdentifier platformIdentifier, - IRepositoryFactory repositoryFactory, + ILibGit2RepositoryFactory repositoryFactory, + ILibGit2Commands repositoryCommands, IServerPortProvider serverPortProvider) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -178,6 +185,7 @@ namespace Tgstation.Server.Host.Components this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.repositoryFactory = repositoryFactory ?? throw new ArgumentNullException(nameof(repositoryFactory)); + this.repositoryCommands = repositoryCommands ?? throw new ArgumentNullException(nameof(repositoryCommands)); this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider)); } @@ -198,6 +206,7 @@ namespace Tgstation.Server.Host.Components var eventConsumer = new EventConsumer(configuration); var repoManager = new RepositoryManager( repositoryFactory, + repositoryCommands, repoIoManager, eventConsumer, loggerFactory.CreateLogger(), diff --git a/src/Tgstation.Server.Host/Components/Repository/ILibGit2Commands.cs b/src/Tgstation.Server.Host/Components/Repository/ILibGit2Commands.cs new file mode 100644 index 0000000000..c125fafdd6 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Repository/ILibGit2Commands.cs @@ -0,0 +1,29 @@ +using LibGit2Sharp; +using System.Collections.Generic; + +namespace Tgstation.Server.Host.Components.Repository +{ + /// + /// For low level interactions with a . + /// + interface ILibGit2Commands + { + /// + /// Runs a blocking fetch operation on a given . + /// + /// The to fetch with. + /// The refspecs to fetch. + /// The to fetch. + /// The . + /// The message to write in the git log. + void Fetch(LibGit2Sharp.IRepository repository, IEnumerable refSpecs, Remote remote, FetchOptions fetchOptions, string logMessage); + + /// + /// Runs a blocking checkout operation on a given . + /// + /// The to checkout with. + /// The . + /// The git object to checkout. + void Checkout(LibGit2Sharp.IRepository repository, CheckoutOptions checkoutOptions, string commitish); + } +} diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryFactory.cs b/src/Tgstation.Server.Host/Components/Repository/ILibGit2RepositoryFactory.cs similarity index 96% rename from src/Tgstation.Server.Host/Components/Repository/IRepositoryFactory.cs rename to src/Tgstation.Server.Host/Components/Repository/ILibGit2RepositoryFactory.cs index 1b9d7b8c41..ac5ee07b91 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/ILibGit2RepositoryFactory.cs @@ -8,7 +8,7 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Factory for creating s. /// - interface IRepositoryFactory : ICredentialsProvider + interface ILibGit2RepositoryFactory : ICredentialsProvider { /// /// Create an in-memeory . diff --git a/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs b/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs new file mode 100644 index 0000000000..65c1f21049 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs @@ -0,0 +1,34 @@ +using LibGit2Sharp; +using System; +using System.Collections.Generic; + +namespace Tgstation.Server.Host.Components.Repository +{ + /// + sealed class LibGit2Commands : ILibGit2Commands + { + /// + public void Checkout(LibGit2Sharp.IRepository libGit2Repo, CheckoutOptions checkoutOptions, string commitish) + => Commands.Checkout(libGit2Repo, commitish, checkoutOptions); + + /// + public void Fetch( + LibGit2Sharp.IRepository libGit2Repo, + IEnumerable refSpecs, + Remote remote, + FetchOptions fetchOptions, + string logMessage) + { + if (libGit2Repo == null) + throw new ArgumentNullException(nameof(libGit2Repo)); + + if (!(libGit2Repo is LibGit2Sharp.Repository concreteRepo)) + throw new ArgumentException("libGit2Repo must be an instance of LibGit2Sharp.Repository!", nameof(libGit2Repo)); + + if (remote == null) + throw new ArgumentNullException(nameof(remote)); + + Commands.Fetch(concreteRepo, remote.Name, refSpecs, fetchOptions, logMessage); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryFactory.cs b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs similarity index 73% rename from src/Tgstation.Server.Host/Components/Repository/RepositoryFactory.cs rename to src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs index 69865ce950..06eb1926c9 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs @@ -9,18 +9,18 @@ using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Repository { /// - sealed class RepositoryFactory : IRepositoryFactory + sealed class LibGit2RepositoryFactory : ILibGit2RepositoryFactory { /// - /// The for the . + /// The for the . /// - readonly ILogger logger; + readonly ILogger logger; /// - /// Initializes a new instance of the . + /// Initializes a new instance of the . /// /// The value of . - public RepositoryFactory(ILogger logger) + public LibGit2RepositoryFactory(ILogger logger) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -28,8 +28,18 @@ namespace Tgstation.Server.Host.Components.Repository /// public LibGit2Sharp.IRepository CreateInMemory() { - logger.LogTrace("Creating in-memory repository..."); - return new LibGit2Sharp.Repository(); + logger.LogTrace("Creating in-memory libgit2 repository..."); + var repo = new LibGit2Sharp.Repository(); + try + { + logger.LogTrace("Successfully created in-memory libgit2 repository."); + return repo; + } + catch + { + repo.Dispose(); + throw; + } } /// @@ -37,9 +47,13 @@ namespace Tgstation.Server.Host.Components.Repository { if (path == null) throw new ArgumentNullException(nameof(path)); - logger.LogTrace("Creating repostory at {0}...", path); - return Task.Factory.StartNew( - () => (LibGit2Sharp.IRepository)new LibGit2Sharp.Repository(path), + + return Task.Factory.StartNew( + () => + { + logger.LogTrace("Creating libgit2 repostory at {0}...", path); + return new LibGit2Sharp.Repository(path); + }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 961a7910b8..6a1c0abf2d 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -43,21 +43,26 @@ namespace Tgstation.Server.Host.Components.Repository public string GitHubRepoName { get; } /// - public bool Tracking => Reference != null && repository.Head.IsTracking; + public bool Tracking => Reference != null && libGitRepo.Head.IsTracking; /// - public string Head => repository.Head.Tip.Sha; + public string Head => libGitRepo.Head.Tip.Sha; /// - public string Reference => repository.Head.FriendlyName; + public string Reference => libGitRepo.Head.FriendlyName; /// - public string Origin => repository.Network.Remotes.First().Url; + public string Origin => libGitRepo.Network.Remotes.First().Url; /// /// The for the /// - readonly LibGit2Sharp.IRepository repository; + readonly LibGit2Sharp.IRepository libGitRepo; + + /// + /// The for the . + /// + readonly ILibGit2Commands commands; /// /// The for the @@ -99,21 +104,32 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Construct a /// - /// The value of + /// The value of + /// The value of . /// The value of /// The value of /// The value of /// The value of /// The value if - public Repository(LibGit2Sharp.IRepository repository, IIOManager ioMananger, IEventConsumer eventConsumer, ICredentialsProvider credentialsProvider, ILogger logger, Action onDispose) + public Repository( + LibGit2Sharp.IRepository libGitRepo, + ILibGit2Commands commands, + IIOManager ioMananger, + IEventConsumer eventConsumer, + ICredentialsProvider credentialsProvider, + ILogger logger, + Action onDispose) { - this.repository = repository ?? throw new ArgumentNullException(nameof(repository)); + this.libGitRepo = libGitRepo ?? throw new ArgumentNullException(nameof(libGitRepo)); + this.commands = commands ?? throw new ArgumentNullException(nameof(commands)); this.ioMananger = ioMananger ?? throw new ArgumentNullException(nameof(ioMananger)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.credentialsProvider = credentialsProvider ?? throw new ArgumentNullException(nameof(credentialsProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); + IsGitHubRepository = Origin.Contains(GitHubUrl, StringComparison.InvariantCultureIgnoreCase); + if (IsGitHubRepository) { GetRepositoryOwnerName(Origin, out var owner, out var name); @@ -134,7 +150,7 @@ namespace Tgstation.Server.Host.Components.Repository } logger.LogTrace("Disposing..."); - repository.Dispose(); + libGitRepo.Dispose(); onDispose(); } @@ -191,15 +207,18 @@ namespace Tgstation.Server.Host.Components.Repository progressReporter(0); cancellationToken.ThrowIfCancellationRequested(); - Commands.Checkout(repository, committish, new CheckoutOptions - { - CheckoutModifiers = CheckoutModifiers.Force, - OnCheckoutProgress = CheckoutProgressHandler(progressReporter) - }); + commands.Checkout( + libGitRepo, + new CheckoutOptions + { + CheckoutModifiers = CheckoutModifiers.Force, + OnCheckoutProgress = CheckoutProgressHandler(progressReporter) + }, + committish); cancellationToken.ThrowIfCancellationRequested(); - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); } /// @@ -248,7 +267,7 @@ namespace Tgstation.Server.Host.Components.Repository var refSpecList = new List { refSpec }; var logMessage = String.Format(CultureInfo.InvariantCulture, "Merge remote pull request #{0}", testMergeParameters.Number); - var originalCommit = repository.Head; + var originalCommit = libGitRepo.Head; MergeResult result = null; @@ -261,37 +280,42 @@ namespace Tgstation.Server.Host.Components.Repository { logger.LogTrace("Fetching refspec {0}...", refSpec); - var remote = repository.Network.Remotes.First(); + var remote = libGitRepo.Network.Remotes.First(); progressReporter(0); - Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, refSpecList, new FetchOptions - { - Prune = true, - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => + commands.Fetch( + libGitRepo, + refSpecList, + remote, + new FetchOptions { - var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter((int)percentage); - return !cancellationToken.IsCancellationRequested; + Prune = true, + OnProgress = (a) => !cancellationToken.IsCancellationRequested, + OnTransferProgress = (a) => + { + var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); + progressReporter((int)percentage); + return !cancellationToken.IsCancellationRequested; + }, + OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password) }, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password) - }, logMessage); + logMessage); } catch (UserCancelledException) { } cancellationToken.ThrowIfCancellationRequested(); - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); cancellationToken.ThrowIfCancellationRequested(); - testMergeParameters.PullRequestRevision = repository.Lookup(testMergeParameters.PullRequestRevision ?? localBranchName).Sha; + testMergeParameters.PullRequestRevision = libGitRepo.Lookup(testMergeParameters.PullRequestRevision ?? localBranchName).Sha; cancellationToken.ThrowIfCancellationRequested(); logger.LogTrace("Merging {0} into {1}...", testMergeParameters.PullRequestRevision.Substring(0, 7), Reference); - result = repository.Merge(testMergeParameters.PullRequestRevision, sig, new MergeOptions + result = libGitRepo.Merge(testMergeParameters.PullRequestRevision, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, FailOnConflict = true, @@ -302,7 +326,7 @@ namespace Tgstation.Server.Host.Components.Repository } finally { - repository.Branches.Remove(localBranchName); + libGitRepo.Branches.Remove(localBranchName); } cancellationToken.ThrowIfCancellationRequested(); @@ -315,7 +339,7 @@ namespace Tgstation.Server.Host.Components.Repository cancellationToken.ThrowIfCancellationRequested(); } - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); if (result.Status == MergeStatus.Conflicts) @@ -327,7 +351,7 @@ namespace Tgstation.Server.Host.Components.Repository if (commitMessage != null && result.Status != MergeStatus.UpToDate) { logger.LogTrace("Committing merge: \"{0}\"...", commitMessage); - await Task.Factory.StartNew(() => repository.Commit(commitMessage, sig, sig, new CommitOptions + await Task.Factory.StartNew(() => libGitRepo.Commit(commitMessage, sig, sig, new CommitOptions { PrettifyMessage = true }), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); @@ -359,7 +383,7 @@ namespace Tgstation.Server.Host.Components.Repository await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish }, cancellationToken).ConfigureAwait(false); await Task.Factory.StartNew(() => { - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); RawCheckout(committish, progressReporter, cancellationToken); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } @@ -373,10 +397,10 @@ namespace Tgstation.Server.Host.Components.Repository await eventConsumer.HandleEvent(EventType.RepoFetch, Array.Empty(), cancellationToken).ConfigureAwait(false); await Task.Factory.StartNew(() => { - var remote = repository.Network.Remotes.First(); + var remote = libGitRepo.Network.Remotes.First(); try { - Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, remote.FetchRefSpecs.Select(x => x.Specification), new FetchOptions + Commands.Fetch((LibGit2Sharp.Repository)libGitRepo, remote.Name, remote.FetchRefSpecs.Select(x => x.Specification), new FetchOptions { Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, @@ -408,17 +432,17 @@ namespace Tgstation.Server.Host.Components.Repository Task PushHeadToTemporaryBranch(string username, string password, Action progressReporter, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { logger.LogInformation("Pushing changes to temporary remote branch..."); - var branch = repository.CreateBranch(RemoteTemporaryBranchName); + var branch = libGitRepo.CreateBranch(RemoteTemporaryBranchName); try { cancellationToken.ThrowIfCancellationRequested(); - var remote = repository.Network.Remotes.First(); + var remote = libGitRepo.Network.Remotes.First(); try { var forcePushString = String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName); - repository.Network.Push(remote, forcePushString, GeneratePushOptions(progress => progressReporter((int)(0.9f * progress)), username, password, cancellationToken)); + libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions(progress => progressReporter((int)(0.9f * progress)), username, password, cancellationToken)); var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName); - repository.Network.Push(remote, removalString, GeneratePushOptions(progress => progressReporter(90 + (int)(0.1f * progress)), username, password, cancellationToken)); + libGitRepo.Network.Push(remote, removalString, GeneratePushOptions(progress => progressReporter(90 + (int)(0.1f * progress)), username, password, cancellationToken)); } catch (UserCancelledException) { @@ -431,7 +455,7 @@ namespace Tgstation.Server.Host.Components.Repository } finally { - repository.Branches.Remove(branch); + libGitRepo.Branches.Remove(branch); } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); @@ -443,7 +467,7 @@ namespace Tgstation.Server.Host.Components.Repository if (!Tracking) throw new JobException("Cannot reset to origin while not on a tracked reference!"); logger.LogTrace("Reset to origin..."); - var trackedBranch = repository.Head.TrackedBranch; + var trackedBranch = libGitRepo.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false); await ResetToSha(trackedBranch.Tip.Sha, progressReporter, cancellationToken).ConfigureAwait(false); } @@ -458,16 +482,16 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogDebug("Reset to sha: {0}", sha.Substring(0, 7)); - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); cancellationToken.ThrowIfCancellationRequested(); - var gitObject = repository.Lookup(sha, ObjectType.Commit); + var gitObject = libGitRepo.Lookup(sha, ObjectType.Commit); cancellationToken.ThrowIfCancellationRequested(); if (gitObject == null) throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Cannot reset to non-existent SHA: {0}", sha)); - repository.Reset(ResetMode.Hard, gitObject.Peel(), new CheckoutOptions + libGitRepo.Reset(ResetMode.Hard, gitObject.Peel(), new CheckoutOptions { OnCheckoutProgress = CheckoutProgressHandler(progressReporter) }); @@ -491,7 +515,7 @@ namespace Tgstation.Server.Host.Components.Repository MergeResult result = null; Branch trackedBranch = null; - var oldHead = repository.Head; + var oldHead = libGitRepo.Head; var oldTip = oldHead.Tip; await Task.Factory.StartNew(() => @@ -499,13 +523,13 @@ namespace Tgstation.Server.Host.Components.Repository if (!Tracking) throw new JobException("Cannot reset to origin while not on a tracked reference!"); - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); cancellationToken.ThrowIfCancellationRequested(); - trackedBranch = repository.Head.TrackedBranch; + trackedBranch = libGitRepo.Head.TrackedBranch; logger.LogDebug("Merge origin/{2}: <{0} ({1})>", committerName, committerEmail, trackedBranch.FriendlyName); - result = repository.Merge(trackedBranch, new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now), new MergeOptions + result = libGitRepo.Merge(trackedBranch, new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now), new MergeOptions { CommitOnSuccess = true, FailOnConflict = true, @@ -519,14 +543,14 @@ namespace Tgstation.Server.Host.Components.Repository if (result.Status == MergeStatus.Conflicts) { logger.LogDebug("Merge conflict, aborting and reverting to {0}", oldHead.FriendlyName); - repository.Reset(ResetMode.Hard, oldTip, new CheckoutOptions + libGitRepo.Reset(ResetMode.Hard, oldTip, new CheckoutOptions { OnCheckoutProgress = CheckoutProgressHandler(progressReporter) }); cancellationToken.ThrowIfCancellationRequested(); } - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); if (result.Status == MergeStatus.Conflicts) @@ -566,9 +590,9 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Configuring <{0} ({1})> as author/committer", committerName, committerEmail); await Task.Factory.StartNew(() => { - repository.Config.Set("user.name", committerName); + libGitRepo.Config.Set("user.name", committerName); cancellationToken.ThrowIfCancellationRequested(); - repository.Config.Set("user.email", committerEmail); + libGitRepo.Config.Set("user.email", committerEmail); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); @@ -592,9 +616,9 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Resetting and cleaning untracked files..."); await Task.Factory.StartNew(() => { - repository.RemoveUntrackedFiles(); + libGitRepo.RemoveUntrackedFiles(); cancellationToken.ThrowIfCancellationRequested(); - repository.Reset(ResetMode.Hard, repository.Head.Tip, new CheckoutOptions + libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions { OnCheckoutProgress = CheckoutProgressHandler(progress => progressReporter(progress / 10)) }); @@ -620,10 +644,10 @@ namespace Tgstation.Server.Host.Components.Repository return await Task.Factory.StartNew(() => { - var remote = repository.Network.Remotes.First(); + var remote = libGitRepo.Network.Remotes.First(); try { - repository.Network.Push(repository.Head, GeneratePushOptions(FinalReporter, username, password, cancellationToken)); + libGitRepo.Network.Push(libGitRepo.Head, GeneratePushOptions(FinalReporter, username, password, cancellationToken)); return true; } catch (NonFastForwardException) @@ -648,18 +672,18 @@ namespace Tgstation.Server.Host.Components.Repository public Task IsSha(string committish, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { // check if it's a tag - var gitObject = repository.Lookup(committish, ObjectType.Tag); + var gitObject = libGitRepo.Lookup(committish, ObjectType.Tag); if (gitObject != null) return false; cancellationToken.ThrowIfCancellationRequested(); // check if it's a branch - if (repository.Branches[committish] != null) + if (libGitRepo.Branches[committish] != null) return false; cancellationToken.ThrowIfCancellationRequested(); // err on the side of references, if we can't look it up, assume its a reference - if (repository.Lookup(committish) != null) + if (libGitRepo.Lookup(committish) != null) return true; return false; }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index aa988f1380..fc053ed6cf 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -19,9 +19,14 @@ namespace Tgstation.Server.Host.Components.Repository public bool CloneInProgress { get; private set; } /// - /// The for the + /// The for the /// - readonly IRepositoryFactory repositoryFactory; + readonly ILibGit2RepositoryFactory repositoryFactory; + + /// + /// The for the . + /// + readonly ILibGit2Commands commands; /// /// The for the @@ -57,13 +62,15 @@ namespace Tgstation.Server.Host.Components.Repository /// Construct a /// /// The value of . + /// The value of . /// The value of /// The value of /// The value of /// The value of /// The value of public RepositoryManager( - IRepositoryFactory repositoryFactory, + ILibGit2RepositoryFactory repositoryFactory, + ILibGit2Commands commands, IIOManager ioManager, IEventConsumer eventConsumer, ILogger repositoryLogger, @@ -71,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Repository RepositorySettings repositorySettings) { this.repositoryFactory = repositoryFactory ?? throw new ArgumentNullException(nameof(repositoryFactory)); + this.commands = commands ?? throw new ArgumentNullException(nameof(commands)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.repositoryLogger = repositoryLogger ?? throw new ArgumentNullException(nameof(repositoryLogger)); @@ -175,12 +183,18 @@ namespace Tgstation.Server.Host.Components.Repository using (var context = await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) try { - var repo = await repositoryFactory.CreateFromPath(ioManager.ResolvePath(), cancellationToken).ConfigureAwait(false); + var libGitRepo = await repositoryFactory.CreateFromPath(ioManager.ResolvePath(), cancellationToken).ConfigureAwait(false); - if (repo == null) + if (libGitRepo == null) return null; - return new Repository(repo, ioManager, eventConsumer, repositoryFactory, repositoryLogger, () => + return new Repository( + libGitRepo, + commands, + ioManager, + eventConsumer, + repositoryFactory, + repositoryLogger, () => { logger.LogTrace("Releasing semaphore due to Repository disposal..."); semaphore.Release(); diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 47fc4e4d30..42376e4c6e 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -289,7 +289,8 @@ namespace Tgstation.Server.Host.Core }); // configure component services - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs index 8d34683556..5d11ac34dc 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs @@ -9,26 +9,26 @@ using System.Threading.Tasks; namespace Tgstation.Server.Host.Components.Repository.Tests { /// - /// Tests for . + /// Tests for . /// [TestClass] public sealed class TestRepositoryFactory { - static IRepositoryFactory CreateFactory() => new RepositoryFactory(Mock.Of>()); + static ILibGit2RepositoryFactory CreateFactory() => new LibGit2RepositoryFactory(Mock.Of>()); static Task TestRepoLoading( string path, - IRepositoryFactory repositoryFactory = null) => + ILibGit2RepositoryFactory repositoryFactory = null) => (repositoryFactory ?? CreateFactory()) .CreateFromPath(path, default); [TestMethod] - public void TestConstructionThrows() => Assert.ThrowsException(() => new RepositoryFactory(null)); + public void TestConstructionThrows() => Assert.ThrowsException(() => new LibGit2RepositoryFactory(null)); [TestMethod] public void TestInMemoryRepoCreation() { - new RepositoryFactory(Mock.Of>()).CreateInMemory().Dispose(); + new LibGit2RepositoryFactory(Mock.Of>()).CreateInMemory().Dispose(); } [TestMethod] From 6ad3a0b2eefcf643d123af3ad33836b7cffab71e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 19:30:16 -0400 Subject: [PATCH 21/23] Fix which JobException is thrown for git credentials errors --- .../Components/Repository/LibGit2RepositoryFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs index 06eb1926c9..3d4663dde3 100644 --- a/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs @@ -92,7 +92,7 @@ namespace Tgstation.Server.Host.Components.Repository if (supportsAnonymous) return new DefaultCredentials(); - if (!hasCreds) + if (supportsUserPass) throw new JobException("Remote does not support anonymous authentication!"); throw new JobException("Server does not support anonymous or username/password authentication!"); From 016637b1e7afd02a23b9c366c961dc6454d2bacd Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 19:30:34 -0400 Subject: [PATCH 22/23] Update .dockerignore --- .dockerignore | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.dockerignore b/.dockerignore index 063b3c17bb..d16f53ed31 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,18 @@ -.dockerignore +*/bin +*/obj + .git .github -.gitignore .vs .vscode -.travis.yml .codecov.yml +.dockerignore +.gitignore +.travis.yml appveyor.yml -README +omnisharp.json +README.md + build/** !build/analyzers.ruleset !build/tgs.docker.sh @@ -15,11 +20,7 @@ build/** !build/tgs.png !build/stylecop.json !build/Version.props -*/bin -*/obj -tests docs -tools src/DMAPI src/Tgstation.Server.Host/ClientApp/build src/Tgstation.Server.Host/ClientApp/node_modules @@ -29,3 +30,5 @@ src/Tgstation.Server.Host/tgs.bat src/Tgstation.Server.Host/tgs.sh src/Tgstation.Server.Host.Client src/Tgstation.Server.Host.Service +tests +tools From 0899b2ce4ae26de332d3665c31563c7cf444307e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 24 Apr 2020 19:36:13 -0400 Subject: [PATCH 23/23] Fixes and performance improvements for Dockerfile --- build/Dockerfile | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 1842ae64e4..c1156ae74b 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build +FROM mcr.microsoft.com/dotnet/core/sdk:3.1-bionic AS build # install node and npm # replace shell with bash so we can source files @@ -18,29 +18,42 @@ RUN . $NVM_DIR/nvm.sh \ ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH -WORKDIR /src +# Build web control panel +WORKDIR /repo/build +COPY build/Version.props Version.props + +WORKDIR /repo/src/Tgstation.Server.Host + +COPY src/Tgstation.Server.Host/ClientApp ClientApp +COPY src/Tgstation.Server.Host/Tgstation.Server.Host.csproj ./ + +RUN dotnet msbuild -target:NpmBuild + +WORKDIR /repo + +# Restore nuget packages COPY tgstation-server.sln ./ COPY src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj src/Tgstation.Server.Host.Console/ COPY src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj src/Tgstation.Server.Host.Watchdog/ -COPY src/Tgstation.Server.Host/Tgstation.Server.Host.csproj src/Tgstation.Server.Host/ COPY src/Tgstation.Server.Api/Tgstation.Server.Api.csproj src/Tgstation.Server.Api/ RUN dotnet restore -nowarn:MSB3202,nu1503 -p:RestoreUseSkipNonexistentTargets=false +# Final copy for building COPY . . #run dos2unix on tgs.docker.sh so we can build without issue on windows RUN dos2unix build/tgs.docker.sh -WORKDIR /src/src/Tgstation.Server.Host.Console +WORKDIR /repo/src/Tgstation.Server.Host.Console RUN dotnet publish -c Release -o /app -WORKDIR /src/src/Tgstation.Server.Host +WORKDIR /repo/src/Tgstation.Server.Host RUN dotnet publish -c Release -o /app/lib/Default && mv /app/lib/Default/appsettings* /app -FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 +FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-bionic EXPOSE 80 #needed for byond @@ -52,7 +65,7 @@ RUN apt-get update \ WORKDIR /app COPY --from=build /app . -COPY --from=build /src/build/tgs.docker.sh tgs.sh +COPY --from=build /repo/build/tgs.docker.sh tgs.sh VOLUME ["/config_data", "/tgs_logs", "/app/lib"]