TGS Test CI Workflow (#7787)

This commit is contained in:
Selis
2024-02-19 17:10:13 +01:00
committed by GitHub
parent 3f80966a8e
commit 7ed297e490
15 changed files with 567 additions and 50 deletions

64
.github/workflows/tgs_test.yml vendored Normal file
View File

@@ -0,0 +1,64 @@
name: TGS Test Suite
on:
push:
branches:
- master
paths:
- '.tgs.yml'
- '.github/workflows/tgs_test.yml'
- '_build_dependencies.sh'
- 'code/__DEFINES/tgs.config.dm'
- 'code/__DEFINES/tgs.dm'
- 'code/game/world.dm'
- 'code/modules/tgs/**'
- 'tools/tgs_scripts/**'
- 'tools/tgs_test/**'
pull_request:
branches:
- master
paths:
- '.tgs.yml'
- '.github/workflows/tgs_test.yml'
- '_build_dependencies.sh'
- 'code/__DEFINES/tgs.config.dm'
- 'code/__DEFINES/tgs.dm'
- 'code/game/world.dm'
- 'code/modules/tgs/**'
- 'tools/tgs_scripts/**'
- 'tools/tgs_test/**'
merge_group:
branches:
- master
env:
TGS_API_PORT: 5000
PR_NUMBER: ${{ github.event.number }}
jobs:
test_tgs_docker:
if: ( !contains(github.event.head_commit.message, '[ci skip]') )
name: Test TGS Docker
runs-on: ubuntu-22.04
concurrency:
group: test_tgs_docker-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
services:
tgs:
image: tgstation/server
env:
Database__DatabaseType: Sqlite
Database__ConnectionString: Data Source=TGS_TGTest.sqlite3;Mode=ReadWriteCreate
General__ConfigVersion: 5.0.0
General__ApiPort: ${{ env.TGS_API_PORT }}
General__SetupWizardMode: Never
ports:
- 5000:5000 #Can't use env here for some reason
steps:
- name: Setup dotnet
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.0.x
- name: Checkout Repository
uses: actions/checkout@v4
- name: Test TGS Integration
run: dotnet run -c Release --project tools/tgs_test ${{ github.repository }} /tgs_instances/chompstation ${{ env.TGS_API_PORT }} ${{ github.event.pull_request.head.sha || github.sha }} ${{ secrets.GITHUB_TOKEN }} ${{ env.PR_NUMBER }}

View File

@@ -14,10 +14,10 @@ static_files:
- name: data
# String dictionary. The value is the location of the file in the repo to upload to TGS. The key is the name of the file to upload to "<instance_path>/Configuration/EventScripts/"
# This one is for Linux hosted servers
#linux_scripts:
# PreCompile.sh: tools/tgs_scripts/PreCompile.sh
# WatchdogLaunch.sh: tools/tgs_scripts/WatchdogLaunch.sh
# InstallDeps.sh: tools/tgs_scripts/InstallDeps.sh
linux_scripts:
PreCompile.sh: tools/tgs_scripts/PreCompile.sh
WatchdogLaunch.sh: tools/tgs_scripts/WatchdogLaunch.sh
InstallDeps.sh: tools/tgs_scripts/InstallDeps.sh
# Same as above for Windows hosted servers
windows_scripts:
PreCompile.bat: tools/tgs_scripts/PreCompile.bat

2
BUILD.cmd Normal file
View File

@@ -0,0 +1,2 @@
@echo off
call "%~dp0\tools\build\build.bat" --wait-on-error build %*

View File

@@ -8,3 +8,6 @@ beautifulsoup4==4.9.3
# ezdb
mysql-connector-python==8.0.33
# icon cutter
numpy==1.26.0

View File

@@ -0,0 +1,33 @@
#!/bin/bash
#find out what we have (+e is important for this)
set +e
has_git="$(command -v git)"
has_curl="$(command -v curl)"
has_cargo="$(command -v ~/.cargo/bin/cargo)"
has_sudo="$(command -v sudo)"
# FIXME: yt-dlp
has_pip3="$(command -v pip3)"
set -e
set -x
# apt packages, libssl needed by rust-g but not included in TGS barebones install
if ! ( [ -x "$has_git" ] && [ -x "$has_curl" ] && [ -f "/usr/lib/i386-linux-gnu/libssl.so" ] ); then
echo "Installing apt dependencies..."
if ! [ -x "$has_sudo" ]; then
dpkg --add-architecture i386
apt-get update
apt-get install -y lib32z1 git pkg-config libssl-dev:i386 libssl-dev zlib1g-dev:i386 curl
else
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get install -y lib32z1 git pkg-config libssl-dev:i386 libssl-dev zlib1g-dev:i386 curl
fi
fi
# install cargo if needed
if ! [ -x "$has_cargo" ]; then
echo "Installing rust..."
curl https://sh.rustup.rs -sSf | sh -s -- -y
. ~/.profile
fi

View File

@@ -1,34 +1,15 @@
@echo off
FOR /F "tokens=* USEBACKQ" %%F IN (`powershell -NoLogo -ExecutionPolicy Bypass -File "get_dependencies.ps1"`) DO (
SET RUST_G_VERSION=%%F
)
SET "original_dir=%CD%"
cd "%~1"
::set "RUST_G_VERSION=3.1.0"
cd "%original_dir%"
IF NOT exist "rust-g"\ (
echo "Cloning rust-g..."
git "clone" "https://github.com/tgstation/rust-g"
cd "rust-g"
rustup "target" "add" "i686-pc-windows-msvc"
cd /D "%~dp0"
set TG_BOOTSTRAP_CACHE=%cd%
IF NOT %1 == "" (
rem TGS4+: we are passed the game directory on the command line
cd %1
) ELSE IF EXIST "..\Game\B\tgstation.dmb" (
rem TGS3: Game/B/tgstation.dmb exists, so build in Game/A
cd ..\Game\A
) ELSE (
echo "Fetching rust-g..."
cd "rust-g"
git "fetch"
rustup "target" "add" "i686-pc-windows-msvc"
rem TGS3: Otherwise build in Game/B
cd ..\Game\B
)
echo "Deploying rust-g..."
git "checkout" "%RUST_G_VERSION%"
set PKG_CONFIG_ALLOW_CROSS=1
cargo build --release --target=i686-pc-windows-msvc
move "%CD%\target\i686-pc-windows-msvc\release\rust_g.dll" "%~1/rust_g.dll"
cd ".."
echo "Compiling tgui..."
cd "%~1"
set TG_BOOTSTRAP_CACHE=%original_dir%
set "CBT_BUILD_MODE=TGS"
tools/bootstrap/node.bat tools/build/build.js
set CBT_BUILD_MODE=TGS
tools\build\build

View File

@@ -1,5 +1,7 @@
#!/bin/bash
./InstallDeps.sh
set -e
set -x
@@ -26,7 +28,7 @@ fi
echo "Deploying rust-g..."
git checkout "$RUST_G_VERSION"
env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --release --target=i686-unknown-linux-gnu
env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --ignore-rust-version --release --target=i686-unknown-linux-gnu
mv target/i686-unknown-linux-gnu/release/librust_g.so "$1/librust_g.so"
cd ..

View File

@@ -0,0 +1,6 @@
#!/bin/bash
# Special file to ensure all dependencies still exist between server launches.
# Mainly for use by people who abuse docker by modifying the container's system.
./InstallDeps.sh

View File

@@ -1,14 +0,0 @@
function Extract-Variable {
param([string] $Path, [string] $Key)
foreach ($Line in Get-Content $Path) {
if ($Line.StartsWith("export $Key=")) {
return $Line.Substring("export $Key=".Length)
}
}
throw "Couldn't find value for $Key in $Path"
}
$BaseDir = Split-Path $script:MyInvocation.MyCommand.Path
$RustgVersion = Extract-Variable -Path "$BaseDir\..\..\_build_dependencies.sh" -Key "RUST_G_VERSION"
return $RustgVersion

366
tools/tgs_test/Program.cs Normal file
View File

@@ -0,0 +1,366 @@
// Simple app meant to test chompstation's TGS integration given a fresh TGS install with the default account
//
// Args: Repository Owner/Name, TGS instance path, TGS API port, Pushed commit hash (For .tgs.yml access), GitHub Token, (OPTIONAL) PR Number
using System.Reflection;
using System.Text;
using Octokit;
using Tgstation.Server.Api;
using Tgstation.Server.Api.Models.Request;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Client;
using Tgstation.Server.Common.Extensions;
using YamlDotNet.Serialization.NamingConventions;
using YamlDotNet.Serialization;
Console.WriteLine("Parsing args...");
if (args.Length < 5 || args.Length > 6)
{
Console.WriteLine($"Incorrect number of args: {args.Length}. Expected 5-6");
return 1;
}
var repoSlug = args[0];
var instancePath = args[1];
var tgsApiPortString = args[2];
var pushedCommitHash = args[3];
var gitHubToken = args[4];
int? pullRequest = default;
if(args.Length == 6)
{
if (!Int32.TryParse(args[5], out int prNumber))
{
Console.WriteLine($"Invalid repo slug: {repoSlug}");
return 10;
}
pullRequest = prNumber;
}
var repoSlugSplits = repoSlug.Split('/', StringSplitOptions.RemoveEmptyEntries);
if(repoSlugSplits.Length != 2)
{
Console.WriteLine($"Invalid repo slug: {repoSlug}");
return 2;
}
var repoOwner = repoSlugSplits[0];
var repoName = repoSlugSplits[1];
if (!ushort.TryParse(tgsApiPortString, out var tgsApiPort))
{
Console.WriteLine($"Invalid port: {tgsApiPortString}");
return 3;
}
try
{
Console.WriteLine($"Retrieving .tgs.yml (@{pushedCommitHash})...");
var assemblyName = Assembly.GetExecutingAssembly().GetName();
var gitHubClient = new GitHubClient(
new ProductHeaderValue(
assemblyName.Name,
assemblyName.Version!.Semver().ToString()))
{
Credentials = new Credentials(gitHubToken)
};
var tgsYmlContent = await gitHubClient.Repository.Content.GetRawContentByRef(repoOwner, repoName, ".tgs.yml", pushedCommitHash);
var tgsYmlString = Encoding.UTF8.GetString(tgsYmlContent);
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new UnderscoredNamingConvention())
.Build();
var tgsYml = deserializer.Deserialize<TgsYml>(tgsYmlString);
const int SupportedTgsYmlVersion = 1;
if (tgsYml.Version != SupportedTgsYmlVersion)
{
Console.WriteLine($"Unsupported .tgs.yml version: {tgsYml.Version}. Expected {SupportedTgsYmlVersion}");
return 4;
}
var targetByondVersion = Version.Parse(tgsYml.Byond);
Console.WriteLine($".tgs.yml Security level: {tgsYml.Security}");
Console.WriteLine("Downloading and checking BYOND version in _build_dependencies.sh...");
var dependenciesShContent = await gitHubClient.Repository.Content.GetRawContentByRef(repoOwner, repoName, "_build_dependencies.sh", pushedCommitHash);
var dependenciesSh = Encoding.UTF8.GetString(dependenciesShContent);
var dependenciesShLines = dependenciesSh.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
int dependenciesShByondMajor = 0;
int dependenciesShByondMinor = 0;
foreach(var dependenciesShLine in dependenciesShLines)
{
var trimmedLine = dependenciesShLine.Trim();
var lineSplit = trimmedLine.Split('=', StringSplitOptions.RemoveEmptyEntries);
if (lineSplit.Length != 2)
continue;
if (lineSplit[0].EndsWith("BYOND_MAJOR"))
dependenciesShByondMajor = Int32.Parse(lineSplit[1]);
else if (lineSplit[0].EndsWith("BYOND_MINOR"))
dependenciesShByondMinor = Int32.Parse(lineSplit[1]);
}
var dependenciesByondVersion = new Version(dependenciesShByondMajor, dependenciesShByondMinor);
if(dependenciesByondVersion != targetByondVersion)
{
Console.WriteLine($".tgs.yml BYOND version does not match _build_dependencies.sh! Expected {dependenciesByondVersion} got {targetByondVersion}!");
return 5;
}
// Connect to TGS
var clientFactory = new ServerClientFactory(
new System.Net.Http.Headers.ProductHeaderValue(
assemblyName.Name!,
assemblyName.Version!.Semver().ToString()));
var tgsApiUrl = new Uri($"http://127.0.0.1:{tgsApiPort}");
var giveUpAt = DateTimeOffset.UtcNow.AddMinutes(2);
IServerClient client;
for (var I = 1; ; ++I)
{
try
{
Console.WriteLine($"TGS Connection Attempt {I}...");
client = await clientFactory.CreateFromLogin(
tgsApiUrl,
DefaultCredentials.AdminUserName,
DefaultCredentials.DefaultAdminUserPassword);
break;
}
catch (HttpRequestException)
{
//migrating, to be expected
if (DateTimeOffset.UtcNow > giveUpAt)
throw;
await Task.Delay(TimeSpan.FromSeconds(1));
}
catch (ServiceUnavailableException)
{
// migrating, to be expected
if (DateTimeOffset.UtcNow > giveUpAt)
throw;
await Task.Delay(TimeSpan.FromSeconds(1));
}
}
Console.WriteLine("Getting TGS information...");
var tgsInfo = await client.ServerInformation(default);
var scriptDictionaryToUse = tgsInfo.WindowsHost ? tgsYml.WindowsScripts : tgsYml.LinuxScripts;
Console.WriteLine($"Downloading {scriptDictionaryToUse.Count} EventScripts...");
var scriptDownloadTasks = new Dictionary<string, Task<byte[]>>();
foreach (var scriptKvp in scriptDictionaryToUse)
{
scriptDownloadTasks.Add(
scriptKvp.Key,
gitHubClient.Repository.Content.GetRawContentByRef(repoOwner, repoName, scriptKvp.Value, pushedCommitHash));
}
await Task.WhenAll(scriptDownloadTasks.Values);
Console.WriteLine("Setting up TGS instance...");
var instance = await client.Instances.CreateOrAttach(
new InstanceCreateRequest
{
ConfigurationType = ConfigurationType.HostWrite,
Name = "chompstation",
Path = instancePath
},
default);
instance = await client.Instances.Update(
new InstanceUpdateRequest
{
Id = instance.Id,
Online = true
},
default);
var instanceClient = client.Instances.CreateClient(instance);
Console.WriteLine("Cloning main branch of repo...");
var repoCloneJob = await instanceClient.Repository.Clone(
new RepositoryCreateRequest
{
Origin = new Uri($"http://github.com/{repoSlug}"),
UpdateSubmodules = true,
AccessUser = "Testing",
AccessToken = gitHubToken
},
default);
Console.WriteLine("Installing BYOND...");
var byondInstallJob = await instanceClient.Engine.SetActiveVersion(
new EngineVersionRequest
{
EngineVersion = new EngineVersion
{
Version = targetByondVersion,
Engine = EngineType.Byond,
}
},
null,
default);
Console.WriteLine("Updating server/compiler settings...");
await instanceClient.DreamMaker.Update(
new DreamMakerRequest
{
ApiValidationSecurityLevel = tgsYml.Security
},
default);
await instanceClient.DreamDaemon.Update(
new DreamDaemonRequest
{
SecurityLevel = tgsYml.Security,
Visibility = DreamDaemonVisibility.Invisible
},
default);
Console.WriteLine("Uploading EventScripts...");
foreach (var scriptDownloadKvp in scriptDownloadTasks)
{
var scriptContent = await scriptDownloadKvp.Value;
var memoryStream = new MemoryStream(scriptContent);
await instanceClient.Configuration.Write(
new ConfigurationFileRequest
{
Path = $"EventScripts/{scriptDownloadKvp.Key}"
},
memoryStream,
default);
}
Console.WriteLine("Creating GameStaticFiles structure...");
var staticFileDownloadTasks = new Dictionary<string, Dictionary<string, Task<byte[]>>>();
foreach (var staticFile in tgsYml.StaticFiles)
{
if (!staticFile.Populate)
{
Console.WriteLine($"Creating empty directory GameStaticFiles/{staticFile.Name}...");
await instanceClient.Configuration.CreateDirectory(new ConfigurationFileRequest
{
Path = $"GameStaticFiles/{staticFile.Name}"
},
default);
}
else
{
// not by ref here as we are relying on master being not broken
Console.WriteLine($"Enumerating repo path {staticFile.Name}...");
var repositoryFilesToUpload = new Queue<RepositoryContent>(await gitHubClient.Repository.Content.GetAllContents(repoOwner, repoName, staticFile.Name));
while (repositoryFilesToUpload.Count != 0)
{
var repositoryFileToUpload = repositoryFilesToUpload.Dequeue();
if (repositoryFileToUpload.Type == ContentType.File)
{
// serial because easier to track errors
Console.WriteLine($"Transferring {repositoryFileToUpload.Path}...");
var fileContent = await gitHubClient.Repository.Content.GetRawContent(repoOwner, repoName, repositoryFileToUpload.Path);
using var memoryStream = new MemoryStream(fileContent);
await instanceClient.Configuration.Write(new ConfigurationFileRequest
{
Path = $"GameStaticFiles/{repositoryFileToUpload.Path}"
},
memoryStream,
default);
}
else
{
Console.WriteLine($"Enumerating repo path {repositoryFileToUpload.Path}...");
var additionalFiles = await gitHubClient.Repository.Content.GetAllContents(repoOwner, repoName, repositoryFileToUpload.Path);
foreach (var additionalFile in additionalFiles)
repositoryFilesToUpload.Enqueue(additionalFile);
}
}
}
}
async Task<bool> WaitForJob(JobResponse originalJob, int timeout)
{
Console.WriteLine($"Waiting for job \"{originalJob.Description}\"...");
var job = originalJob;
var previousProgress = job.Progress;
do
{
if (job.Progress != previousProgress)
Console.WriteLine($"Progress: {previousProgress = job.Progress}");
await Task.Delay(TimeSpan.FromSeconds(1));
job = await instanceClient!.Jobs.GetId(job, default);
--timeout;
}
while (!job.StoppedAt.HasValue && timeout > 0);
if (!job.StoppedAt.HasValue)
{
await instanceClient!.Jobs.Cancel(job, default);
Console.WriteLine($"Timed out!");
return false;
}
else if (job.ExceptionDetails != null)
{
Console.WriteLine($"Error: {job.ExceptionDetails}");
return false;
}
return true;
}
if (!await WaitForJob(byondInstallJob.InstallJob!, 120))
return 6;
if (!await WaitForJob(repoCloneJob.ActiveJob!, 600))
return 7;
if (pullRequest.HasValue)
{
Console.WriteLine($"Applying test merge #{pullRequest}...");
var testMergeJob = await instanceClient.Repository.Update(new RepositoryUpdateRequest
{
NewTestMerges = new List<TestMergeParameters>
{
new TestMergeParameters
{
Comment = "Active Pull Request",
Number = pullRequest.Value,
TargetCommitSha = pushedCommitHash
}
}
}, default);
if (!await WaitForJob(testMergeJob.ActiveJob!, 60))
return 11;
}
Console.WriteLine("Deploying...");
var deploymentJob = await instanceClient.DreamMaker.Compile(default);
if (!await WaitForJob(deploymentJob, 1800))
return 8;
Console.WriteLine("Launching...");
var launchJob = await instanceClient.DreamDaemon.Start(default);
if (!await WaitForJob(launchJob, 300))
return 9;
return 0;
}
catch (Exception ex)
{
Console.WriteLine(ex);
return 4;
}

11
tools/tgs_test/README.md Normal file
View File

@@ -0,0 +1,11 @@
# TGS Test Script
This is a simple app that does a few things
- Downloads .tgs.yml information from a specific commit of a given repository.
- Checks that the BYOND version in the .tgs.yml file matches the dependencies.sh version.
- Connects to a TGS instance via command line parameters.
- Uses the .tgs.yml information to automatically set up a TGS instance.
- Runs a TGS deploy/launch and validates that they succeeded.
Look for its invocation in the GitHub workflows

View File

@@ -0,0 +1,5 @@
sealed class StaticFile
{
public string Name { get; set; } = String.Empty;
public bool Populate { get; set; }
}

15
tools/tgs_test/TgsYml.cs Normal file
View File

@@ -0,0 +1,15 @@
using Tgstation.Server.Api.Models;
sealed class TgsYml
{
public int Version { get; set; }
public string Byond { get; set; } = String.Empty;
public List<StaticFile> StaticFiles { get; set; } = new List<StaticFile>();
public Dictionary<string, string> WindowsScripts { get; set; } = new Dictionary<string, string>();
public Dictionary<string, string> LinuxScripts { get; set; } = new Dictionary<string, string>();
public DreamDaemonSecurity Security { get; set; }
}

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Version>2.0.0</Version>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="Octokit" Version="9.0.0" />
<PackageReference Include="Tgstation.Server.Client" Version="15.0.0" />
<PackageReference Include="YamlDotNet.NetCore" Version="1.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,25 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.4.33213.308
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tgstation.TgsTest", "Tgstation.TgsTest.csproj", "{3146D745-AAE5-4205-8FF2-0CE471B47B4E}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{3146D745-AAE5-4205-8FF2-0CE471B47B4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3146D745-AAE5-4205-8FF2-0CE471B47B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3146D745-AAE5-4205-8FF2-0CE471B47B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3146D745-AAE5-4205-8FF2-0CE471B47B4E}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {001721B4-7740-419D-837E-26CE9DABCAB8}
EndGlobalSection
EndGlobal