Merge pull request #1887 from tgstation/PostReleaseThings [TGSDeploy]

Mostly just CI improvements
This commit is contained in:
Jordan Dominion
2024-08-20 18:51:55 -04:00
committed by GitHub
9 changed files with 123 additions and 70 deletions
+12 -4
View File
@@ -1,10 +1,18 @@
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "nuget"
directory: "/"
- package-ecosystem: nuget
directory: /
schedule:
interval: "daily"
interval: daily
labels:
- "Dependencies"
- Dependencies
open-pull-requests-limit: 100
- package-ecosystem: github-actions
directory: "/"
target-branch: master
schedule:
interval: daily
labels:
- Dependencies
- CI/CD
+53 -34
View File
@@ -67,7 +67,7 @@ jobs:
with:
ref: "refs/pull/${{ inputs.pull_request_number }}/merge"
- name: Build ReleaseNotes
- name: Build ReleaseNotes # Name checked in rerunFlakyTests.js
run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj
- name: Store ReleaseNotes Binaries
@@ -108,7 +108,7 @@ jobs:
- name: Setup Telemetry Key File
run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
- name: Build # Name checked in rerunFlakyTests.js
run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true
- name: Perform CodeQL Analysis
@@ -124,7 +124,7 @@ jobs:
byond: [ '510.1346', '511.1385', '512.1488', '513.1542', '514.1589', 'EDGE' ]
runs-on: ubuntu-latest
steps:
- name: Install x86 libc Dependencies
- name: Install Native x86 libc Dependencies # Name checked in rerunFlakyTests.js
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
@@ -137,7 +137,7 @@ jobs:
path: ~/byond-zips-cache
key: byond-zips
- name: Build BYOND Cache if Necessary and Install
- name: Setup BYOND Cache if Necessary and Install
run: |
echo "Setting up BYOND."
FULL_VERSION=${{ matrix.byond }}
@@ -184,7 +184,7 @@ jobs:
run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
- name: Build DMAPI Test Project
- name: Compile DMAPI Test Project
run: |
set -e
retval=1
@@ -244,17 +244,17 @@ jobs:
git checkout ${{ matrix.committish }}
git submodule update --init --recursive
- name: Build OpenDream
- name: Build OpenDream # Name checked in rerunFlakyTests.js
run: |
cd $HOME/OpenDream/OpenDreamPackageTool
dotnet build -c Release --nologo -v q --property WarningLevel=0 /clp:ErrorsOnly
- name: Create TGS Deployment
- name: Build TGS Deployment # Name checked in rerunFlakyTests.js
run: |
cd $HOME/OpenDream
dotnet run -c Release --project OpenDreamPackageTool --no-build -- --tgs -o tgs_deploy
- name: Build DMAPI
- name: Compile DMAPI Test Project
run: |
cd tests/DMAPI/BasicOperation
$HOME/OpenDream/tgs_deploy/bin/compiler/DMCompiler --verbose --notices-enabled "basic operation_test.dme"
@@ -345,7 +345,7 @@ jobs:
name: release_notes_bins
path: release_notes_bins
- name: Build Changelog (Incremental)
- name: Compile Changelog (Incremental)
run: |
mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null
dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes
@@ -410,7 +410,7 @@ jobs:
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Docker Image
- name: Build Docker Image # Name checked in rerunFlakyTests.js
run: docker build . -f build/Dockerfile --build-arg TGS_TELEMETRY_KEY_FILE=${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Delete Telemetry Key File
@@ -429,7 +429,7 @@ jobs:
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
runs-on: ubuntu-latest
steps:
- name: Install x86 libc Dependencies
- name: Install Native x86 libc Dependencies # Name checked in rerunFlakyTests.js
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
@@ -462,7 +462,7 @@ jobs:
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
- name: Build # Name checked in rerunFlakyTests.js
run: dotnet build -c ${{ matrix.configuration }}NoWindows
- name: Delete Telemetry Key File
@@ -525,7 +525,7 @@ jobs:
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
- name: Build # Name checked in rerunFlakyTests.js
run: dotnet build -c ${{ matrix.configuration }}NoWix
- name: Delete Telemetry Key File
@@ -656,7 +656,7 @@ jobs:
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
- name: Build # Name checked in rerunFlakyTests.js
run: dotnet build -c ${{ matrix.configuration }} tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj
- name: Delete Telemetry Key File
@@ -820,7 +820,7 @@ jobs:
- name: Disable ptrace_scope
run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
- name: Install Native Dependencies
- name: Install Native Dependencies # Name checked in rerunFlakyTests.js
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
@@ -876,7 +876,7 @@ jobs:
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
- name: Build # Name checked in rerunFlakyTests.js
run: dotnet build -c ${{ matrix.configuration }}NoWindows tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj
- name: Delete Telemetry Key File
@@ -1224,24 +1224,25 @@ jobs:
build-deb:
name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595
needs: build-releasenotes
runs-on: ubuntu-latest
env:
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
steps:
- name: Install Native Dependencies
- name: Install Native Dependencies # Name checked in rerunFlakyTests.js
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386
- name: Import GPG Key
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))
if: github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master'
run: |
echo "${{ secrets.PACKAGING_PRIVATE_KEY }}" > private.pgp
echo ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} | gpg --batch --yes --passphrase-fd 0 --import private.pgp
rm private.pgp
- name: Install dotnet-sdk system package
- name: Install Native dotnet-sdk Package # Name checked in rerunFlakyTests.js
if: (!contains(env.TGS_DOTNET_QUALITY, 'preview'))
run: |
sudo apt-get update
@@ -1276,13 +1277,22 @@ jobs:
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Retrieve ReleaseNotes Binaries
uses: actions/download-artifact@v4
with:
name: release_notes_bins
path: release_notes_bins
- name: Execute Build Script (Unsigned)
if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master'))
env:
RELEASE_NOTES_DLL_PATH: ${{ github.workspace }}/release_notes_bins/Tgstation.Server.ReleaseNotes.dll
run: sudo -E build/package/deb/build_package.sh
- name: Execute Build Script (Signed)
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')
env:
RELEASE_NOTES_DLL_PATH: ${{ github.workspace }}/release_notes_bins/Tgstation.Server.ReleaseNotes.dll
PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }}
run: sudo -E build/package/deb/build_package.sh
@@ -1383,7 +1393,7 @@ jobs:
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Host
- name: Build Host # Name checked in rerunFlakyTests.js
run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj
- name: Delete Telemetry Key File
@@ -1391,14 +1401,14 @@ jobs:
if: always()
run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Service
- name: Build Service # Name checked in rerunFlakyTests.js
run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj
- name: Prepare Artifacts
shell: powershell
run: build/package/winget/prepare_installer_input_artifacts.ps1
- name: Build Installer .exe
- name: Build Installer .exe # Name checked in rerunFlakyTests.js
run: |
cd build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle
dotnet build -c Release
@@ -1789,7 +1799,8 @@ jobs:
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Host # We need to rebuild the installer.exe so it can be properly signed
# We need to rebuild the installer.exe so it can be properly signed
- name: Build Host # Name checked in rerunFlakyTests.js
run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj
- name: Delete Telemetry Key File
@@ -1797,14 +1808,14 @@ jobs:
if: always()
run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Service
- name: Build Service # Name checked in rerunFlakyTests.js
run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj
- name: Prepare Artifacts
shell: powershell
run: build/package/winget/prepare_installer_input_artifacts.ps1
- name: Build Installer .exe
- name: Build Installer .exe # Name checked in rerunFlakyTests.js
run: |
cd build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle
dotnet build -c Release
@@ -1995,6 +2006,7 @@ jobs:
changelog-regen:
name: Regenerate Changelog
runs-on: ubuntu-latest
if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
needs: deploy-tgs
steps:
- name: Setup dotnet
@@ -2012,7 +2024,7 @@ jobs:
name: release_notes_bins
path: release_notes_bins
- name: Build Changelog (Incremental)
- name: Compile Changelog (Incremental)
run: |
mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null
dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes
@@ -2042,16 +2054,19 @@ jobs:
deploy-docker:
name: Deploy TGS (Docker)
needs: deploy-tgs
if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
runs-on: ubuntu-latest
steps:
- name: Install Native Packages # Name checked in rerunFlakyTests.js
run: |
sudo apt-get update
sudo apt-get install -y xmlstarlet
- name: Checkout
uses: actions/checkout@v4
- name: Parse TGS version
run: |
sudo apt-get update
sudo apt-get install -y xmlstarlet
echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
run: echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
- name: Docker Build and Push
uses: elgohr/Publish-Docker-Github-Action@43dc228e327224b2eda11c8883232afd5b34943b # v5
@@ -2065,16 +2080,19 @@ jobs:
deploy-ppa:
name: Deploy TGS (PPA)
needs: deploy-tgs
if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
runs-on: ubuntu-latest
steps:
- name: Install Native Packages # Name checked in rerunFlakyTests.js
run: |
sudo apt-get update
sudo apt-get install -y xmlstarlet
- name: Checkout
uses: actions/checkout@v4
- name: Parse TGS version
run: |
sudo apt-get update
sudo apt-get install -y xmlstarlet
echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
run: echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
- name: Trigger tgstation-ppa workflow
run: |
@@ -2083,6 +2101,7 @@ jobs:
deploy-winget:
name: Deploy TGS (winget)
needs: deploy-tgs
if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
runs-on: windows-latest
steps:
- name: Setup dotnet
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333
uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308
with:
message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run and the ci-pipeline.yml at the HEAD of the pull request will be used.
message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml and ci-pipeline.yml will not be reflected.
- name: Comment on dependabot PR
if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333
@@ -0,0 +1,25 @@
name: 'Automerge Dependabot'
on:
pull_request_target:
types: [ opened, reopened ]
branches:
- dev
- master
concurrency:
group: "dependabot-automerge-${{ github.head_ref || github.run_id }}-${{ github.event_name }}"
cancel-in-progress: true
jobs:
automerge:
name: Enable Automerge on Dependabot PRs
runs-on: ubuntu-latest
if: github.event.pull_request.user.id == 49699333
permissions:
pull-requests: write
steps:
- name: Enable Automerge
uses: daneden/enable-automerge-action@f8558b65c5b8d8bfb592c4e74e3d491624a38fbd #v1.0.2-ish
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
+18 -6
View File
@@ -1,10 +1,17 @@
// Only check jobs that start with these.
const MAX_ATTEMPTS = 5;
// If any job fails with a step starting with these it'll be considered flaky
const CONSIDERED_STEP_PREFIXES = [
"Build", // Nuget.org sporadic issues
"Install Native", // apt repository issues
];
// Otherwise only check jobs that start with these.
// Helps make sure we don't restart something like which is not known to be flaky.
const CONSIDERED_JOBS = [
"Windows Live Tests",
"Linux Live Tests",
"Build .deb Package",
"Upload Code Coverage"
"Build .deb Package"
];
async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) {
@@ -25,6 +32,10 @@ async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) {
}
export async function rerunFlakyTests({ github, context }) {
if (context.payload.workflow_run.run_attempt >= MAX_ATTEMPTS) {
console.log(`Jobs have reached maximum attempt count of ${context.payload.workflow_run.run_attempt}, Will not re-run.`)
}
const failingJobs = await getFailedJobsForRun(
github,
context,
@@ -33,7 +44,7 @@ export async function rerunFlakyTests({ github, context }) {
);
if (failingJobs.length > 3) {
console.log("Many jobs failing. PROBABLY not flaky, not rerunning.");
console.log("Many jobs failing. PROBABLY not flaky, Will not re-run.");
return;
}
@@ -41,10 +52,11 @@ export async function rerunFlakyTests({ github, context }) {
console.log(`Failing job: ${job.name}`)
return CONSIDERED_JOBS
.flatMap(jobName => [jobName, 'CI Pipeline / ' + jobName])
.some((title) => job.name.startsWith(title));
.some((title) => job.name.startsWith(title))
|| CONSIDERED_STEP_PREFIXES.some(consideredStep => job.steps.some(step => step.name.startsWith(consideredStep) && step.conclusion == "failure"));
});
if (filteredFailingJobs.length !== failingJobs.length) {
console.log("One or more failing jobs are NOT designated flaky. Not rerunning.");
console.log("One or more failing jobs are NOT designated flaky. Will not re-run.");
return;
}
+4
View File
@@ -2,6 +2,10 @@ name: Size Labelling
on:
pull_request_target:
concurrency:
group: "size-labelling-${{ github.head_ref || github.run_id }}-${{ github.event_name }}"
cancel-in-progress: true
jobs:
size-label:
name: Add Size Label
+5 -1
View File
@@ -59,7 +59,11 @@ dh_make -p tgstation-server_$TGS_VERSION -y --createorig -s
rm -f debian/README* debian/changelog debian/*.ex debian/upstream/*.ex
pushd ..
dotnet run -c Release -p:TGS_HOST_NO_WEBPANEL=true --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $CURRENT_COMMIT
if [[ -z "$RELEASE_NOTES_DLL_PATH" ]]; then
dotnet run -c Release -p:TGS_HOST_NO_WEBPANEL=true --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $CURRENT_COMMIT
else
dotnet $RELEASE_NOTES_DLL_PATH $TGS_VERSION --debian packaging/debian/changelog $CURRENT_COMMIT
fi
popd
cp -r build/package/deb/debian/* debian/
+4
View File
@@ -9,3 +9,7 @@ This is a series of README.md files aimed at directing people around the codebas
- To explore the C# client code navigate to [Tgstation.Server.Client](./Tgstation.Server.Client).
[Tgstation.Server.Host.Watchdog](./Tgstation.Server.Host.Watchdog), [Tgstation.Server.Host.Service](./Tgstation.Server.Host.Service), and [Tgstation.Server.Host.Console](./Tgstation.Server.Host.Console) are related to the Service/Console runners which have the simple task of executing Tgstation.Server.Host and updating it when requested.
[Tgstation.Server.Common](./Tgstation.Server.Common) are functions and dependencies shared publically (Published to Nuget).
[Tgstation.Server.Shared](./Tgstation.Server.Shared) are functions and dependencies shared internally (Not published to Nuget).
+1 -24
View File
@@ -60,7 +60,6 @@ namespace Tgstation.Server.ReleaseNotes
var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase);
var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase);
var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase);
var waitCodecov = versionString.Equals("--wait-codecov", StringComparison.OrdinalIgnoreCase);
if ((!Version.TryParse(versionString, out var version) || version.Revision != -1)
&& !ensureRelease
@@ -68,8 +67,7 @@ namespace Tgstation.Server.ReleaseNotes
&& !shaCheck
&& !fullNotes
&& !nuget
&& !genToken
&& !waitCodecov)
&& !genToken)
{
Console.WriteLine("Invalid version: " + versionString);
return 2;
@@ -157,11 +155,6 @@ namespace Tgstation.Server.ReleaseNotes
client.Credentials = new Credentials(githubToken);
}
if (waitCodecov)
{
return await CodecovCheck(client, Int64.Parse(args[1]));
}
if (linkWinget)
{
if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl))
@@ -1678,21 +1671,5 @@ package (version) distribution(s); urgency=urgency
else
Debug.Assert(condition);
}
static async ValueTask<int> CodecovCheck(IGitHubClient client, long runId)
{
var currentRun = await client.Actions.Workflows.Runs.Get(RepoOwner, RepoName, runId);
bool foundRun = false;
for(int i = 0; i < 15 && !foundRun; ++i)
{
var allRuns = await client.Check.Run.GetAllForReference(RepoOwner, RepoName, currentRun.HeadSha);
foundRun = allRuns.CheckRuns.Any(x => x.CheckSuite.Id == currentRun.Id && x.Name == "codecov/project");
if (!foundRun && i != 14)
await Task.Delay(TimeSpan.FromMinutes(1));
}
return foundRun ? 0 : 24398;
}
}
}