diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 233c7fb51e..1c6c65e278 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -36,6 +36,8 @@ You can of course, as always, ask for help at [#coderbus](irc://irc.rizon.net/co You need the .NET 8.0 SDK, node>=v20, and npm>=v5.7 (in your PATH) to compile the server. On Linux, you also need the `libgdiplus` package installed to generate icons. +You need to run `corepack enable` to configure node to correctly build the webpanel. + The recommended IDE is Visual Studio 2022 or VSCode. In order to build the service version and/or the Windows installer you need a to run on Windows. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 54007f37f3..e70b298c56 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,90 +25,31 @@ on: branches: - dev - master - pull_request: - branches: - - dev - - master - pull_request_target: - types: [ opened, reopened, labeled, synchronize ] - branches: - - dev - - master + workflow_call: + inputs: + pull_request_number: + description: 'Pull Request Number' + required: true + type: string env: TGS_DOTNET_VERSION: 8 OD_MIN_COMPAT_DOTNET_VERSION: 7 OD_DOTNET_VERSION: 8 TGS_DOTNET_QUALITY: ga + TGS_WEBPANEL_NODE_VERSION: 20.x TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} concurrency: - group: "ci-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true jobs: - security-checkpoint: - name: Check CI Clearance + build-releasenotes: + name: Build ReleaseNotes for Other Jobs runs-on: ubuntu-latest - permissions: - pull-requests: write - if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' - steps: - - name: Comment on new Fork PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 - with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Please note that any changes to the workflow file will not be reflected in the run. - - - name: Comment on dependabot PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 - with: - message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. - - - name: "Remove Stale 'CI Cleared' Label" - if: github.event.action == 'synchronize' || github.event.action == 'reopened' - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - with: - labels: CI Cleared - - - name: "Remove 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - with: - labels: CI Approval Required - - - name: "Add 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 - with: - labels: CI Approval Required - github_token: ${{ github.token }} - - - name: Fail Clearance Check if PR has Unlabeled new Commits from User - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - run: exit 1 - - start-ci-run-gate: - name: CI Start Gate - needs: security-checkpoint - runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (needs.security-checkpoint.result == 'skipped' && (github.event_name == 'push' || github.event_name == 'schedule' || ((github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333) && github.event_name != 'pull_request_target'))))) - steps: - - name: GitHub Requires at Least One Step for a Job - run: exit 0 - - code-scanning: - name: Code Scanning - needs: start-ci-run-gate - runs-on: ubuntu-latest - permissions: - security-events: write - actions: read - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -124,13 +65,49 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Store ReleaseNotes Binaries + uses: actions/upload-artifact@v4 + with: + name: release_notes_bins + path: ./release_notes_bins/ + + code-scanning: + name: Run CodeQL + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout (Branch) + uses: actions/checkout@v4 + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: csharp + - name: Setup Telemetry Key File + run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true @@ -141,8 +118,6 @@ jobs: dmapi-build: name: Build DMAPI - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -200,7 +175,14 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Build DMAPI Test Project run: | @@ -224,8 +206,6 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -254,7 +234,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Checkout OpenDream run: | @@ -264,11 +244,6 @@ jobs: git checkout ${{ matrix.committish }} git submodule update --init --recursive - - name: Restore OpenDream - run: | - cd $HOME/OpenDream - dotnet restore - - name: Build OpenDream run: | cd $HOME/OpenDream/OpenDreamPackageTool @@ -287,8 +262,6 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -298,7 +271,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool @@ -344,9 +317,8 @@ jobs: pages-build: name: Build gh-pages + needs: build-releasenotes runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -362,25 +334,25 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -421,8 +393,8 @@ jobs: docker-build: name: Build Docker Image runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -432,15 +404,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Docker Image - run: docker build . -f build/Dockerfile + run: docker build . -f build/Dockerfile --build-arg TGS_TELEMETRY_KEY_FILE=${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} linux-unit-tests: name: Linux Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -448,6 +426,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: - name: Install x86 libc Dependencies @@ -462,6 +441,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -470,14 +454,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -496,8 +487,6 @@ jobs: windows-unit-tests: name: Windows Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -505,6 +494,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - name: Setup dotnet @@ -513,6 +503,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -521,14 +516,23 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWix + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -545,25 +549,19 @@ jobs: name: windows-unit-test-coverage-${{ matrix.configuration }} path: ./TestResults/ - windows-integration-test: + windows-integration-tests: name: Windows Live Tests - needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') + needs: [ dmapi-build, opendream-build ] strategy: fail-fast: false matrix: database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later - shell: powershell - if: ${{ matrix.database-type == 'SqlServer' }} - run: | - Write-Host "Checking" - sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" - - name: Setup dotnet uses: actions/setup-dotnet@v4 with: @@ -573,6 +571,18 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later + shell: powershell + if: ${{ matrix.database-type == 'SqlServer' }} + run: | + Write-Host "Checking" + sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Set TGS_TEST_DUMP_API_SPEC if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV @@ -637,14 +647,23 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }} tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -738,7 +757,6 @@ jobs: linux-integration-tests: name: Linux Live Tests needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') services: # We start all dbs here so we can just code the stuff once mssql: image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} @@ -786,8 +804,19 @@ jobs: database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + ${{ env.TGS_DOTNET_VERSION }}.0.x + ${{ env.OD_DOTNET_VERSION }}.0.x + ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -797,14 +826,10 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus - - name: Setup dotnet - uses: actions/setup-dotnet@v4 + - name: Setup Node.JS + uses: actions/setup-node@v4 with: - dotnet-version: | - ${{ env.TGS_DOTNET_VERSION }}.0.x - ${{ env.OD_DOTNET_VERSION }}.0.x - ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} - name: Set Sqlite Connection Info if: ${{ matrix.database-type == 'Sqlite' }} @@ -843,14 +868,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -907,8 +939,7 @@ jobs: validate-openapi-spec: name: OpenAPI Spec Validation - needs: windows-integration-test - if: (!(cancelled() || failure()) && needs.windows-integration-test.result == 'success') + needs: windows-integration-tests runs-on: ubuntu-latest steps: - name: Install IBM OpenAPI Validator @@ -922,7 +953,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 @@ -935,8 +966,7 @@ jobs: upload-code-coverage: name: Upload Code Coverage - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] - if: (!(cancelled() || failure()) && needs.linux-unit-tests.result == 'success' && needs.linux-integration-tests.result == 'success' && needs.windows-unit-tests.result == 'success' && needs.windows-integration-test.result == 'success') + needs: [ linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-tests, build-releasenotes ] runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -947,7 +977,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 @@ -1183,23 +1213,35 @@ jobs: name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Upload Coverage to CodeCov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: directory: ./code_coverage fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + handle_no_reports_found: true + + - name: Wait for CodeCov Status + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes --wait-codecov ${{ github.run_id }} build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 - needs: start-ci-run-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt steps: - name: Install Native Dependencies run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 xmlstarlet libgdiplus + sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 - name: Import GPG Key if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) @@ -1235,29 +1277,39 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Grab Most Recent Changelog + run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Execute Build Script (Unsigned) + if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')) + run: sudo -E build/package/deb/build_package.sh + + - name: Execute Build Script (Signed) + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') + env: + PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }} + run: sudo -E build/package/deb/build_package.sh - name: Parse TGS version run: | echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV - - name: Grab Most Recent Changelog - run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml - - - name: Execute Build Script (Unsigned) - if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) - run: sudo -E build/package/deb/build_package.sh - - - name: Execute Build Script (Signed) - if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) - env: - PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }} - run: | - sudo -E build/package/deb/build_package.sh + - name: Verify Package Files are Signed + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') + run: gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1.dsc gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.changes gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.buildinfo + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Test Install run: | sudo mkdir /etc/tgstation-server @@ -1295,9 +1347,9 @@ jobs: build-msi: name: Build Windows Installer .exe - needs: start-ci-run-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1310,6 +1362,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -1318,7 +1375,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Restore Wix dotnet Tool run: | @@ -1328,12 +1385,21 @@ jobs: - name: Validate winget Manifest run: winget validate --manifest build/package/winget/manifest - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -1426,8 +1492,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + needs: build-releasenotes runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1436,72 +1501,50 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Retrieve Latest winget-pkgs PULL_REQUEST_TEMPLATE commit SHA from GitHub API id: get-sha run: | curl -L -u "${{ vars.DEV_PUSH_USERNAME }}:${{ secrets.DEV_PUSH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -o commits.json https://api.github.com/repos/microsoft/winget-pkgs/commits?path=.github/PULL_REQUEST_TEMPLATE.md echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes Check - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} - ci-completion-gate: # This job exists so there isn't a moving target for branch protections + ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, code-scanning ] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success') steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - - name: Run ReleaseNotes Create CI Completion Check (PR HEAD) - if: github.event_name != 'push' && github.event_name != 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} - - - name: Run ReleaseNotes Create CI Completion Check (Branch) - if: github.event_name == 'push' || github.event_name == 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + - name: Mandatory Empty Step + run: exit 0 deployment-gate: name: Deployment Start Gate needs: ci-completion-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.ci-completion-gate.result == 'success' && github.event_name == 'push' && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) + if: github.event_name == 'push' steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 @@ -1510,7 +1553,7 @@ jobs: name: Deploy HTTP API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[APIDeploy]')) + if: contains(github.event.head_commit.message, '[APIDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1521,12 +1564,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse API version shell: powershell run: | @@ -1546,13 +1583,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_API_VERSION }} --httpapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_API_VERSION }} --httpapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1582,7 +1625,7 @@ jobs: name: Deploy DreamMaker API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[DMDeploy]')) + if: contains(github.event.head_commit.message, '[DMDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1593,12 +1636,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse DMAPI version shell: powershell run: | @@ -1617,13 +1654,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --dmapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_DM_VERSION }} --dmapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1653,7 +1696,7 @@ jobs: name: Deploy Nuget Packages needs: deployment-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[NugetDeploy]')) + if: contains(github.event.head_commit.message, '[NugetDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1664,17 +1707,17 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --nuget + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --nuget - name: Publish Tgstation.Server.Common to NuGet uses: alirezanet/publish-nuget@e276c40afeb2a154046f0997820f2a9ea74832d9 # v3.1.0 @@ -1704,7 +1747,7 @@ jobs: name: Ensure TGS Release is Latest GitHub Release needs: [deploy-dm, deploy-http] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.deploy-dm.result == 'success' || needs.deploy-http.result == 'success') && !contains(github.event.head_commit.message, '[TGSDeploy]')) + if: (!contains(github.event.head_commit.message, '[TGSDeploy]')) steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1712,23 +1755,22 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes with --ensure-release - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} deploy-tgs: name: Deploy TGS needs: [deploy-dm, deploy-http, deployment-gate] runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]')) + if: github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1736,28 +1778,37 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - name: Restore Wix dotnet Tool run: | cd build/package/winget dotnet tool restore -# We need to rebuild the installer.exe so it can be properly signed + - name: Enable Corepack + run: corepack enable - - name: Build Host + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Build Host # We need to rebuild the installer.exe so it can be properly signed run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj - - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Prepare Artifacts shell: powershell run: build/package/winget/prepare_installer_input_artifacts.ps1 @@ -1842,13 +1893,19 @@ jobs: &"C:/Program Files/7-Zip/7z.exe" a ServerConsole.zip ./ServerConsole/* -tzip &"C:/Program Files/7-Zip/7z.exe" a ServerUpdatePackage.zip ./ServerUpdatePackage/* -tzip + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_VERSION }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_VERSION }} - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1948,7 +2005,6 @@ jobs: name: Regenerate Changelog runs-on: ubuntu-latest needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1956,26 +2012,23 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -1998,7 +2051,6 @@ jobs: deploy-docker: name: Deploy TGS (Docker) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2022,7 +2074,6 @@ jobs: deploy-ppa: name: Deploy TGS (PPA) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2041,7 +2092,6 @@ jobs: deploy-winget: name: Deploy TGS (winget) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: windows-latest steps: - name: Setup dotnet @@ -2061,15 +2111,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes - - name: Retrieve Server Service uses: actions/download-artifact@v4 with: name: packaging-windows-raw-msi path: artifacts + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Execute Push Script shell: powershell run: build/package/winget/push_manifest.ps1 @@ -2082,4 +2135,4 @@ jobs: - name: Run ReleaseNotes with --link-winget shell: powershell - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml new file mode 100644 index 0000000000..d89d2d3055 --- /dev/null +++ b/.github/workflows/ci-security.yml @@ -0,0 +1,68 @@ +name: 'CI Security' + +on: + pull_request: + branches: + - dev + - master + pull_request_target: + types: [ opened, reopened, labeled, synchronize ] + branches: + - dev + - master + +concurrency: + group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + security-checkpoint: + name: Check CI Clearance + if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment on new Fork PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run and the ci-pipeline.yml at the HEAD of the pull request will be used. + + - name: Comment on dependabot PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. + + - name: "Remove Stale 'CI Cleared' Label" + if: github.event.action == 'synchronize' || github.event.action == 'reopened' + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Cleared + + - name: "Remove 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Approval Required + + - name: "Add 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 + with: + labels: CI Approval Required + github_token: ${{ github.token }} + + - name: Fail if PR has Unlabeled new Commits from User + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + run: exit 1 + + ci-pipline-workflow-call: + name: CI Pipeline + needs: security-checkpoint + if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (github.event_name != 'pull_request_target' && github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333))) + uses: ./.github/workflows/ci-pipeline.yml + secrets: inherit + with: + pull_request_number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/rerun-flaky-tests.yml b/.github/workflows/rerun-flaky-tests.yml index 9016637a49..806d89e359 100644 --- a/.github/workflows/rerun-flaky-tests.yml +++ b/.github/workflows/rerun-flaky-tests.yml @@ -1,7 +1,7 @@ name: Rerun Flaky Live Tests on: workflow_run: - workflows: [CI Pipeline] + workflows: ['CI Pipeline', 'CI Security'] types: - completed jobs: diff --git a/.github/workflows/scripts/rerunFlakyTests.js b/.github/workflows/scripts/rerunFlakyTests.js index 5bacef1278..7dda0f389a 100644 --- a/.github/workflows/scripts/rerunFlakyTests.js +++ b/.github/workflows/scripts/rerunFlakyTests.js @@ -3,7 +3,8 @@ const CONSIDERED_JOBS = [ "Windows Live Tests", "Linux Live Tests", - "Build .deb Package" + "Build .deb Package", + "Upload Code Coverage" ]; async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) { @@ -31,17 +32,19 @@ export async function rerunFlakyTests({ github, context }) { context.payload.workflow_run.run_attempt ); - if (failingJobs.length > 1) { - console.log("Multiple jobs failing. PROBABLY not flaky, not rerunning."); + if (failingJobs.length > 3) { + console.log("Many jobs failing. PROBABLY not flaky, not rerunning."); return; } const filteredFailingJobs = failingJobs.filter((job) => { console.log(`Failing job: ${job.name}`) - return CONSIDERED_JOBS.some((title) => job.name.startsWith(title)); + return CONSIDERED_JOBS + .flatMap(jobName => [jobName, 'CI Pipeline / ' + jobName]) + .some((title) => job.name.startsWith(title)); }); - if (filteredFailingJobs.length === 0) { - console.log("Failing jobs are NOT designated flaky. Not rerunning."); + if (filteredFailingJobs.length !== failingJobs.length) { + console.log("One or more failing jobs are NOT designated flaky. Not rerunning."); return; } diff --git a/.github/workflows/size-label.yml b/.github/workflows/size-label.yml new file mode 100644 index 0000000000..096ce4c643 --- /dev/null +++ b/.github/workflows/size-label.yml @@ -0,0 +1,16 @@ +name: Size Labelling +on: + pull_request_target: + +jobs: + size-label: + name: Add Size Label + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: size-label + uses: "pascalgn/size-label-action@bbbaa0d5ccce8e2e76254560df5c64b82dac2e12" # v0.5.2, consider upgrading after https://github.com/pascalgn/size-label-action/pull/54 is merged + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stable-merge.yml b/.github/workflows/stable-merge.yml index 143a619805..509c9e85c5 100644 --- a/.github/workflows/stable-merge.yml +++ b/.github/workflows/stable-merge.yml @@ -8,7 +8,7 @@ on: env: TGS_DOTNET_VERSION: 8 - OD_MIN_COMPAT_DOTNET_VERSION: 7 + TGS_DOTNET_QUALITY: ga jobs: master-merge: @@ -26,11 +26,6 @@ jobs: with: path: temp_workspace - - name: Restore - run: | - cd temp_workspace - dotnet restore - - name: Build ReleaseNotes run: | cd temp_workspace diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml new file mode 100644 index 0000000000..f6562220f7 --- /dev/null +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -0,0 +1,72 @@ +name: 'Sync spacestation13/tgstation-server' + +on: + push: + branches: + - dev + tags: + - '*' + workflow_dispatch: + +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga + +concurrency: + group: "ss13-mirror-sync" + cancel-in-progress: true + +jobs: + fork-sync: + name: Fork Sync + runs-on: ubuntu-latest + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Build Checkout + uses: actions/checkout@v4 + with: + path: temp_workspace + + - name: Build ReleaseNotes + run: | + cd temp_workspace + dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + cd temp_workspace + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} --spacestation13 + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Main Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + token: ${{ env.INSTALLATION_TOKEN }} + + - name: Build ReleaseNotes + run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Push to Spacestation13 Fork + run: | + git config user.name "tgstation-server-ci[bot]" + git config user.email "161980869+tgstation-server-ci[bot]@users.noreply.github.com" + git push "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" + git push --tags "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" diff --git a/README.md b/README.md index 794543dc85..a646eda7c0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # tgstation-server -![CI Pipeline](https://github.com/tgstation/tgstation-server/workflows/CI%20Pipeline/badge.svg) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) +[![CI Pipeline](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml/badge.svg)](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) [![GitHub license](https://img.shields.io/github/license/tgstation/tgstation-server.svg)](LICENSE) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/tgstation/tgstation-server.svg)](http://isitmaintained.com/project/tgstation/tgstation-server "Average time to resolve an issue") [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Api.svg)](https://www.nuget.org/packages/Tgstation.Server.Api) [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Client.svg)](https://www.nuget.org/packages/Tgstation.Server.Client) @@ -239,7 +239,7 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:InstanceLimit`: Maximum number of instances that may be created -- `General:GitHubAccessToken`: Specify a GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests +- `General:GitHubAccessToken`: Specify a classic GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests - `General:SkipAddingByondFirewallException`: Set to `true` if you have Windows firewall disabled @@ -313,6 +313,12 @@ The following providers use the `ServerUrl` setting: - Keycloak - InvisionCommunity +- `Telemetry:DisableVersionReporting`: Prevents you installation and the version you're using from being reported on the source repository's deployments list + +- `Telemetry:ServerFriendlyName`: Prevents anonymous TGS version usage statistics from being sent to be displayed on the repository. + +- `Telemetry:VersionReportingRepositoryId`: The repository telemetry is sent to. For security reasons, this is not the main TGS repo. See the [tgstation-server-deployments](https://github.com/tgstation/tgstation-server-deployments) repository for more information. + ### Database Configuration If using a MariaDB/MySQL server, our client library [recommends you set 'utf8mb4' as your default charset](https://github.com/PomeloFoundation/Pomelo.EntityFrameworkCore.MySql#1-recommended-server-charset) disregard at your own risk. diff --git a/build/Dockerfile b/build/Dockerfile index cba330156e..b301591833 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,5 +1,8 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim AS build +# Set in CI +ARG TGS_TELEMETRY_KEY_FILE= + # install node and npm # replace shell with bash so we can source files RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/v0.39.1/install.sh | sh @@ -17,7 +20,8 @@ RUN . $NVM_DIR/nvm.sh \ && apt-get install -y \ dos2unix \ libgdiplus \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && corepack enable # Build web control panel WORKDIR /repo/build @@ -45,7 +49,9 @@ RUN dotnet publish -c Release -o /app \ && build/RemoveUnsupportedRuntimes.sh /app WORKDIR /repo/src/Tgstation.Server.Host -RUN dotnet publish -c Release -o /app/lib/Default \ + +RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ + && dotnet publish -c Release -o /app/lib/Default \ && cd ../.. \ && build/RemoveUnsupportedRuntimes.sh /app/lib/Default \ && mv /app/lib/Default/appsettings* /app diff --git a/build/TestCommon.props b/build/TestCommon.props index 1864c27e2a..79449e89df 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,9 +18,9 @@ - + - + diff --git a/build/Version.props b/build/Version.props index 0099619aac..f524e2ee69 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,13 +3,13 @@ - 6.8.0 - 5.1.0 - 10.6.0 + 6.9.0 + 5.2.0 + 10.7.0 7.0.0 - 13.6.0 - 15.6.0 - 7.1.3 + 13.7.0 + 16.0.0 + 7.2.1 5.9.0 1.4.1 1.2.1 @@ -18,9 +18,8 @@ 8 https://download.visualstudio.microsoft.com/download/pr/751d3fcd-72db-4da2-b8d0-709c19442225/33cc492bde704bfd6d70a2b9109005a0/dotnet-hosting-8.0.6-win.exe - 10.11.8 + 11.4.2 - https://mirror.its.dal.ca/mariadb//mariadb-10.11.8/winx64-packages/mariadb-10.11.8-winx64.msi - 1.22.21 + diff --git a/build/WebpanelVersion.props b/build/WebpanelVersion.props index 806c5ed542..2613f5510c 100644 --- a/build/WebpanelVersion.props +++ b/build/WebpanelVersion.props @@ -1,6 +1,6 @@ - 5.9.0 + 6.1.0 diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 1a25546da8..a48071f219 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -18,7 +18,7 @@ apt-get install -y \ devscripts \ ca-certificates \ curl \ - gnupg \ + gnupg2 \ xmlstarlet \ libgdiplus @@ -35,6 +35,8 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co apt-get update apt-get install nodejs dotnet-sdk-8.0 -y +corepack enable + CURRENT_COMMIT=$(git rev-parse HEAD) rm -rf packaging @@ -67,7 +69,6 @@ cp build/tgstation-server.service debian/ SIGN_COMMAND="$SCRIPT_DIR/wrap_gpg.sh" rm -f /tmp/tgs_wrap_gpg_output.log - set +e if [[ -z "$PACKAGING_KEYGRIP" ]]; then diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index f0a7ecbbe3..67356359b6 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -24,5 +24,5 @@ Depends: Recommends: libsystemd0, gdb, -Description: A production scale tool for BYOND server management - This is a toolset to manage production BYOND servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. +Description: A production scale tool for DreamMaker server management + This is a toolset to manage production DreamMaker servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. diff --git a/build/package/deb/debian/rules b/build/package/deb/debian/rules index 7a13e96dd2..cbd846c2da 100755 --- a/build/package/deb/debian/rules +++ b/build/package/deb/debian/rules @@ -10,7 +10,6 @@ override_dh_auto_clean: dotnet clean -c ReleaseNoWindows override_dh_auto_build: - dotnet restore cd src/Tgstation.Server.Host.Console && dotnet publish -c Release -o ../../artifacts cd src/Tgstation.Server.Host && dotnet publish -c Release -o ../../artifacts/lib/Default rm artifacts/lib/Default/appsettings.yml diff --git a/build/tgstation-server.service b/build/tgstation-server.service index 6f3c087584..98c03fee57 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -17,7 +17,7 @@ Restart=always KillMode=process ReloadSignal=SIGUSR2 RestartKillSignal=SIGUSR2 -AmbientCapabilities=CAP_SYS_NICE +AmbientCapabilities=CAP_SYS_NICE CAP_SYS_PTRACE WatchdogSec=60 WatchdogSignal=SIGTERM diff --git a/docs/Features.dox b/docs/Features.dox index 64eca5a3f7..6600cb295f 100644 --- a/docs/Features.dox +++ b/docs/Features.dox @@ -5,7 +5,7 @@ @section features_list Comprehensive Feature List -tgstation-server is a BYOND server managment suite. It includes all the following features +tgstation-server is a DreamMaker server managment suite. It includes all the following features - Standalone server with OpenAPI 3.0 defined HTTP REST API - Web based client included diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 17464b44da..4766b3dfe6 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,18 +1,19 @@ // tgstation-server DMAPI +// The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in IETF RFC 2119. -#define TGS_DMAPI_VERSION "7.1.3" +#define TGS_DMAPI_VERSION "7.2.1" // All functions and datums outside this document are subject to change with any version and should not be relied on. // CONFIGURATION -/// Create this define if you want to do TGS configuration outside of this file. +/// Consumers SHOULD create this define if you want to do TGS configuration outside of this file. #ifndef TGS_EXTERNAL_CONFIGURATION -// Comment this out once you've filled in the below. +// Consumers MUST comment this out once you've filled in the below and are not using [TGS_EXTERNAL_CONFIGURATION]. #error TGS API unconfigured -// Uncomment this if you wish to allow the game to interact with TGS 3.. +// Consumers MUST uncomment this if you wish to allow the game to interact with TGS version 3. // This will raise the minimum required security level of your game to TGS_SECURITY_TRUSTED due to it utilizing call()(). //#define TGS_V3_API @@ -52,7 +53,7 @@ #ifndef TGS_FILE2TEXT_NATIVE #ifdef file2text -#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You can fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses +#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You SHOULD fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses #endif #define TGS_FILE2TEXT_NATIVE file2text #endif @@ -152,16 +153,17 @@ //REQUIRED HOOKS /** - * Call this somewhere in [/world/proc/New] that is always run. This function may sleep! + * Consumers MUST call this somewhere in [/world/proc/New] that is always run. This function may sleep! * * * event_handler - Optional user defined [/datum/tgs_event_handler]. * * minimum_required_security_level: The minimum required security level to run the game in which the DMAPI is integrated. Can be one of [TGS_SECURITY_ULTRASAFE], [TGS_SECURITY_SAFE], or [TGS_SECURITY_TRUSTED]. + * * http_handler - Optional user defined [/datum/tgs_http_handler]. */ -/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler) return /** - * Call this when your initializations are complete and your game is ready to play before any player interactions happen. + * Consumers MUST call this when world initializations are complete and the game is ready to play before any player interactions happen. * * This may use [/world/var/sleep_offline] to make this happen so ensure no changes are made to it while this call is running. * Afterwards, consider explicitly setting it to what you want to avoid this BYOND bug: http://www.byond.com/forum/post/2575184 @@ -170,12 +172,10 @@ /world/proc/TgsInitializationComplete() return -/// Put this at the start of [/world/proc/Topic]. +/// Consumers MUST run this macro at the start of [/world/proc/Topic]. #define TGS_TOPIC var/tgs_topic_return = TgsTopic(args[1]); if(tgs_topic_return) return tgs_topic_return -/** - * Call this as late as possible in [world/proc/Reboot] (BEFORE ..()). - */ +/// Consumers MUST call this as late as possible in [world/proc/Reboot] (BEFORE ..()). /world/proc/TgsReboot() return @@ -269,7 +269,7 @@ /// The [/datum/tgs_chat_channel] the user was from. var/datum/tgs_chat_channel/channel -/// User definable handler for TGS events. +/// User definable handler for TGS events This abstract version SHOULD be overridden to be used. /datum/tgs_event_handler /// If the handler receieves [TGS_EVENT_HEALTH_CHECK] events. var/receive_health_checks = FALSE @@ -283,7 +283,41 @@ set waitfor = FALSE return -/// User definable chat command. +/// User definable handler for HTTP calls. This abstract version MUST be overridden to be used. +/datum/tgs_http_handler + +/** + * User definable callback for executing HTTP GET requests. + * MUST perform BYOND sleeps while the request is in flight. + * MUST return a [/datum/tgs_http_result]. + * SHOULD log its own errors + * + * url - The full URL to execute the GET request for including query parameters. + */ +/datum/tgs_http_handler/proc/PerformGet(url) + CRASH("[type]/PerformGet not implemented!") + +/// Result of a [/datum/tgs_http_handler] call. MUST NOT be overridden. +/datum/tgs_http_result + /// HTTP response as text + var/response_text + /// Boolean request success flag. Set for any 2XX response code. + var/success + +/** + * Create a [/datum/tgs_http_result]. + * + * * response_text - HTTP response as text. Must be provided in New(). + * * success - Boolean request success flag. Set for any 2XX response code. Must be provided in New(). + */ +/datum/tgs_http_result/New(response_text, success) + if(response_text && !istext(response_text)) + CRASH("response_text was not text!") + + src.response_text = response_text + src.success = success + +/// User definable chat command. This abstract version MUST be overridden to be used. /datum/tgs_chat_command /// The string to trigger this command on a chat bot. e.g `@bot name ...` or `!tgs name ...`. var/name = "" @@ -296,21 +330,27 @@ /** * Process command activation. Should return a [/datum/tgs_message_content] to respond to the issuer with. + * MUST be implemented * - * sender - The [/datum/tgs_chat_user] who issued the command. - * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. + * * sender - The [/datum/tgs_chat_user] who issued the command. + * * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. */ /datum/tgs_chat_command/proc/Run(datum/tgs_chat_user/sender, params) CRASH("[type] has no implementation for Run()") -/// User definable chat message. +/// User definable chat message. MUST NOT be overridden. /datum/tgs_message_content - /// The tring content of the message. Must be provided in New(). + /// The string content of the message. Must be provided in New(). var/text /// The [/datum/tgs_chat_embed] to embed in the message. Not supported on all chat providers. var/datum/tgs_chat_embed/structure/embed +/** + * Create a [/datum/tgs_message_content]. + * + * * text - The string content of the message. + */ /datum/tgs_message_content/New(text) ..() if(!istext(text)) @@ -319,7 +359,7 @@ src.text = text -/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/channel#embed-object-embed-structure for details. +/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/message#embed-object for details. /datum/tgs_chat_embed/structure var/title var/description @@ -331,13 +371,13 @@ /// Colour must be #AARRGGBB or #RRGGBB hex string. var/colour - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-image-structure for details. var/datum/tgs_chat_embed/media/image - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-thumbnail-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-thumbnail-structure for details. var/datum/tgs_chat_embed/media/thumbnail - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-video-structure for details. var/datum/tgs_chat_embed/media/video var/datum/tgs_chat_embed/footer/footer @@ -346,7 +386,7 @@ var/list/datum/tgs_chat_embed/field/fields -/// Common datum for similar discord embed medias. +/// Common datum for similar Discord embed medias. /datum/tgs_chat_embed/media /// Must be set in New(). var/url @@ -354,6 +394,7 @@ var/height var/proxy_url +/// Create a [/datum/tgs_chat_embed]. /datum/tgs_chat_embed/media/New(url) ..() if(!istext(url)) @@ -361,13 +402,14 @@ src.url = url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-footer-structure for details. /datum/tgs_chat_embed/footer /// Must be set in New(). var/text var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/footer/New(text) ..() if(!istext(text)) @@ -375,16 +417,17 @@ src.text = text -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-provider-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-provider-structure for details. /datum/tgs_chat_embed/provider var/name var/url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure for details. Must have name set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-author-structure for details. Must have name set in New(). /datum/tgs_chat_embed/provider/author var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/provider/author/New(name) ..() if(!istext(name)) @@ -392,12 +435,15 @@ src.name = name -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure for details. Must have name and value set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-field-structure for details. /datum/tgs_chat_embed/field + /// Must be set in New(). var/name + /// Must be set in New(). var/value var/is_inline +/// Create a [/datum/tgs_chat_embed/field]. /datum/tgs_chat_embed/field/New(name, value) ..() if(!istext(name)) diff --git a/src/DMAPI/tgs/core/README.md b/src/DMAPI/tgs/core/README.md index b82d8f49e2..965e21b549 100644 --- a/src/DMAPI/tgs/core/README.md +++ b/src/DMAPI/tgs/core/README.md @@ -3,7 +3,7 @@ This folder contains all DMAPI code not directly involved in an API. - [_definitions.dm](./definitions.dm) contains defines needed across DMAPI internals. +- [byond_world_export.dm](./byond_world_export.dm) contains the default `/datum/tgs_http_handler` implementation which uses `world.Export()`. - [core.dm](./core.dm) contains the implementations of the `/world/proc/TgsXXX()` procs. Many map directly to the `/datum/tgs_api` functions. It also contains the /datum selection and setup code. - [datum.dm](./datum.dm) contains the `/datum/tgs_api` declarations that all APIs must implement. - [tgs_version.dm](./tgs_version.dm) contains the `/datum/tgs_version` definition -- diff --git a/src/DMAPI/tgs/core/byond_world_export.dm b/src/DMAPI/tgs/core/byond_world_export.dm new file mode 100644 index 0000000000..6ef8d841b8 --- /dev/null +++ b/src/DMAPI/tgs/core/byond_world_export.dm @@ -0,0 +1,22 @@ +/datum/tgs_http_handler/byond_world_export + +/datum/tgs_http_handler/byond_world_export/PerformGet(url) + // This is an infinite sleep until we get a response + var/export_response = world.Export(url) + TGS_DEBUG_LOG("byond_world_export: Export complete") + + if(!export_response) + TGS_ERROR_LOG("byond_world_export: Failed request: [url]") + return new /datum/tgs_http_result(null, FALSE) + + var/content = export_response["CONTENT"] + if(!content) + TGS_ERROR_LOG("byond_world_export: Failed request, missing content!") + return new /datum/tgs_http_result(null, FALSE) + + var/response_json = TGS_FILE2TEXT_NATIVE(content) + if(!response_json) + TGS_ERROR_LOG("byond_world_export: Failed request, failed to load content!") + return new /datum/tgs_http_result(null, FALSE) + + return new /datum/tgs_http_result(response_json, TRUE) diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 15622228e9..63cb5a2c35 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -1,4 +1,4 @@ -/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler = null) var/current_api = TGS_READ_GLOBAL(tgs) if(current_api) TGS_ERROR_LOG("API datum already set (\ref[current_api] ([current_api]))! Was TgsNew() called more than once?") @@ -55,7 +55,10 @@ TGS_ERROR_LOG("Invalid parameter for event_handler: [event_handler]") event_handler = null - var/datum/tgs_api/new_api = new api_datum(event_handler, version) + if(!http_handler) + http_handler = new /datum/tgs_http_handler/byond_world_export + + var/datum/tgs_api/new_api = new api_datum(event_handler, version, http_handler) TGS_WRITE_GLOBAL(tgs, new_api) diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index f734fd0527..3ca53e9bf7 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -6,7 +6,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) var/list/warned_deprecated_command_runs -/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version) +/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) ..() src.event_handler = event_handler src.version = version diff --git a/src/DMAPI/tgs/includes.dm b/src/DMAPI/tgs/includes.dm index 23b714f9d0..f5118ed55a 100644 --- a/src/DMAPI/tgs/includes.dm +++ b/src/DMAPI/tgs/includes.dm @@ -1,4 +1,5 @@ #include "core\_definitions.dm" +#include "core\byond_world_export.dm" #include "core\core.dm" #include "core\datum.dm" #include "core\tgs_version.dm" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 05d0dee25b..3e328fc7c2 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -31,9 +31,12 @@ var/detached = FALSE -/datum/tgs_api/v5/New() + var/datum/tgs_http_handler/http_handler + +/datum/tgs_api/v5/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) . = ..() interop_version = version + src.http_handler = http_handler TGS_DEBUG_LOG("V5 API created: [json_encode(args)]") /datum/tgs_api/v5/ApiVersion() diff --git a/src/DMAPI/tgs/v5/bridge.dm b/src/DMAPI/tgs/v5/bridge.dm index 0c5e701a32..62201fcc9e 100644 --- a/src/DMAPI/tgs/v5/bridge.dm +++ b/src/DMAPI/tgs/v5/bridge.dm @@ -78,27 +78,24 @@ WaitForReattach(FALSE) TGS_DEBUG_LOG("Bridge request start") - // This is an infinite sleep until we get a response - var/export_response = world.Export(bridge_request) + var/datum/tgs_http_result/result = http_handler.PerformGet(bridge_request) TGS_DEBUG_LOG("Bridge request complete") - if(!export_response) - TGS_ERROR_LOG("Failed bridge request: [bridge_request]") + if(isnull(result)) + TGS_ERROR_LOG("Failed bridge request, handler returned null!") return - var/content = export_response["CONTENT"] - if(!content) - TGS_ERROR_LOG("Failed bridge request, missing content!") + if(!istype(result) || result.type != /datum/tgs_http_result) + TGS_ERROR_LOG("Failed bridge request, handler returned non-[/datum/tgs_http_result]!") return - var/response_json = TGS_FILE2TEXT_NATIVE(content) - if(!response_json) - TGS_ERROR_LOG("Failed bridge request, failed to load content!") + if(!result.success) + TGS_DEBUG_LOG("Failed bridge request, HTTP request failed!") return - var/list/bridge_response = json_decode(response_json) + var/list/bridge_response = json_decode(result.response_text) if(!bridge_response) - TGS_ERROR_LOG("Failed bridge request, bad json: [response_json]") + TGS_ERROR_LOG("Failed bridge request, bad json: [result.response_text]") return var/error = bridge_response[DMAPI5_RESPONSE_ERROR_MESSAGE] diff --git a/src/Tgstation.Server.Api/Models/JobCode.cs b/src/Tgstation.Server.Api/Models/JobCode.cs index be5db8d5a6..1c20107b59 100644 --- a/src/Tgstation.Server.Api/Models/JobCode.cs +++ b/src/Tgstation.Server.Api/Models/JobCode.cs @@ -108,5 +108,11 @@ namespace Tgstation.Server.Api.Models /// [Description("Reconnect chat bot")] ReconnectChatBot, + + /// + /// When a repository is recloned. + /// + [Description("Reclone repository")] + RepositoryReclone, } } diff --git a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs index 55d7f646e5..2db5d354a7 100644 --- a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs @@ -16,5 +16,10 @@ namespace Tgstation.Server.Api.Models.Response /// The latest available version of the Tgstation.Server.Host assembly from the upstream repository. If is not equal to 4 the update cannot be applied due to API changes. /// public Version? LatestVersion { get; set; } + + /// + /// This response is cached. This field indicates the when it was generated. + /// + public DateTimeOffset? GeneratedAt { get; set; } } } diff --git a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs index cad5094d1a..d0b86cdaa4 100644 --- a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs @@ -59,5 +59,11 @@ namespace Tgstation.Server.Api.Models.Response /// [ResponseOptions] public bool? CurrentAllowWebclient { get; set; } + + /// + /// The amount of RAM in use by the game server in bytes. + /// + [ResponseOptions] + public long? ImmediateMemoryUsage { get; set; } } } diff --git a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs index f9d134820b..407437ce5d 100644 --- a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs +++ b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs @@ -82,5 +82,10 @@ namespace Tgstation.Server.Api.Rights /// User may change submodule update settings. /// ChangeSubmoduleUpdate = 1 << 13, + + /// + /// User may trigger repository recloning. + /// + Reclone = 1 << 14, } } diff --git a/src/Tgstation.Server.Client/AdministrationClient.cs b/src/Tgstation.Server.Client/AdministrationClient.cs index 2d6841795c..68c7fecd34 100644 --- a/src/Tgstation.Server.Client/AdministrationClient.cs +++ b/src/Tgstation.Server.Client/AdministrationClient.cs @@ -24,7 +24,7 @@ namespace Tgstation.Server.Client } /// - public ValueTask Read(CancellationToken cancellationToken) => ApiClient.Read(Routes.Administration, cancellationToken); + public ValueTask Read(bool forceFresh, CancellationToken cancellationToken) => ApiClient.Read($"{Routes.Administration}?fresh={forceFresh}", cancellationToken); /// public async ValueTask Update( diff --git a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs index cf4eb3ea87..1ea2dccb3b 100644 --- a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs @@ -43,7 +43,10 @@ namespace Tgstation.Server.Client.Components public ValueTask Restart(CancellationToken cancellationToken) => apiClient.Patch(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); /// - public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); + public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read( + Routes.DreamDaemon, + instance.Id!.Value, + cancellationToken); /// public ValueTask Update(DreamDaemonRequest dreamDaemon, CancellationToken cancellationToken) => apiClient.Update(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id!.Value, cancellationToken); diff --git a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs index 7a7a6ead7c..823c8d6a0c 100644 --- a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs @@ -16,7 +16,7 @@ namespace Tgstation.Server.Client.Components /// /// The for the operation. /// A resulting in the information. - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(CancellationToken cancellationToken = default); /// /// Start . diff --git a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs index 9d3bef806c..51d8bdfd80 100644 --- a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs @@ -40,5 +40,12 @@ namespace Tgstation.Server.Client.Components /// The for the operation. /// A resulting in the . ValueTask Delete(CancellationToken cancellationToken); + + /// + /// Deletes and reclones the repository. + /// + /// The for the operation. + /// A resulting in the . + ValueTask Reclone(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Components/RepositoryClient.cs b/src/Tgstation.Server.Client/Components/RepositoryClient.cs index 124f1ec5b2..5711612024 100644 --- a/src/Tgstation.Server.Client/Components/RepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/RepositoryClient.cs @@ -44,5 +44,8 @@ namespace Tgstation.Server.Client.Components /// public ValueTask Update(RepositoryUpdateRequest repository, CancellationToken cancellationToken) => apiClient.Update(Routes.Repository, repository ?? throw new ArgumentNullException(nameof(repository)), instance.Id!.Value, cancellationToken); + + /// + public ValueTask Reclone(CancellationToken cancellationToken) => apiClient.Patch(Routes.Repository, instance.Id!.Value, cancellationToken); } } diff --git a/src/Tgstation.Server.Client/IAdministrationClient.cs b/src/Tgstation.Server.Client/IAdministrationClient.cs index 38d0898345..6d5d88f951 100644 --- a/src/Tgstation.Server.Client/IAdministrationClient.cs +++ b/src/Tgstation.Server.Client/IAdministrationClient.cs @@ -17,9 +17,10 @@ namespace Tgstation.Server.Client /// /// Get the represented by the . /// + /// If the response will be forcefully regenerated. /// The for the operation. /// A resulting in the represented by the . - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(bool forceFresh = false, CancellationToken cancellationToken = default); /// /// Updates the setttings. diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 89c048c68f..3bca360847 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index af52642792..a3847dcdfb 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "8.0.7", + "version": "8.0.8", "commands": [ "dotnet-ef" ] diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs index 39ed128d59..1bd2a8b91e 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs @@ -50,7 +50,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands public async ValueTask Invoke(string arguments, ChatUser user, CancellationToken cancellationToken) { string result; - if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO")) + if (arguments.Split(' ').Any(x => x.Equals("--repo", StringComparison.OrdinalIgnoreCase))) { if (repositoryManager.CloneInProgress || repositoryManager.InUse) return new MessageContent @@ -58,15 +58,13 @@ namespace Tgstation.Server.Host.Components.Chat.Commands Text = "Repository busy! Try again later", }; - using (var repo = await repositoryManager.LoadRepository(cancellationToken)) - { - if (repo == null) - return new MessageContent - { - Text = "Repository unavailable!", - }; - result = repo.Head; - } + using var repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + return new MessageContent + { + Text = "Repository unavailable!", + }; + result = repo.Head; } else { diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 4e37e7d008..9a04c0ac94 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -79,13 +79,13 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote IGitHubService gitHubService; if (instanceAuthenticated) { - authenticatedGitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + authenticatedGitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); gitHubService = authenticatedGitHubService; } else { authenticatedGitHubService = null; - gitHubService = gitHubServiceFactory.CreateService(); + gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); } var repoOwner = remoteInformation.RemoteRepositoryOwner!; @@ -175,8 +175,8 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote } var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); var tasks = revisionInformation .ActiveTestMerges @@ -255,7 +255,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote int testMergeNumber, CancellationToken cancellationToken) { - var gitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + var gitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); try { @@ -343,7 +343,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote return; } - var gitHubService = gitHubServiceFactory.CreateService(gitHubAccessToken); + var gitHubService = await gitHubServiceFactory.CreateService(gitHubAccessToken, cancellationToken); try { diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index c7c73e3a23..40fbde1a40 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -146,7 +146,7 @@ namespace Tgstation.Server.Host.Components.Engine } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter progressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); diff --git a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs index 91887e7314..c24eee3e8b 100644 --- a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs @@ -37,7 +37,7 @@ namespace Tgstation.Server.Host.Components.Engine => DelegateCall(version, installer => installer.CreateInstallation(version, path, installationTask)); /// - public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) => DelegateCall(version, installer => installer.DownloadVersion(version, jobProgressReporter, cancellationToken)); /// diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs index 12cf8e657c..6ca2b94030 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs @@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Engine public abstract ValueTask UpgradeInstallation(EngineVersion version, string path, CancellationToken cancellationToken); /// - public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// public abstract ValueTask TrustDmbPath(EngineVersion version, string fullDmbPath, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index f2a60ae5b9..970e3ab89d 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -118,7 +118,7 @@ namespace Tgstation.Server.Host.Components.Engine /// public async ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, @@ -166,8 +166,11 @@ namespace Tgstation.Server.Host.Components.Engine "Acquiring lock on BYOND version {version}...", requiredVersion?.ToString() ?? $"{ActiveVersion} (active)"); var versionToUse = requiredVersion ?? ActiveVersion ?? throw new JobException(ErrorCode.EngineNoVersionsInstalled); + + using var progressReporter = new JobProgressReporter(); + var installLock = await AssertAndLockVersion( - null, + progressReporter, versionToUse, null, requiredVersion != null, @@ -388,7 +391,7 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Ensures a BYOND is installed if it isn't already. /// - /// The optional for the operation. + /// The for the operation. /// The to install. /// Optional custom zip file to use. Will cause a number to be added. /// If this BYOND version is required as part of a locking operation. @@ -396,7 +399,7 @@ namespace Tgstation.Server.Host.Components.Engine /// The for the operation. /// A resulting in the . async ValueTask AssertAndLockVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool neededForLock, @@ -443,8 +446,7 @@ namespace Tgstation.Server.Host.Components.Engine { if (installedOrInstalling) { - if (progressReporter != null) - progressReporter.StageName = "Waiting for existing installation job..."; + progressReporter.StageName = "Waiting for existing installation job..."; if (neededForLock && !installation.InstallationTask.IsCompleted) logger.LogWarning("The required engine version ({version}) is not readily available! We will have to wait for it to install.", version); @@ -468,8 +470,7 @@ namespace Tgstation.Server.Host.Components.Engine else logger.LogInformation("Requested engine version {version} not currently installed. Doing so now...", version); - if (progressReporter != null) - progressReporter.StageName = "Running event"; + progressReporter.StageName = "Running event"; var versionString = version.ToString(); await eventConsumer.HandleEvent(EventType.EngineInstallStart, new List { versionString }, deploymentPipelineProcesses, cancellationToken); @@ -504,14 +505,14 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Installs the files for a given BYOND . /// - /// The optional for the operation. + /// The for the operation. /// The being installed with the number set if appropriate. /// Custom zip file to use. Will cause a number to be added. /// If processes should be launched as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. async ValueTask InstallVersionFiles( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool deploymentPipelineProcesses, @@ -528,14 +529,12 @@ namespace Tgstation.Server.Host.Components.Engine try { IEngineInstallationData engineInstallationData; + var remainingProgress = 1.0; if (customVersionStream == null) { - if (progressReporter != null) - progressReporter.StageName = "Downloading version"; - - engineInstallationData = await engineInstaller.DownloadVersion(version, progressReporter, cancellationToken); - - progressReporter?.ReportProgress(null); + using var subReporter = progressReporter.CreateSection("Downloading Version", 0.5); + remainingProgress -= 0.5; + engineInstallationData = await engineInstaller.DownloadVersion(version, subReporter, cancellationToken); } else #pragma warning disable CA2000 // Dispose objects before losing scope, false positive @@ -544,33 +543,45 @@ namespace Tgstation.Server.Host.Components.Engine customVersionStream); #pragma warning restore CA2000 // Dispose objects before losing scope - await using (engineInstallationData) + JobProgressReporter remainingReporter; + try { - if (progressReporter != null) - progressReporter.StageName = "Cleaning target directory"; - - await directoryCleanupTask; - - if (progressReporter != null) - progressReporter.StageName = "Extracting data"; - - logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); - await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); + remainingReporter = progressReporter.CreateSection(null, remainingProgress); + } + catch + { + await engineInstallationData.DisposeAsync(); + throw; } - if (progressReporter != null) - progressReporter.StageName = "Running installation actions"; + using (remainingReporter) + { + await using (engineInstallationData) + { + remainingReporter.StageName = "Cleaning target directory"; - await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); + await directoryCleanupTask; + remainingReporter.ReportProgress(0.1); + remainingReporter.StageName = "Extracting data"; - if (progressReporter != null) - progressReporter.StageName = "Writing version file"; + logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); + await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); + remainingReporter.ReportProgress(0.3); + } - // make sure to do this last because this is what tells us we have a valid version in the future - await ioManager.WriteAllBytes( - ioManager.ConcatPath(installFullPath, VersionFileName), - Encoding.UTF8.GetBytes(version.ToString()), - cancellationToken); + remainingReporter.StageName = "Running installation actions"; + + await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); + + remainingReporter.ReportProgress(0.9); + remainingReporter.StageName = "Writing version file"; + + // make sure to do this last because this is what tells us we have a valid version in the future + await ioManager.WriteAllBytes( + ioManager.ConcatPath(installFullPath, VersionFileName), + Encoding.UTF8.GetBytes(version.ToString()), + cancellationToken); + } } catch (HttpRequestException ex) { diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs index a4ad57a4c0..0df793e219 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs @@ -24,10 +24,10 @@ namespace Tgstation.Server.Host.Components.Engine /// Download a given engine . /// /// The of the engine to download. - /// The optional for the operation. + /// The for the operation. /// The for the operation. /// A resulting in the for the download. - ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// /// Does actions necessary to get an extracted installation working. diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs index 18af1469e6..7ce4b883db 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs @@ -28,14 +28,14 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Change the active . /// - /// The optional for the operation. + /// The for the operation. /// The new . /// Optional of a custom BYOND version zip file. /// If an installation should be performed if the is not installed. If and an installation is required an will be thrown. /// The for the operation. /// A representing the running operation. ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index e552d16b40..ac4c0964e9 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -133,23 +133,32 @@ namespace Tgstation.Server.Host.Components.Engine } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); + ArgumentNullException.ThrowIfNull(jobProgressReporter); // get a lock on a system wide OD repo Logger.LogTrace("Cloning OD repo..."); - var progressSection1 = jobProgressReporter?.CreateSection("Updating OpenDream git repository", 0.5f); - - var repo = await repositoryManager.CloneRepository( - GeneralConfiguration.OpenDreamGitUrl, - null, - null, - null, - progressSection1, - true, - cancellationToken); + var progressSection1 = jobProgressReporter.CreateSection("Updating OpenDream git repository", 0.5f); + IRepository? repo; + try + { + repo = await repositoryManager.CloneRepository( + GeneralConfiguration.OpenDreamGitUrl, + null, + null, + null, + progressSection1, + true, + cancellationToken); + } + catch + { + progressSection1.Dispose(); + throw; + } try { @@ -157,6 +166,8 @@ namespace Tgstation.Server.Host.Components.Engine { Logger.LogTrace("OD repo seems to already exist, attempting load and fetch..."); repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + throw new JobException("Can't load OpenDream repository! Please delete cache from disk!"); await repo!.FetchOrigin( progressSection1, @@ -166,18 +177,23 @@ namespace Tgstation.Server.Host.Components.Engine cancellationToken); } - var progressSection2 = jobProgressReporter?.CreateSection("Checking out OpenDream version", 0.5f); + progressSection1.Dispose(); + progressSection1 = null; - var committish = version.SourceSHA - ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; + using (var progressSection2 = jobProgressReporter.CreateSection("Checking out OpenDream version", 0.5f)) + { + var committish = version.SourceSHA + ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; - await repo.CheckoutObject( - committish, - null, - null, - true, - progressSection2, - cancellationToken); + await repo.CheckoutObject( + committish, + null, + null, + true, + false, + progressSection2, + cancellationToken); + } if (!await repo.CommittishIsParent("tgs-min-compat", cancellationToken)) throw new JobException(ErrorCode.OpenDreamTooOld); @@ -189,6 +205,10 @@ namespace Tgstation.Server.Host.Components.Engine repo?.Dispose(); throw; } + finally + { + progressSection1?.Dispose(); + } } /// diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index 35bcbf63dc..ce2c7704e2 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -224,7 +224,7 @@ namespace Tgstation.Server.Host.Components.Engine /// protected override string GetDreamDaemonName(Version byondVersion, out bool supportsCli) { - supportsCli = byondVersion >= DDExeVersion; + supportsCli = byondVersion >= DDExeVersion && !sessionConfiguration.ForceUseDreamDaemonExe; return supportsCli ? "dd.exe" : "dreamdaemon.exe"; } diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 5d8c0b9054..711d6ca246 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -12,7 +12,7 @@ RepoResetOrigin, /// - /// Parameters: Checkout target. + /// Parameters: Checkout target, hard reset flag (If "True", this is actually a hard reset, not a checkout). /// [EventScript("RepoCheckout")] RepoCheckout, diff --git a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs index 6109f490bb..69289bb5d4 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs @@ -49,8 +49,8 @@ namespace Tgstation.Server.Host.Components.Repository CancellationToken cancellationToken) { var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); PullRequest? pr = null; ApiException? exception = null; diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 65bcd1d61e..74417818f1 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -47,7 +47,8 @@ namespace Tgstation.Server.Host.Components.Repository /// The optional username used for fetching from submodule repositories. /// The optional password used for fetching from submodule repositories. /// If a submodule update should be attempted after the merge. - /// The optional to report progress of the operation. + /// If a hard reset to the target committish should be performed instead of a checkout. + /// The to report progress of the operation. /// The for the operation. /// A representing the running operation. ValueTask CheckoutObject( @@ -55,7 +56,8 @@ namespace Tgstation.Server.Host.Components.Repository string? username, string? password, bool updateSubmodules, - JobProgressReporter? progressReporter, + bool moveCurrentReference, + JobProgressReporter progressReporter, CancellationToken cancellationToken); /// @@ -83,14 +85,14 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Fetch commits from the origin repository. /// - /// The optional to report progress of the operation. + /// The to report progress of the operation. /// The optional username to fetch from the origin repository. /// The optional password to fetch from the origin repository. /// If any events created should be marked as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs index 596301611a..87cbe63d95 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs @@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Repository /// The optional branch to clone. /// The optional username to clone from . /// The optional password to clone from . - /// The optional for progress of the clone. + /// The for progress of the clone. /// If submodules should be recusively cloned and initialized. /// The for the operation. /// A resulting i the newly cloned , if one already exists. @@ -44,7 +44,7 @@ namespace Tgstation.Server.Host.Components.Repository string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 972fae31cc..9a71bcdceb 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -232,13 +232,14 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Fetching refspec {refSpec}...", refSpec); var remote = libGitRepo.Network.Remotes.First(); + using var fetchReporter = progressReporter.CreateSection($"Fetch {refSpec}", progressFactor); commands.Fetch( libGitRepo, refSpecList, remote, new FetchOptions().Hydrate( logger, - progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken), logMessage); @@ -267,14 +268,14 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Merging {targetCommitSha} into {currentReference}...", testMergeParameters.TargetCommitSha[..7], Reference); + using var mergeReporter = progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor); result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, FailOnConflict = false, // Needed to get conflicting files FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, - OnCheckoutProgress = CheckoutProgressHandler( - progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor)), + OnCheckoutProgress = CheckoutProgressHandler(mergeReporter), }); } finally @@ -295,7 +296,8 @@ namespace Tgstation.Server.Host.Components.Repository var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {revertTarget}", revertTo); progressReporter.ReportProgress(0); - RawCheckout(revertTo, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); + using var revertReporter = progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0); + RawCheckout(revertTo, false, revertReporter, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -343,8 +345,9 @@ namespace Tgstation.Server.Host.Components.Repository if (updateSubmodules) { + using var progressReporter2 = progressReporter.CreateSection("Update Submodules", progressFactor); await UpdateSubmodules( - progressReporter.CreateSection("Update Submodules", progressFactor), + progressReporter2, username, password, false, @@ -376,20 +379,23 @@ namespace Tgstation.Server.Host.Components.Repository string? username, string? password, bool updateSubmodules, - JobProgressReporter? progressReporter, + bool moveCurrentReference, + JobProgressReporter progressReporter, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(committish); logger.LogDebug("Checkout object: {committish}...", committish); - await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish }, false, cancellationToken); + await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish, moveCurrentReference.ToString() }, false, cancellationToken); await Task.Factory.StartNew( () => { libGitRepo.RemoveUntrackedFiles(); + using var progressReporter3 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0); RawCheckout( committish, - progressReporter?.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), + moveCurrentReference, + progressReporter3, cancellationToken); }, cancellationToken, @@ -397,17 +403,20 @@ namespace Tgstation.Server.Host.Components.Repository TaskScheduler.Current); if (updateSubmodules) + { + using var progressReporter2 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter?.CreateSection(null, 1.0 / 3), + progressReporter2, username, password, false, cancellationToken); + } } /// public async ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -421,13 +430,14 @@ namespace Tgstation.Server.Host.Components.Repository var remote = libGitRepo.Network.Remotes.First(); try { + using var subReporter = progressReporter.CreateSection("Fetch Origin", 1.0); var fetchOptions = new FetchOptions { Prune = true, TagFetchMode = TagFetchMode.All, }.Hydrate( logger, - progressReporter?.CreateSection("Fetch Origin", 1.0), + subReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); @@ -469,18 +479,23 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Reset to origin..."); var trackedBranch = libGitRepo.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, deploymentPipeline, cancellationToken); - await ResetToSha( - trackedBranch.Tip.Sha, - progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), - cancellationToken); + + using (var progressReporter2 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0)) + await ResetToSha( + trackedBranch.Tip.Sha, + progressReporter2, + cancellationToken); if (updateSubmodules) + { + using var progressReporter3 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter.CreateSection(null, 1.0 / 3), + progressReporter3, username, password, deploymentPipeline, cancellationToken); + } } /// @@ -682,9 +697,10 @@ namespace Tgstation.Server.Host.Components.Repository await Task.Factory.StartNew( () => { + using var resetProgress = progressReporter.CreateSection("Hard reset and remove untracked files", 0.1); libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection("Hard reset and remove untracked files", 0.1)), + OnCheckoutProgress = CheckoutProgressHandler(resetProgress), }); cancellationToken.ThrowIfCancellationRequested(); libGitRepo.RemoveUntrackedFiles(); @@ -697,10 +713,11 @@ namespace Tgstation.Server.Host.Components.Repository var remainingProgressFactor = 0.9; if (!synchronizeTrackedBranch) { + using var progressReporter2 = progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor); await PushHeadToTemporaryBranch( username, password, - progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor), + progressReporter2, cancellationToken); return false; } @@ -720,13 +737,24 @@ namespace Tgstation.Server.Host.Components.Repository var remote = libGitRepo.Network.Remotes.First(); try { - libGitRepo.Network.Push( - libGitRepo.Head, - GeneratePushOptions( - progressReporter.CreateSection("Push to origin", remainingProgressFactor), - username, - password, - cancellationToken)); + using var pushReporter = progressReporter.CreateSection("Push to origin", remainingProgressFactor); + var (pushOptions, progressReporters) = GeneratePushOptions( + pushReporter, + username, + password, + cancellationToken); + try + { + libGitRepo.Network.Push( + libGitRepo.Head, + pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + return true; } catch (NonFastForwardException) @@ -879,9 +907,10 @@ namespace Tgstation.Server.Host.Components.Repository /// Runs a blocking force checkout to . /// /// The committish to checkout. - /// The optional for the operation. + /// If a hard reset should actually be performed. + /// The for the operation. /// The for the operation. - void RawCheckout(string committish, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter progressReporter, CancellationToken cancellationToken) { logger.LogTrace("Checkout: {committish}", committish); @@ -890,44 +919,59 @@ namespace Tgstation.Server.Host.Components.Repository CheckoutModifiers = CheckoutModifiers.Force, }; - if (progressReporter != null) - { - var stage = $"Checkout {committish}"; - progressReporter = progressReporter.CreateSection(stage, 1.0); - progressReporter.ReportProgress(0); - checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(progressReporter); - } + var stage = $"Checkout {committish}"; + using var newProgressReporter = progressReporter.CreateSection(stage, 1.0); + newProgressReporter.ReportProgress(0); + checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(newProgressReporter); cancellationToken.ThrowIfCancellationRequested(); - void RunCheckout() => commands.Checkout( - libGitRepo, - checkoutOptions, - committish); - - try + if (moveCurrentReference) { - RunCheckout(); + if (Reference == NoReference) + throw new InvalidOperationException("Cannot move current reference when not on reference!"); + + var gitObject = libGitRepo.Lookup(committish); + if (gitObject == null) + throw new JobException($"Could not find committish: {committish}"); + + var commit = gitObject.Peel(); + + cancellationToken.ThrowIfCancellationRequested(); + + libGitRepo.Reset(ResetMode.Hard, commit, checkoutOptions); } - catch (NotFoundException) + else { - // Maybe (likely) a remote? - var remoteName = $"origin/{committish}"; - var remoteBranch = libGitRepo.Branches.FirstOrDefault( - branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); - cancellationToken.ThrowIfCancellationRequested(); + void RunCheckout() => commands.Checkout( + libGitRepo, + checkoutOptions, + committish); - if (remoteBranch == default) - throw; + try + { + RunCheckout(); + } + catch (NotFoundException) + { + // Maybe (likely) a remote? + var remoteName = $"origin/{committish}"; + var remoteBranch = libGitRepo.Branches.FirstOrDefault( + branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); + cancellationToken.ThrowIfCancellationRequested(); - logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); - var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); + if (remoteBranch == default) + throw; - libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); + logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); + var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); - cancellationToken.ThrowIfCancellationRequested(); + libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); - RunCheckout(); + cancellationToken.ThrowIfCancellationRequested(); + + RunCheckout(); + } } cancellationToken.ThrowIfCancellationRequested(); @@ -955,9 +999,38 @@ namespace Tgstation.Server.Host.Components.Repository try { var forcePushString = String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions(progressReporter.CreateSection(null, 0.9), username, password, cancellationToken)); + + using (var mainPushReporter = progressReporter.CreateSection(null, 0.9)) + { + var (pushOptions, progressReporters) = GeneratePushOptions( + mainPushReporter, + username, + password, + cancellationToken); + + try + { + libGitRepo.Network.Push(remote, forcePushString, pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + } + var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, removalString, GeneratePushOptions(progressReporter.CreateSection(null, 0.1), username, password, cancellationToken)); + using var forcePushReporter = progressReporter.CreateSection(null, 0.1); + var (forcePushOptions, forcePushReporters) = GeneratePushOptions(forcePushReporter, username, password, cancellationToken); + try + { + libGitRepo.Network.Push(remote, removalString, forcePushOptions); + } + finally + { + foreach (var subForcePushReporter in forcePushReporters) + forcePushReporter.Dispose(); + } } catch (UserCancelledException) { @@ -984,32 +1057,39 @@ namespace Tgstation.Server.Host.Components.Repository /// The username for the . /// The password for the . /// The for the operation. - /// A new set of . - PushOptions GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) + /// A new set of and the associated s based off . + (PushOptions PushOptions, IEnumerable SubProgressReporters) GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) { - var subProgressReporter = progressReporter.CreateSection(null, 0.5); + var packFileCountingReporter = progressReporter.CreateSection(null, 0.25); + var packFileDeltafyingReporter = progressReporter.CreateSection(null, 0.25); + var transferProgressReporter = progressReporter.CreateSection(null, 0.5); - return new PushOptions - { - OnPackBuilderProgress = (stage, current, total) => + return ( + PushOptions: new PushOptions { - var baseProgress = stage == PackBuilderStage.Counting ? 0 : 0.5; - var addon = total > 0 && current <= total ? (0.5 * ((double)current / total)) : 0; - progressReporter.ReportProgress(baseProgress + addon); - return !cancellationToken.IsCancellationRequested; + OnPackBuilderProgress = (stage, current, total) => + { + if (total < current) + total = current; + + var percentage = ((double)current) / total; + (stage == PackBuilderStage.Counting ? packFileCountingReporter : packFileDeltafyingReporter).ReportProgress(percentage); + return !cancellationToken.IsCancellationRequested; + }, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, sentBytes, totalBytes) => + { + packFileCountingReporter.ReportProgress((double)sentBytes / totalBytes); + return !cancellationToken.IsCancellationRequested; + }, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, - OnNegotiationCompletedBeforePush = (a) => + SubProgressReporters: new List { - subProgressReporter = progressReporter.CreateSection(null, 0.5); - return !cancellationToken.IsCancellationRequested; - }, - OnPushTransferProgress = (a, sentBytes, totalBytes) => - { - progressReporter.ReportProgress((double)sentBytes / totalBytes); - return !cancellationToken.IsCancellationRequested; - }, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }; + packFileCountingReporter, + packFileDeltafyingReporter, + transferProgressReporter, + }); } /// @@ -1033,7 +1113,7 @@ namespace Tgstation.Server.Host.Components.Repository /// The for the operation. /// A representing the running operation. ValueTask UpdateSubmodules( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -1041,7 +1121,7 @@ namespace Tgstation.Server.Host.Components.Repository { logger.LogTrace("Updating submodules {withOrWithout} credentials...", username == null ? "without" : "with"); - async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter? currentProgressReporter, string parentGitDirectory) + async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter currentProgressReporter, string parentGitDirectory) { var submoduleCount = libGitRepo.Submodules.Count(); if (submoduleCount == 0) @@ -1060,15 +1140,16 @@ namespace Tgstation.Server.Host.Components.Repository OnCheckoutNotify = (_, _) => !cancellationToken.IsCancellationRequested, }; + using var fetchReporter = currentProgressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor); + submoduleUpdateOptions.FetchOptions.Hydrate( logger, - currentProgressReporter?.CreateSection($"Fetch submodule {submodule.Name}", factor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); - if (currentProgressReporter != null) - submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler( - currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)); + using var checkoutReporter = currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor); + submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler(checkoutReporter); logger.LogDebug("Updating submodule {submoduleName}...", submodule.Name); Task RawSubModuleUpdate() => Task.Factory.StartNew( @@ -1085,7 +1166,7 @@ namespace Tgstation.Server.Host.Components.Repository { // workaround for https://github.com/libgit2/libgit2/issues/3820 // kill off the modules/ folder in .git and try again - currentProgressReporter?.ReportProgress(null); + currentProgressReporter.ReportProgress(0); credentialsProvider.CheckBadCredentialsException(ex); logger.LogWarning(ex, "Initial update of submodule {submoduleName} failed. Deleting submodule directories and re-attempting...", submodule.Name); @@ -1124,9 +1205,11 @@ namespace Tgstation.Server.Host.Components.Repository using var submoduleRepo = await submoduleFactory.CreateFromPath( submodulePath, cancellationToken); + + using var submoduleReporter = currentProgressReporter.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor); await RecursiveUpdateSubmodules( submoduleRepo, - currentProgressReporter?.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor), + submoduleReporter, submodulePath); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 94ae176951..74a6bc6751 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -123,7 +123,7 @@ namespace Tgstation.Server.Host.Components.Repository string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken) { @@ -146,8 +146,8 @@ namespace Tgstation.Server.Host.Components.Repository if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try { - var cloneProgressReporter = progressReporter?.CreateSection(null, 0.75f); - var checkoutProgressReporter = progressReporter?.CreateSection(null, 0.25f); + using var cloneProgressReporter = progressReporter.CreateSection(null, 0.75f); + using var checkoutProgressReporter = progressReporter.CreateSection(null, 0.25f); var cloneOptions = new CloneOptions { RecurseSubmodules = recurseSubmodules, diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs index a7aca78c4b..17e820433a 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs @@ -22,11 +22,6 @@ namespace Tgstation.Server.Host.Components.Repository /// sealed class RepositoryUpdateService { - /// - /// The for the . - /// - readonly RepositoryUpdateRequest model; - /// /// The current for the . /// @@ -50,19 +45,16 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Initializes a new instance of the class. /// - /// The value of . /// The value of . /// The value of . /// The value of . /// The value of . public RepositoryUpdateService( - RepositoryUpdateRequest model, RepositorySettings currentModel, User initiatingUser, ILogger logger, long instanceId) { - this.model = model ?? throw new ArgumentNullException(nameof(model)); this.currentModel = currentModel ?? throw new ArgumentNullException(nameof(currentModel)); this.initiatingUser = initiatingUser ?? throw new ArgumentNullException(nameof(initiatingUser)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -139,24 +131,25 @@ namespace Tgstation.Server.Host.Components.Repository /// /// The job entrypoint used by to update the repository's current HEAD. /// - /// The the job is running on. only when performing an instance move operation. + /// The . + /// The the job is running on. /// The for the operation. - /// The running , ignored. /// The for the job. /// The for the operation. /// A representing the running operation. #pragma warning disable CA1502, CA1506 // TODO: Decomplexify public async ValueTask RepositoryUpdateJob( + RepositoryUpdateRequest model, IInstanceCore? instance, IDatabaseContextFactory databaseContextFactory, - Job job, JobProgressReporter progressReporter, CancellationToken cancellationToken) #pragma warning restore CA1502, CA1506 { + ArgumentNullException.ThrowIfNull(model); ArgumentNullException.ThrowIfNull(instance); - - _ = job; // shuts up an IDE warning + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); var repoManager = instance.RepositoryManager; using var repo = await repoManager.LoadRepository(cancellationToken) ?? throw new JobException(ErrorCode.RepoMissing); @@ -180,10 +173,7 @@ namespace Tgstation.Server.Host.Components.Repository var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); var progressFactor = 1.0 / numSteps; - JobProgressReporter NextProgressReporter(string? stage) - { - return progressReporter.CreateSection(stage, progressFactor); - } + JobProgressReporter NextProgressReporter(string? stage) => progressReporter.CreateSection(stage, progressFactor); progressReporter.ReportProgress(0); @@ -253,29 +243,35 @@ namespace Tgstation.Server.Host.Components.Repository { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceRequired); - await repo.FetchOrigin( - NextProgressReporter("Fetch Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - false, - cancellationToken); + using (var fetchReporter = NextProgressReporter("Fetch Origin")) + await repo.FetchOrigin( + fetchReporter, + currentModel.AccessUser, + currentModel.AccessToken, + false, + cancellationToken); if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin( - NextProgressReporter("Merge Origin"), - committerName, - currentModel.CommitterEmail!, - false, - cancellationToken); + bool? fastForward; + using (var mergeReporter = NextProgressReporter("Merge Origin")) + fastForward = await repo.MergeOrigin( + mergeReporter, + committerName, + currentModel.CommitterEmail!, + false, + cancellationToken); + if (!fastForward.HasValue) throw new JobException(ErrorCode.RepoMergeConflict); + lastRevisionInfo!.OriginCommitSha = await repo.GetOriginSha(cancellationToken); await UpdateRevInfo(); if (fastForward.Value) { + using var syncReporter = NextProgressReporter("Sychronize"); await repo.Synchronize( - NextProgressReporter("Sychronize"), + syncReporter, currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName!, @@ -286,7 +282,7 @@ namespace Tgstation.Server.Host.Components.Repository postUpdateSha = repo.Head; } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); } } @@ -310,38 +306,44 @@ namespace Tgstation.Server.Host.Components.Repository if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) throw new JobException(ErrorCode.RepoSwappedShaOrReference); - await repo.CheckoutObject( - committish, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - NextProgressReporter("Checkout"), - cancellationToken); + using (var checkoutReporter = NextProgressReporter("Checkout")) + await repo.CheckoutObject( + committish, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + checkoutReporter, + cancellationToken); await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); if (hardResettingToOriginReference) { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceNotTracking); - await repo.ResetToOrigin( - NextProgressReporter("Reset to Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - false, - cancellationToken); - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - true, - false, - cancellationToken); + using (var resetReporter = NextProgressReporter("Reset to Origin")) + await repo.ResetToOrigin( + resetReporter, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + cancellationToken); + + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + true, + false, + cancellationToken); + await CallLoadRevInfo(); // repo head is on origin so force this @@ -492,7 +494,8 @@ namespace Tgstation.Server.Host.Components.Repository // goteem var commitSha = revInfoWereLookingFor.CommitSha!; logger.LogDebug("Reusing existing SHA {sha}...", commitSha); - await repo.ResetToSha(commitSha, NextProgressReporter($"Reset to {commitSha[..7]}"), cancellationToken); + using var resetReporter = NextProgressReporter($"Reset to {commitSha[..7]}"); + await repo.ResetToSha(commitSha, resetReporter, cancellationToken); lastRevisionInfo = revInfoWereLookingFor; } @@ -505,15 +508,17 @@ namespace Tgstation.Server.Host.Components.Repository var fullTestMergeTask = repo.GetTestMerge(newTestMerge, currentModel, cancellationToken); - var mergeResult = await repo.AddTestMerge( - newTestMerge, - committerName, - currentModel.CommitterEmail!, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - NextProgressReporter($"Test merge #{newTestMerge.Number}"), - cancellationToken); + TestMergeResult mergeResult; + using (var testMergeReporter = NextProgressReporter($"Test merge #{newTestMerge.Number}")) + mergeResult = await repo.AddTestMerge( + newTestMerge, + committerName, + currentModel.CommitterEmail!, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + testMergeReporter, + cancellationToken); if (mergeResult.Status == MergeStatus.Conflicts) throw new JobException( @@ -552,15 +557,17 @@ namespace Tgstation.Server.Host.Components.Repository var currentHead = repo.Head; if (currentModel.PushTestMergeCommits!.Value && (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead))) { - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - false, - false, - cancellationToken); + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + false, + false, + cancellationToken); + await UpdateRevInfo(); } } @@ -574,19 +581,109 @@ namespace Tgstation.Server.Host.Components.Repository var secondStep = startReference != null && repo.Head != startSha; // DCTx2: Cancellation token is for job, operations should always run - await repo.CheckoutObject( - startReference ?? startSha, - currentModel.AccessUser, - currentModel.AccessToken, - true, - progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0), - default); + using (var checkoutReporter = progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0)) + await repo.CheckoutObject( + startReference ?? startSha, + currentModel.AccessUser, + currentModel.AccessToken, + true, + false, + checkoutReporter, + default); if (secondStep) - await repo.ResetToSha(startSha, progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5), default); + using (var resetReporter = progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5)) + await repo.ResetToSha(startSha, resetReporter, default); throw; } } + + /// + /// The job entrypoint used by to reclone a repository. + /// + /// The the job is running on. + /// The for the operation. + /// The for the job. + /// The for the operation. + /// A representing the running operation. + public async ValueTask RepositoryRecloneJob( + IInstanceCore? instance, + IDatabaseContextFactory databaseContextFactory, + JobProgressReporter progressReporter, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instance); + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); + + progressReporter.StageName = "Loading Old Repository"; + + Uri origin; + string? oldReference; + string oldSha; + ValueTask deleteTask; + using (var deleteReporter = progressReporter.CreateSection("Deleting Old Repository", 0.1)) + { + using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken)) + { + if (oldRepo == null) + throw new JobException(ErrorCode.RepoMissing); + + origin = oldRepo.Origin; + oldSha = oldRepo.Head; + oldReference = oldRepo.Reference; + if (oldReference == Repository.NoReference) + oldReference = null; + + deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken); + } + + await deleteTask; + } + + IRepository newRepo; + try + { + using var cloneReporter = progressReporter.CreateSection("Cloning New Repository", 0.8); + newRepo = await instance.RepositoryManager.CloneRepository( + origin, + oldReference, + currentModel.AccessUser, + currentModel.AccessToken, + cloneReporter, + true, // TODO: Make configurable maybe... + cancellationToken) + ?? throw new JobException("A race condition occurred while recloning the repository. Somehow, it was fully cloned instantly after being deleted!"); // I'll take lines of code that should never be hit for $10k + } + catch (Exception ex) when (ex is not JobException) + { + logger.LogWarning("Reclone failed, clearing credentials!"); + + // need to clear credentials here + await databaseContextFactory.UseContextTaskReturn(context => + { + context.RepositorySettings.Attach(currentModel); + currentModel.AccessUser = null; + currentModel.AccessToken = null; + return context.Save(CancellationToken.None); // DCT: Must always run + }); + + throw; + } + + using (newRepo) + using (var checkoutReporter = progressReporter.CreateSection("Checking out previous Detached Commit", 0.1)) + { + await newRepo.CheckoutObject( + oldSha, + currentModel.AccessUser, + currentModel.AccessToken, + false, + oldReference != null, + checkoutReporter, + cancellationToken); + } + } } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 10556ca9fc..754b672b7d 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -121,6 +121,9 @@ namespace Tgstation.Server.Host.Components.Session /// public FifoSemaphore TopicSendSemaphore { get; } + /// + public long? MemoryUsage => process.MemoryUsage; + /// /// The for the . /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 96c6c9ccf9..2220c56932 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -24,6 +24,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// WatchdogStatus Status { get; } + /// + /// Gets the memory usage of the game server in bytes. + /// + long? MemoryUsage { get; } + /// /// If the alpha server is the active server. /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 28156db6e5..5d9326ce7d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -50,6 +50,9 @@ namespace Tgstation.Server.Host.Components.Watchdog } } + /// + public long? MemoryUsage => GetActiveController()?.MemoryUsage; + /// public abstract bool AlphaIsActive { get; } diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index bc9c11e2f3..1790316e37 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -89,7 +89,7 @@ namespace Tgstation.Server.Host.Configuration public ushort ApiPort { get; set; } /// - /// A GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. + /// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. /// public string? GitHubAccessToken { get; set; } diff --git a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs index c543698b90..d7ee3e1c9d 100644 --- a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs @@ -29,5 +29,10 @@ /// If , deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined. /// public bool DelayCleaningFailedDeployments { get; set; } + + /// + /// If set dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. + /// + public bool ForceUseDreamDaemonExe { get; set; } } } diff --git a/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs new file mode 100644 index 0000000000..7ec0361759 --- /dev/null +++ b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs @@ -0,0 +1,33 @@ +namespace Tgstation.Server.Host.Configuration +{ + /// + /// Configuration options for telemetry. + /// + public sealed class TelemetryConfiguration + { + /// + /// The key for the the resides in. + /// + public const string Section = "Telemetry"; + + /// + /// The default value of . + /// + private const long DefaultVersionReportingRepositoryId = 841149827; // https://github.com/tgstation/tgstation-server-deployments + + /// + /// If version reporting telemetry is disabled. + /// + public bool DisableVersionReporting { get; set; } + + /// + /// The friendly name used on GitHub deployments for version reporting. If only the server will be shown. + /// + public string? ServerFriendlyName { get; set; } + + /// + /// The GitHub repository ID used for version reporting. + /// + public long? VersionReportingRepositoryId { get; set; } = DefaultVersionReportingRepositoryId; + } +} diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index 2ccc4f4c8b..c41113069d 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using System.Web; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -42,6 +43,11 @@ namespace Tgstation.Server.Host.Controllers /// const string OctokitException = "Bad GitHub API response, check configuration!"; + /// + /// The key for . + /// + static readonly object ReadCacheKey = new(); + /// /// The for the . /// @@ -77,6 +83,11 @@ namespace Tgstation.Server.Host.Controllers /// readonly IFileTransferTicketProvider fileTransferService; + /// + /// The for the . + /// + readonly IMemoryCache cacheService; + /// /// The for the . /// @@ -94,6 +105,7 @@ namespace Tgstation.Server.Host.Controllers /// The value of . /// The value of . /// The value of . + /// The value of . /// The for the . /// The containing value of . /// The for the . @@ -107,6 +119,7 @@ namespace Tgstation.Server.Host.Controllers IIOManager ioManager, IPlatformIdentifier platformIdentifier, IFileTransferTicketProvider fileTransferService, + IMemoryCache cacheService, ILogger logger, IOptions fileLoggingConfigurationOptions, IApiHeadersProvider apiHeadersProvider) @@ -124,12 +137,14 @@ namespace Tgstation.Server.Host.Controllers this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); + this.cacheService = cacheService ?? throw new ArgumentNullException(nameof(cacheService)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); } /// /// Get server information. /// + /// If , the cache should be bypassed. /// The for the operation. /// A resulting in the for the operation. /// Retrieved data successfully. @@ -140,39 +155,56 @@ namespace Tgstation.Server.Host.Controllers [ProducesResponseType(typeof(AdministrationResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 424)] [ProducesResponseType(typeof(ErrorMessageResponse), 429)] - public async ValueTask Read(CancellationToken cancellationToken) + public async ValueTask Read([FromQuery] bool? fresh, CancellationToken cancellationToken) { try { - Version? greatestVersion = null; - Uri? repoUrl = null; - try + async Task CacheFactory() { - var gitHubService = gitHubServiceFactory.CreateService(); - var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); - var releases = await gitHubService.GetTgsReleases(cancellationToken); - - foreach (var kvp in releases) + Version? greatestVersion = null; + Uri? repoUrl = null; + try { - var version = kvp.Key; - var release = kvp.Value; - if (version.Major > 3 // Forward/backward compatible but not before TGS4 - && (greatestVersion == null || version > greatestVersion)) - greatestVersion = version; + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); + var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); + var releases = await gitHubService.GetTgsReleases(cancellationToken); + + foreach (var kvp in releases) + { + var version = kvp.Key; + var release = kvp.Value; + if (version.Major > 3 // Forward/backward compatible but not before TGS4 + && (greatestVersion == null || version > greatestVersion)) + greatestVersion = version; + } + + repoUrl = await repositoryUrlTask; + } + catch (NotFoundException e) + { + Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); } - repoUrl = await repositoryUrlTask; - } - catch (NotFoundException e) - { - Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); + return Json(new AdministrationResponse + { + LatestVersion = greatestVersion, + TrackedRepositoryUrl = repoUrl, + GeneratedAt = DateTimeOffset.UtcNow, + }); } - return Json(new AdministrationResponse + var ttl = TimeSpan.FromMinutes(30); + Task task; + if (fresh == true || !cacheService.TryGetValue(ReadCacheKey, out var rawCacheObject)) { - LatestVersion = greatestVersion, - TrackedRepositoryUrl = repoUrl, - }); + using var entry = cacheService.CreateEntry(ReadCacheKey); + entry.AbsoluteExpirationRelativeToNow = ttl; + entry.Value = task = CacheFactory(); + } + else + task = (Task)rawCacheObject!; + + return await task; } catch (RateLimitExceededException e) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index d60fda967d..79e03ebb84 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -317,11 +317,12 @@ namespace Tgstation.Server.Host.Controllers /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. +#pragma warning disable CA1502 // TODO: Decomplexify ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForcedReboot, CancellationToken cancellationToken) +#pragma warning restore CA1502 => WithComponentInstance(async instance => { var dd = instance.Watchdog; - var metadata = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadMetadata) != 0; var revision = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadRevision) != 0; @@ -372,6 +373,7 @@ namespace Tgstation.Server.Host.Controllers result.Visibility = settings.Visibility!.Value; result.SoftRestart = rstate == RebootState.Restart; result.SoftShutdown = rstate == RebootState.Shutdown; + result.ImmediateMemoryUsage = dd.MemoryUsage; if (rstate == RebootState.Normal && knownForcedReboot) result.SoftRestart = true; diff --git a/src/Tgstation.Server.Host/Controllers/EngineController.cs b/src/Tgstation.Server.Host/Controllers/EngineController.cs index c1a20fe99c..83a113e3b4 100644 --- a/src/Tgstation.Server.Host/Controllers/EngineController.cs +++ b/src/Tgstation.Server.Host/Controllers/EngineController.cs @@ -184,7 +184,8 @@ namespace Tgstation.Server.Host.Controllers try { - await byondManager.ChangeVersion(null, model.EngineVersion, null, false, cancellationToken); + using var progressReporter = new JobProgressReporter(); + await byondManager.ChangeVersion(progressReporter, model.EngineVersion, null, false, cancellationToken); } catch (InvalidOperationException ex) { diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 5d4d686aaf..735b9efa83 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -223,6 +223,42 @@ namespace Tgstation.Server.Host.Controllers return Accepted(api); } + /// + /// Delete the repository. + /// + /// The for the operation. + /// A resulting in the of the operation. + /// Job to delete the repository created successfully. + /// The database entity for the requested instance could not be retrieved. The instance was likely detached. + [HttpPatch] + [TgsAuthorize(RepositoryRights.Reclone)] + [ProducesResponseType(typeof(RepositoryResponse), 202)] + [ProducesResponseType(typeof(ErrorMessageResponse), 410)] + public async ValueTask Reclone(CancellationToken cancellationToken) + { + var currentModel = await DatabaseContext + .RepositorySettings + .AsQueryable() + .Where(x => x.InstanceId == Instance.Id) + .FirstOrDefaultAsync(cancellationToken); + + if (currentModel == default) + return this.Gone(); + + Logger.LogInformation("Instance {instanceId} repository reclone initiated by user {userId}", Instance.Id, AuthenticationContext.User.Require(x => x.Id)); + + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); + + var job = Job.Create(JobCode.RepositoryReclone, AuthenticationContext.User, Instance); + var api = currentModel.ToApi(); + await jobManager.RegisterOperation( + job, + (core, databaseContextFactory, paramJob, progressReporter, ct) => repositoryUpdater.RepositoryRecloneJob(core, databaseContextFactory, progressReporter, ct), + cancellationToken); + api.ActiveJob = job.ToApi(); + return Accepted(api); + } + /// /// Get the repository's status. /// @@ -443,17 +479,12 @@ namespace Tgstation.Server.Host.Controllers var job = Job.Create(JobCode.RepositoryUpdate, AuthenticationContext.User, Instance, RepositoryRights.CancelPendingChanges); job.Description = description; - var repositoryUpdater = new RepositoryUpdateService( - model, - currentModel, - AuthenticationContext.User, - loggerFactory.CreateLogger(), - Instance.Require(x => x.Id)); + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); // Time to access git, do it in a job await jobManager.RegisterOperation( job, - repositoryUpdater.RepositoryUpdateJob, + (instance, databaseContextFactory, _, progressReporter, jobToken) => repositoryUpdater.RepositoryUpdateJob(model, instance, databaseContextFactory, progressReporter, jobToken), cancellationToken); api.ActiveJob = job.ToApi(); @@ -494,5 +525,17 @@ namespace Tgstation.Server.Host.Controllers cancellationToken); return needsDbUpdate; } + + /// + /// Creates a . + /// + /// The current . + /// A new . + RepositoryUpdateService CreateRepositoryUpdateService(RepositorySettings currentModel) + => new( + currentModel, + AuthenticationContext.User, + loggerFactory.CreateLogger(), + Instance.Require(x => x.Id)); } } diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index df363970bb..577fecf1c6 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -152,11 +152,17 @@ namespace Tgstation.Server.Host.Controllers [HttpGet("logo.svg")] public IActionResult GetLogo() { - var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- - ? LogoSvgWindowsName - : LogoSvgLinuxName; + // these are different because of motherfucking line endings -_- + if (platformIdentifier.IsWindows) + { + VirtualFileResult? result = this.TryServeFile(hostEnvironment, logger, $"{LogoSvgWindowsName}.svg"); + if (result != null) + return result; - return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg") ?? NotFound(); + // BUT THE UPDATE PACKAGES ARE BUILT ON LINUX RAAAAAGH + } + + return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{LogoSvgLinuxName}.svg") ?? NotFound(); } /// diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 36970000c5..7e981066c0 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -143,6 +143,7 @@ namespace Tgstation.Server.Host.Core services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); + services.UseStandardConfig(Configuration); // enable options which give us config reloading services.AddOptions(); @@ -423,6 +424,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); + services.AddHostedService(); services.AddFileDownloader(); services.AddGitHub(); diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index 8c9d968168..1170537a12 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -288,7 +288,7 @@ namespace Tgstation.Server.Host.Core { logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var releases = await gitHubService.GetTgsReleases(cancellationToken); foreach (var kvp in releases) { diff --git a/src/Tgstation.Server.Host/Core/VersionReportingService.cs b/src/Tgstation.Server.Host/Core/VersionReportingService.cs new file mode 100644 index 0000000000..dd0eef6f78 --- /dev/null +++ b/src/Tgstation.Server.Host/Core/VersionReportingService.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Octokit; + +using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Properties; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; + +namespace Tgstation.Server.Host.Core +{ + /// + /// Handles TGS version reporting, if enabled. + /// + sealed class VersionReportingService : BackgroundService + { + /// + /// The for the . + /// + readonly IGitHubClientFactory gitHubClientFactory; + + /// + /// The for the . + /// + readonly IIOManager ioManager; + + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly IAssemblyInformationProvider assemblyInformationProvider; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The for the . + /// + readonly TelemetryConfiguration telemetryConfiguration; + + /// + /// The passed to . + /// + CancellationToken shutdownCancellationToken; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + /// The containing the value of . + /// The value of . + public VersionReportingService( + IGitHubClientFactory gitHubClientFactory, + IIOManager ioManager, + IAsyncDelayer asyncDelayer, + IAssemblyInformationProvider assemblyInformationProvider, + IOptions telemetryConfigurationOptions, + ILogger logger) + { + this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + telemetryConfiguration = telemetryConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(telemetryConfigurationOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public override Task StopAsync(CancellationToken cancellationToken) + { + shutdownCancellationToken = cancellationToken; + return base.StopAsync(cancellationToken); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (telemetryConfiguration.DisableVersionReporting) + { + logger.LogDebug("Version telemetry disabled"); + return; + } + + if (!telemetryConfiguration.VersionReportingRepositoryId.HasValue) + { + logger.LogError("Version reporting repository is misconfigured. Telemetry cannot be sent!"); + return; + } + + var attribute = TelemetryAppSerializedKeyAttribute.Instance; + if (attribute == null) + { + logger.LogDebug("TGS build configuration does not allow for version telemetry"); + return; + } + + logger.LogDebug("Starting..."); + + try + { + var telemetryIdFile = ioManager.ResolvePath( + ioManager.ConcatPath( + ioManager.GetPathInLocalDirectory(assemblyInformationProvider), + "telemetry.id")); + + Guid telemetryId; + if (!await ioManager.FileExists(telemetryIdFile, stoppingToken)) + { + telemetryId = Guid.NewGuid(); + await ioManager.WriteAllBytes(telemetryIdFile, Encoding.UTF8.GetBytes(telemetryId.ToString()), stoppingToken); + logger.LogInformation("Generated telemetry ID {telemetryId} and wrote to {file}", telemetryId, telemetryIdFile); + } + else + { + var contents = await ioManager.ReadAllBytes(telemetryIdFile, stoppingToken); + + string guidStr; + try + { + guidStr = Encoding.UTF8.GetString(contents); + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot decode telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + + if (!Guid.TryParse(guidStr, out telemetryId)) + { + logger.LogError("Cannot parse telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + } + + try + { + while (!stoppingToken.IsCancellationRequested) + { + var nextDelayHours = await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + false, + stoppingToken) + ? 24 + : 1; + + logger.LogDebug("Next version report in {hours} hours", nextDelayHours); + await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken); + } + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Inner cancellation"); + } + + shutdownCancellationToken.ThrowIfCancellationRequested(); + + logger.LogDebug("Sending shutdown telemetry"); + await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + true, + shutdownCancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Exiting due to outer cancellation..."); + } + catch (Exception ex) + { + logger.LogError(ex, "Crashed!"); + } + } + + /// + /// Make an attempt to report the current to the configured GitHub repository. + /// + /// The telemetry for the installation. + /// The serialized authentication for the . + /// The ID of the repository to send telemetry to. + /// If this is shutdown telemetry. + /// The for the operation. + /// A resulting in if telemetry was reported successfully, otherwise. + async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, bool shutdown, CancellationToken cancellationToken) + { + logger.LogDebug("Sending version telemetry..."); + + var serverFriendlyName = telemetryConfiguration.ServerFriendlyName; + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + + logger.LogTrace( + "Repository ID: {repoId}, Server friendly name: {friendlyName}", + repositoryId, + serverFriendlyName == null + ? "(null)" + : $"\"{serverFriendlyName}\""); + try + { + var gitHubClient = await gitHubClientFactory.CreateInstallationClient( + serializedPem, + repositoryId, + cancellationToken); + + if (gitHubClient == null) + { + logger.LogWarning("Could not create GitHub client to connect to repository ID {repoId}!", repositoryId); + return false; + } + + // remove this lookup once https://github.com/octokit/octokit.net/pull/2960 is merged and released + var repository = await gitHubClient.Repository.Get(repositoryId); + + logger.LogTrace("Repository ID {id} resolved to {owner}/{name}", repositoryId, repository.Owner.Name, repository.Name); + + var inputs = new Dictionary + { + { "telemetry_id", telemetryId.ToString() }, + { "tgs_semver", assemblyInformationProvider.Version.Semver().ToString() }, + { "shutdown", shutdown ? "true" : "false" }, + }; + + if (serverFriendlyName != null) + inputs.Add("server_friendly_name", serverFriendlyName); + + await gitHubClient.Actions.Workflows.CreateDispatch( + repository.Owner.Login, + repository.Name, + ".github/workflows/tgs_deployments_telemetry.yml", + new CreateWorkflowDispatch("main") + { + Inputs = inputs, + }); + + logger.LogTrace("Telemetry sent successfully"); + + return true; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to report version!"); + return false; + } + } + } +} diff --git a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs index 6d71a68edf..b5e996c311 100644 --- a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs @@ -20,14 +20,14 @@ namespace Tgstation.Server.Host.Extensions /// /// The to hydrate. /// The for the operation. - /// The optional . + /// The . /// The optional . /// The for the operation. /// The hydrated . public static FetchOptions Hydrate( this FetchOptions fetchOptions, ILogger logger, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, CredentialsHandler credentialsHandler, CancellationToken cancellationToken) { @@ -60,10 +60,10 @@ namespace Tgstation.Server.Host.Extensions /// Generate a from a given and . /// /// The for the operation. - /// The optional of the operation. + /// The of the operation. /// The for the operation. /// A new based on . - static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter? progressReporter, CancellationToken cancellationToken) => transferProgress => + static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter progressReporter, CancellationToken cancellationToken) => transferProgress => { double? percentage; var totalObjectsToProcess = transferProgress.TotalObjects * 2; diff --git a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs index 753f8cf519..12ee246858 100644 --- a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs +++ b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs @@ -1,6 +1,7 @@ using System; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Tgstation.Server.Host.Models; @@ -9,7 +10,7 @@ namespace Tgstation.Server.Host.Jobs /// /// Progress reporter for a . /// - public sealed class JobProgressReporter + public sealed class JobProgressReporter : IDisposable { /// /// The name of the current stage. @@ -52,6 +53,24 @@ namespace Tgstation.Server.Host.Jobs /// double sectionProgression; + /// + /// The total progress reserved for use in this section. + /// + double? sectionReservations; + + /// + /// Initializes a new instance of the class. + /// + /// This variant has no function. + public JobProgressReporter() + : this( + NullLogger.Instance, + null, + (_, _) => { }, + false) + { + } + /// /// Initializes a new instance of the class. /// @@ -59,27 +78,84 @@ namespace Tgstation.Server.Host.Jobs /// The value of . /// The value of . public JobProgressReporter(ILogger logger, string? stageName, Action callback) + : this( + logger, + stageName, + callback, + true) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// If an initial call to will be made with only the . + private JobProgressReporter(ILogger logger, string? stageName, Action callback, bool setStageName) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); - StageName = stageName; + if (setStageName) + { + StageName = stageName; + } + else + { + this.stageName = stageName; + } logger.LogDebug("Job progress reporter created. Stage: {stageName}", stageName ?? "(null)"); } + /// + public void Dispose() + { + if (sectionReservations.HasValue) + if (sectionReservations.Value != 1.0) + { + // not an error, processes can throw + sectionReservations = null; + } + else if (sectionProgression < 1.0) + { + logger.LogError( + new InvalidOperationException($"Parent progress reporter has child sections that didn't complete! Current: {sectionProgression}"), + "TGS BUG: Progress reporter children didn't complete!"); + sectionReservations = null; + } + + if (!sectionReservations.HasValue) + ReportProgress(1); + } + /// /// Report progress. /// /// A percentage value from 0.0f-1.0f. public void ReportProgress(double? progress) { + if (sectionReservations.HasValue) + if (progress == 0) + { + // might be a stage reset + sectionReservations = null; + } + else + { + logger.LogError( + new InvalidOperationException("Progress reporter is reporting progress with existing nested sections!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + var clampedProgress = progress; if (progress.HasValue) if (progress > 1 || progress < 0) { logger.LogError( new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-1!"), - "Invalid progress value for stage {stageName}", + "TGS BUG: Invalid progress value for stage {stageName}", StageName ?? "(null)"); clampedProgress = null; } @@ -103,16 +179,28 @@ namespace Tgstation.Server.Host.Jobs { logger.LogError( new ArgumentOutOfRangeException(nameof(percentage), percentage, "Percentage must be a value from 0-1!"), - "Invalid percentage value for stage {newStageName}! Clamping...", + "TGS BUG: Invalid percentage value for stage {newStageName}! Clamping...", newStageName ?? "(null)"); percentage = Math.Min(Math.Max(percentage, 0.0), 1.0); } - var childBaseProgress = sectionProgression; - if (percentage + childBaseProgress > 1.0) + if (!sectionReservations.HasValue) { - var remainingPercentage = 1.0 - childBaseProgress; + if (sectionProgression != 0) + { + logger.LogError( + new InvalidOperationException("Progress reporter is creating a section with local progress!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + + sectionReservations = 0; + } + + // floating point >.< + if (percentage + sectionReservations.Value > 1.0001) + { + var remainingPercentage = 1.0 - sectionReservations.Value; logger.LogError( "Stage {newStageName} is overbudgeted ({budget}/{remainingPercentage})! Clamping...", newStageName, @@ -121,6 +209,9 @@ namespace Tgstation.Server.Host.Jobs percentage = remainingPercentage; } + Math.Min(sectionReservations.Value + percentage, 1); + + var childLocalProgress = 0.0; var newReporter = new JobProgressReporter( logger, newStageName, @@ -133,11 +224,17 @@ namespace Tgstation.Server.Host.Jobs return; } - var childLocalProgress = progress.Value * percentage; + var progressWithoutChild = sectionProgression - childLocalProgress; + childLocalProgress = progress.Value * percentage; + + // floating point >.< + sectionProgression = Math.Min(progressWithoutChild + childLocalProgress, 1); + if (sectionProgression > 9.9999) + sectionProgression = 1; - sectionProgression = childLocalProgress + childBaseProgress; callback(currentStage, sectionProgression); - }); + }, + false); newReporter.ReportProgress(0); return newReporter; diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index e9967f5189..d78cc7a287 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -460,14 +460,16 @@ namespace Tgstation.Server.Host.Jobs QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); + using var progressReporter = new JobProgressReporter( + loggerFactory.CreateLogger(), + null, + UpdateProgress); + using var innerReporter = progressReporter.CreateSection(null, 1.0); await operation( instanceCoreProvider.GetInstance(job.Instance!), databaseContextFactory, job, - new JobProgressReporter( - loggerFactory.CreateLogger(), - null, - UpdateProgress), + innerReporter, cancellationToken); logger.LogDebug("Job {jobId} completed!", job.Id); diff --git a/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs new file mode 100644 index 0000000000..6d2ff1be5b --- /dev/null +++ b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs @@ -0,0 +1,33 @@ +using System; +using System.Reflection; + +namespace Tgstation.Server.Host.Properties +{ + /// + /// Attribute for bundling the GitHub App serialized private key used for version telemetry. + /// + [AttributeUsage(AttributeTargets.Assembly)] + sealed class TelemetryAppSerializedKeyAttribute : Attribute + { + /// + /// Return the 's instance of the . + /// + public static TelemetryAppSerializedKeyAttribute? Instance => Assembly + .GetExecutingAssembly() + .GetCustomAttribute(); + + /// + /// The serialized GitHub App Client ID and private key. + /// + public string SerializedKey { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public TelemetryAppSerializedKeyAttribute(string serializedKey) + { + SerializedKey = serializedKey ?? throw new ArgumentNullException(nameof(serializedKey)); + } + } +} diff --git a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs index 3f3180f9da..d535d97884 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs @@ -60,12 +60,12 @@ namespace Tgstation.Server.Host.Security.OAuth { logger.LogTrace("Validating response code..."); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var token = await gitHubService.CreateOAuthAccessToken(oAuthConfiguration, code, cancellationToken); if (token == null) return null; - var authenticatedClient = gitHubServiceFactory.CreateService(token); + var authenticatedClient = await gitHubServiceFactory.CreateService(token, cancellationToken); logger.LogTrace("Getting user details..."); var userId = await authenticatedClient.GetCurrentUserId(cancellationToken); diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 36d0bf3111..32e82877e5 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -713,7 +713,7 @@ namespace Tgstation.Server.Host.Setup while (true); await console.WriteAsync(null, true, cancellationToken); - await console.WriteAsync("Enter a GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); + await console.WriteAsync("Enter a classic GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); await console.WriteAsync("GitHub personal access token: ", false, cancellationToken); newGeneralConfiguration.GitHubAccessToken = await console.ReadLineAsync(true, cancellationToken); if (String.IsNullOrWhiteSpace(newGeneralConfiguration.GitHubAccessToken)) @@ -958,6 +958,31 @@ namespace Tgstation.Server.Host.Setup }; } + /// + /// Prompts the user to create a . + /// + /// The for the operation. + /// A resulting in the new . + async ValueTask ConfigureTelemetry(CancellationToken cancellationToken) + { + bool enableReporting = await PromptYesNo("Enable version telemetry? This anonymously reports the TGS version in use.", true, cancellationToken); + + string? serverFriendlyName = null; + if (enableReporting) + { + await console.WriteAsync("(Optional) Publically associate your reported version with a friendly name:", false, cancellationToken); + serverFriendlyName = await console.ReadLineAsync(false, cancellationToken); + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + } + + return new TelemetryConfiguration + { + DisableVersionReporting = !enableReporting, + ServerFriendlyName = serverFriendlyName, + }; + } + /// /// Saves a given set to . /// @@ -969,6 +994,7 @@ namespace Tgstation.Server.Host.Setup /// The to save. /// The to save. /// The to save. + /// The to save. /// The for the operation. /// A representing the running operation. async ValueTask SaveConfiguration( @@ -980,6 +1006,7 @@ namespace Tgstation.Server.Host.Setup ElasticsearchConfiguration? elasticsearchConfiguration, ControlPanelConfiguration controlPanelConfiguration, SwarmConfiguration? swarmConfiguration, + TelemetryConfiguration? telemetryConfiguration, CancellationToken cancellationToken) { newGeneralConfiguration.ApiPort = hostingPort ?? GeneralConfiguration.DefaultApiPort; @@ -992,6 +1019,7 @@ namespace Tgstation.Server.Host.Setup { ElasticsearchConfiguration.Section, elasticsearchConfiguration }, { ControlPanelConfiguration.Section, controlPanelConfiguration }, { SwarmConfiguration.Section, swarmConfiguration }, + { TelemetryConfiguration.Section, telemetryConfiguration }, }; var versionConverter = new VersionConverter(); @@ -1065,6 +1093,8 @@ namespace Tgstation.Server.Host.Setup var swarmConfiguration = await ConfigureSwarm(cancellationToken); + var telemetryConfiguration = await ConfigureTelemetry(cancellationToken); + await console.WriteAsync(null, true, cancellationToken); await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Configuration complete! Saving to {0}", userConfigFileName), true, cancellationToken); @@ -1077,6 +1107,7 @@ namespace Tgstation.Server.Host.Setup elasticSearchConfiguration, controlPanelConfiguration, swarmConfiguration, + telemetryConfiguration, cancellationToken); } @@ -1194,6 +1225,7 @@ namespace Tgstation.Server.Host.Setup AllowAnyOrigin = true, }, null, + null, cancellationToken); } else diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index 92f6bdb159..6ae88dce50 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -13,6 +13,11 @@ namespace Tgstation.Server.Host.System /// Task Lifetime { get; } + /// + /// Gets the process' memory usage in bytes. + /// + long? MemoryUsage { get; } + /// /// Set's the owned to a non-normal value. /// diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 896195944c..13a1a278f5 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -22,6 +22,23 @@ namespace Tgstation.Server.Host.System /// public Task Lifetime { get; } + /// + public long? MemoryUsage + { + get + { + try + { + return handle.VirtualMemorySize64; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to get PID {pid}'s memory usage!", Id); + return null; + } + } + } + /// /// The for the . /// diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index cf412dc3ce..8fc7dca66b 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -29,13 +29,13 @@ - + - + @@ -45,21 +45,43 @@ - + <_Parameter1>$(TgsConfigVersion) <_Parameter2>$(TgsInteropVersion) <_Parameter3>$(TgsWebpanelVersion) <_Parameter4>$(TgsHostWatchdogVersion) <_Parameter5>$(TgsMariaDBRedistVersion) - + - - + + + + + + + + + <_Parameter1>@(SerializedTelemetryKey) + + + + + + + + + + + + @@ -78,21 +100,21 @@ - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + @@ -115,9 +137,9 @@ - + - + diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index e479388b79..4cf31a5d28 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -1,9 +1,16 @@ using System; using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + using Octokit; using Tgstation.Server.Host.Configuration; @@ -12,7 +19,7 @@ using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Utils.GitHub { /// - sealed class GitHubClientFactory : IGitHubClientFactory + sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable { /// /// Limit to the amount of days a can live in the . @@ -45,6 +52,11 @@ namespace Tgstation.Server.Host.Utils.GitHub /// readonly Dictionary clientCache; + /// + /// The used to guard access to . + /// + readonly SemaphoreSlim clientCacheSemaphore; + /// /// Initializes a new instance of the class. /// @@ -61,50 +73,139 @@ namespace Tgstation.Server.Host.Utils.GitHub generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); clientCache = new Dictionary(); + clientCacheSemaphore = new SemaphoreSlim(1, 1); } /// - public IGitHubClient CreateClient() => GetOrCreateClient(generalConfiguration.GitHubAccessToken); + public void Dispose() => clientCacheSemaphore.Dispose(); /// - public IGitHubClient CreateClient(string accessToken) - => GetOrCreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken))); + public async ValueTask CreateClient(CancellationToken cancellationToken) + => (await GetOrCreateClient( + generalConfiguration.GitHubAccessToken, + null, + cancellationToken))!; + + /// + public async ValueTask CreateClient(string accessToken, CancellationToken cancellationToken) + => (await GetOrCreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + null, + cancellationToken))!; + + /// + public ValueTask CreateInstallationClient(string serializedPem, long repositoryId, CancellationToken cancellationToken) + => GetOrCreateClient(serializedPem, repositoryId, cancellationToken); /// - /// Retrieve a from the or add a new one based on a given . + /// Retrieve a from the or add a new one based on a given . /// - /// Optional access token to use as credentials. - /// The for the given . - GitHubClient GetOrCreateClient(string? accessToken) + /// Optional access token to use as credentials or GitHub App private key. If using a private key, must be set. + /// Setting this specifies is a private key and a GitHub App installation authenticated client will be returned. + /// The for the operation. + /// A resulting in the for the given or if authentication failed. +#pragma warning disable CA1506 // TODO: Decomplexify + async ValueTask GetOrCreateClient(string? accessTokenOrSerializedPem, long? installationRepositoryId, CancellationToken cancellationToken) +#pragma warning restore CA1506 { GitHubClient client; bool cacheHit; DateTimeOffset? lastUsed; - lock (clientCache) + using (await SemaphoreSlimContext.Lock(clientCacheSemaphore, cancellationToken)) { string cacheKey; - if (String.IsNullOrWhiteSpace(accessToken)) + if (String.IsNullOrWhiteSpace(accessTokenOrSerializedPem)) { - accessToken = null; + accessTokenOrSerializedPem = null; cacheKey = DefaultCacheKey; } else - cacheKey = accessToken; + cacheKey = accessTokenOrSerializedPem; cacheHit = clientCache.TryGetValue(cacheKey, out var tuple); var now = DateTimeOffset.UtcNow; if (!cacheHit) { + logger.LogTrace("Creating new GitHubClient..."); var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!; client = new GitHubClient( new ProductHeaderValue( product.Name, product.Version)); - if (accessToken != null) - client.Credentials = new Credentials(accessToken); + if (accessTokenOrSerializedPem != null) + { + if (installationRepositoryId.HasValue) + { + logger.LogTrace("Performing GitHub App authentication for installation on repository {installationRepositoryId}", installationRepositoryId.Value); + var splits = accessTokenOrSerializedPem.Split(':'); + if (splits.Length != 2) + { + logger.LogError("Failed to parse serialized Client ID & PEM! Expected 2 chunks, got {chunkCount}", splits.Length); + return null; + } + + byte[] pemBytes; + try + { + pemBytes = Convert.FromBase64String(splits[1]); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to parse supposed base64 PEM!"); + return null; + } + + var pem = Encoding.UTF8.GetString(pemBytes); + + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + + var signingCredentials = new SigningCredentials(new RsaSecurityKey(rsa), SecurityAlgorithms.RsaSha256); + var jwtSecurityTokenHandler = new JwtSecurityTokenHandler { SetDefaultTimesOnTokenCreation = false }; + + var nowDateTime = DateTime.UtcNow; + + var jwt = jwtSecurityTokenHandler.CreateToken(new SecurityTokenDescriptor + { + Issuer = splits[0], + Expires = nowDateTime.AddMinutes(10), + IssuedAt = nowDateTime, + SigningCredentials = signingCredentials, + }); + + var jwtStr = jwtSecurityTokenHandler.WriteToken(jwt); + + client.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); + + Installation installation; + try + { + installation = await client.GitHubApps.GetRepositoryInstallationForCurrent(installationRepositoryId.Value); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform app authentication!"); + return null; + } + + cancellationToken.ThrowIfCancellationRequested(); + try + { + var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id); + + client.Credentials = new Credentials(installToken.Token); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform installation authentication!"); + return null; + } + } + else + client.Credentials = new Credentials(accessTokenOrSerializedPem); + } clientCache.Add(cacheKey, (Client: client, LastUsed: now)); lastUsed = null; diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs index efa08d13bb..a38b84b271 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -44,13 +46,16 @@ namespace Tgstation.Server.Host.Utils.GitHub } /// - public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient()); + public async ValueTask CreateService(CancellationToken cancellationToken) + => CreateServiceImpl( + await gitHubClientFactory.CreateClient(cancellationToken)); /// - public IAuthenticatedGitHubService CreateService(string accessToken) + public async ValueTask CreateService(string accessToken, CancellationToken cancellationToken) => CreateServiceImpl( - gitHubClientFactory.CreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken)))); + await gitHubClientFactory.CreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + cancellationToken)); /// /// Create a . diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs index ec95fd16ab..1808b4ca3c 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs @@ -1,4 +1,7 @@ -using Octokit; +using System.Threading; +using System.Threading.Tasks; + +using Octokit; namespace Tgstation.Server.Host.Utils.GitHub { @@ -10,14 +13,25 @@ namespace Tgstation.Server.Host.Utils.GitHub /// /// Create a client. Low rate limit unless the server's GitHubAccessToken is set to bypass it. /// + /// The for the operation. /// A new . - IGitHubClient CreateClient(); + ValueTask CreateClient(CancellationToken cancellationToken); /// /// Create a client with authentication using a personal access token. /// /// The GitHub personal access token. + /// The for the operation. /// A new . - IGitHubClient CreateClient(string accessToken); + ValueTask CreateClient(string accessToken, CancellationToken cancellationToken); + + /// + /// Creates a GitHub App client for an installation. + /// + /// The private key . + /// The GitHub repository ID. + /// The for the operation. + /// A resulting in a new for the given or if authentication failed. + ValueTask CreateInstallationClient(string pem, long repositoryId, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs index f4e7be70f5..adb0ec88eb 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs @@ -1,4 +1,7 @@ -namespace Tgstation.Server.Host.Utils.GitHub +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Utils.GitHub { /// /// Factory for s. @@ -8,14 +11,16 @@ /// /// Create a . /// - /// A new . - public IGitHubService CreateService(); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(CancellationToken cancellationToken); /// /// Create an . /// /// The access token to use for communication with GitHub. - /// A new . - public IAuthenticatedGitHubService CreateService(string accessToken); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index a700ccc9cb..dc15d9db68 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -76,7 +76,7 @@ namespace Tgstation.Server.Host.Utils Name = "/tg/station 13", Url = new Uri("https://github.com/tgstation"), }, - Description = "A production scale tool for BYOND server management", + Description = "A production scale tool for DreamMaker server management", }); // Important to do this before applying our own filters diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 4868f519fd..353104e6de 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -3,7 +3,7 @@ General: # ConfigVersion: # Basic semver. Differs from TGS version to version. See changelog for current version MinimumPasswordLength: 15 # Minimum TGS user password length - GitHubAccessToken: # GitHub personal access token with no scopes used to bypass rate-limits + GitHubAccessToken: # A classic GitHub personal access token with no scopes used to bypass rate-limits SetupWizardMode: AutoDetect # If the interactive TGS setup wizard should run ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API @@ -23,6 +23,7 @@ Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes DelayCleaningFailedDeployments: false # If true, deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined + ForceUseDreamDaemonExe: false # If true, dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. FileLogging: Directory: # Directory in which log files are stored. Windows default: %PROGRAMDATA%/tgstation-server. Linux default: /var/log/tgstation-server Disable: true # Disable file logging entirely @@ -77,3 +78,7 @@ Swarm: # Should be left empty if using swarm mode is not desired # PublicAddress: # The public address of the swarm node # ControllerAddress: # Required on non-controller nodes. The internal address of the swarm controller's API'. Should be left empty on the controller itself # UpdateRequiredNodeCount: # The number of nodes expected to be in the swarm before initiating an update. This should count every server irrespective of whether or not they are the controller MINUS 1 +Telemetry: + DisableVersionReporting: false # Prevents you installation and the version you're using from being reported on the source repository's deployments list + ServerFriendlyName: null # Sets a friendly name for your server in reported telemetry. Must be unique. First come first serve + VersionReportingRepositoryId: 841149827 # GitHub repostiory ID where the tgs_version_telemetry workflow can be found diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index 01518a39b0..c092a103f7 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs index 2a62fea29a..8950acc6ab 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -52,7 +53,7 @@ namespace Tgstation.Server.Host.Components.Engine.Tests null, null, null, - null, + It.IsNotNull(), true, It.IsAny())) .Callback(() => ++cloneAttempts) @@ -85,7 +86,7 @@ namespace Tgstation.Server.Host.Components.Engine.Tests Engine = EngineType.OpenDream, SourceSHA = new string('a', Limits.MaximumCommitShaLength), }, - null, + new JobProgressReporter(), CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs index c4a17f5140..2cdd59d19b 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs @@ -15,6 +15,7 @@ using Remora.Rest.Core; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Repository.Tests { @@ -86,7 +87,7 @@ namespace Tgstation.Server.Host.Components.Repository.Tests null, null, null, - null, + new JobProgressReporter(), false, CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs new file mode 100644 index 0000000000..f6d6cd9348 --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + +namespace Tgstation.Server.Host.Jobs.Tests +{ + [TestClass] + public sealed class TestJobProgressReporter + { + string expectedStageName = null; + double? expectedProgress = null; + void Validate(string stageName, double? progress) + { + Assert.AreEqual(expectedStageName, stageName); + Assert.AreEqual(expectedProgress, progress); + } + + JobProgressReporter Setup() + { + expectedStageName = null; + expectedProgress = 0; + return new JobProgressReporter( + Mock.Of>(), + null, + Validate); + } + + [TestMethod] + public void TestBasicUsage() + { + var progressReporter = Setup(); + + expectedProgress = 0.4; + progressReporter.ReportProgress(0.4); + expectedProgress = 1.0; + progressReporter.ReportProgress(1.0); + } + + [TestMethod] + public void TestNestedUsage() + { + var progressReporter = Setup(); + + expectedStageName = "Test1"; + var subReporter1 = progressReporter.CreateSection("Test1", 0.5); + expectedProgress = 0.1; + subReporter1.ReportProgress(0.2); + expectedProgress = 0.4; + subReporter1.ReportProgress(0.8); + + expectedStageName = "Test2"; + var subReporter2 = progressReporter.CreateSection("Test2", 0.5); + + expectedStageName = "Test1"; + expectedProgress = 0.5; + subReporter1.ReportProgress(1); + + expectedStageName = "Test2"; + expectedProgress = 0.6; + subReporter2.ReportProgress(0.2); + expectedProgress = 1.0; + subReporter2.ReportProgress(1); + } + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs index 1aa3576277..5598673dc2 100644 --- a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs +++ b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs @@ -195,6 +195,8 @@ namespace Tgstation.Server.Host.Setup.Tests "y", // swarm config "n", + // telemetry config + "n", //saved, now for second run //this time use defaults amap String.Empty, @@ -234,6 +236,9 @@ namespace Tgstation.Server.Host.Setup.Tests "privatekey", "n", "http://controller.com", + // telemetry config + "y", + "telemetry name", //third run, we already hit all the code coverage so just get through it String.Empty, nameof(DatabaseType.MariaDB), @@ -271,7 +276,9 @@ namespace Tgstation.Server.Host.Setup.Tests "https://controllerinternal.com", "https://controllerpublic.com", "privatekey", - "y" + "y", + // telemetry config + "n", }; var inputPos = 0; diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index 5e2a7f71e9..261c136fc7 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 6f4fac0ae2..33058be75f 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -1,5 +1,6 @@ using System; using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -57,14 +58,14 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests mockOptions.SetupGet(x => x.Value).Returns(gc); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client = factory.CreateClient(); + var client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); Assert.AreEqual(AuthenticationType.Anonymous, credentials.AuthenticationType); gc.GitHubAccessToken = "asdfasdfasdfasdfasdfasdf"; - client = factory.CreateClient(); + client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -83,9 +84,9 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateClient(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateClient(null, CancellationToken.None).AsTask()); - var client = factory.CreateClient("asdf"); + var client = await factory.CreateClient("asdf", CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -96,7 +97,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests } [TestMethod] - public void TestClientCaching() + public async Task TestClientCaching() { var mockApp = new Mock(); mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable(); @@ -105,10 +106,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client1 = factory.CreateClient(); - var client2 = factory.CreateClient("asdf"); - var client3 = factory.CreateClient(); - var client4 = factory.CreateClient("asdf"); + var client1 = await factory.CreateClient(CancellationToken.None); + var client2 = await factory.CreateClient("asdf", CancellationToken.None); + var client3 = await factory.CreateClient(CancellationToken.None); + var client4 = await factory.CreateClient("asdf", CancellationToken.None); Assert.ReferenceEquals(client1, client3); Assert.ReferenceEquals(client2, client4); } diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs index daa9afdd4c..24212eaa93 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -28,27 +30,29 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests } [TestMethod] - public void TestCreateService() + public async Task TestCreateService() { var mockFactory = new Mock(); - mockFactory.Setup(x => x.CreateClient()).Returns(Mock.Of()).Verifiable(); +#pragma warning disable CA2012 // Use ValueTasks correctly + mockFactory.Setup(x => x.CreateClient(It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); var mockToken = "asdf"; - mockFactory.Setup(x => x.CreateClient(mockToken)).Returns(Mock.Of()).Verifiable(); + mockFactory.Setup(x => x.CreateClient(mockToken, It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); +#pragma warning restore CA2012 // Use ValueTasks correctly var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration()); var factory = new GitHubServiceFactory(mockFactory.Object, Mock.Of(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateService(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateService(null, CancellationToken.None).AsTask()); Assert.AreEqual(0, mockFactory.Invocations.Count); - var result1 = factory.CreateService(); + var result1 = await factory.CreateService(CancellationToken.None); Assert.IsNotNull(result1); - var result2 = factory.CreateService(mockToken); + var result2 = factory.CreateService(mockToken, CancellationToken.None); Assert.IsNotNull(result2); mockFactory.VerifyAll(); diff --git a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs index adfb009d69..afeabef6f5 100644 --- a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs @@ -1,10 +1,10 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; +using System; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; @@ -54,10 +54,24 @@ namespace Tgstation.Server.Tests.Live async Task TestRead(CancellationToken cancellationToken) { - var model = await client.Read(cancellationToken); + var model = await client.Read(false, cancellationToken); //we've released a few 5.x versions now, check the release checker is at least somewhat functional Assert.IsTrue(4 < model.LatestVersion.Major); + Assert.IsNotNull(model.TrackedRepositoryUrl); + Assert.IsTrue(model.GeneratedAt.HasValue); + Assert.IsTrue(model.GeneratedAt.Value <= DateTimeOffset.UtcNow); + + // test the cache + var newerModel = await client.Read(false, cancellationToken); + Assert.AreEqual(model.GeneratedAt, newerModel.GeneratedAt); + + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + + var newestModel = await client.Read(true, cancellationToken); + Assert.AreNotEqual(model.GeneratedAt, newestModel.GeneratedAt); + Assert.IsNotNull(newestModel.GeneratedAt); + Assert.IsTrue(model.GeneratedAt < newestModel.GeneratedAt); } } } diff --git a/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs index 2f65757782..d05faed9a6 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -18,13 +20,13 @@ namespace Tgstation.Server.Tests.Live this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public IGitHubService CreateService() => CreateDummyService(); + public ValueTask CreateService(CancellationToken cancellationToken) => ValueTask.FromResult(CreateDummyService()); - public IAuthenticatedGitHubService CreateService(string accessToken) + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(accessToken); - return CreateDummyService(); + return ValueTask.FromResult(CreateDummyService()); } TestingGitHubService CreateDummyService() => new TestingGitHubService(cryptographySuite, logger); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 9c2b44c962..67ed9e2178 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -22,6 +22,7 @@ using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -142,7 +143,7 @@ namespace Tgstation.Server.Tests.Live.Instance using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; // get the bytes for stable - return await byondInstaller.DownloadVersion(compatVersion, null, cancellationToken); + return await byondInstaller.DownloadVersion(compatVersion, new JobProgressReporter(), cancellationToken); } public async Task RunCompatTests( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index b3149556e5..70b7e9c030 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -262,7 +262,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(HubConnectionState.Disconnected, permlessConn.State); // force token refreshs - await Task.WhenAll(permedUser.Administration.Read(cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask()); + await Task.WhenAll(permedUser.Administration.Read(false, cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask()); if (!permlessPsId.HasValue) { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 0d6ecce864..86fe4ac911 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -116,6 +116,8 @@ namespace Tgstation.Server.Tests.Live.Instance await ApiAssert.ThrowsException(() => Checkout(new RepositoryUpdateRequest { Reference = "master", CheckoutSha = "286bb75" }, false, false, cancellationToken), ErrorCode.RepoMismatchShaAndReference); var updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "286bb75" }, false, false, cancellationToken); + await RecloneTest(cancellationToken); + // Fake SHA updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, true, false, cancellationToken); @@ -142,6 +144,23 @@ namespace Tgstation.Server.Tests.Live.Instance await TestMergeTests(updated, prNumber, cancellationToken); } + async ValueTask RecloneTest(CancellationToken cancellationToken) + { + var initialState = await repositoryClient.Read(cancellationToken); + Assert.IsNotNull(initialState.Reference); + Assert.IsNotNull(initialState.RevisionInformation); + Assert.IsNotNull(initialState.RevisionInformation.CommitSha); + Assert.IsNotNull(initialState.RevisionInformation.OriginCommitSha); + + var reclone = await repositoryClient.Reclone(cancellationToken); + await WaitForJob(reclone.ActiveJob, 70, false, null, cancellationToken); + + var newState = await repositoryClient.Read(cancellationToken); + Assert.AreEqual(initialState.Reference, newState.Reference); + Assert.AreEqual(initialState.RevisionInformation.CommitSha, newState.RevisionInformation.CommitSha); + Assert.AreEqual(initialState.RevisionInformation.OriginCommitSha, newState.RevisionInformation.OriginCommitSha); + } + async ValueTask Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken) { var newRef = isRef ? updated.Reference : updated.CheckoutSha; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 26bba79305..08f436b26d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -150,6 +150,7 @@ namespace Tgstation.Server.Tests.Live.Instance }, cancellationToken); Assert.AreEqual(47, updated.OpenDreamTopicPort); + Assert.IsFalse(updated.ImmediateMemoryUsage.HasValue); } catch (ConflictException ex) when (ex.ErrorCode == ErrorCode.PortNotAvailable) { @@ -721,6 +722,8 @@ namespace Tgstation.Server.Tests.Live.Instance await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); await GracefulWatchdogShutdown(cancellationToken); @@ -1365,6 +1368,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(daemonStatus.ActiveCompileJob); ValidateSessionId(daemonStatus, true); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); + Assert.AreEqual(initialStatus.ActiveCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; Assert.AreNotEqual(daemonStatus.ActiveCompileJob.EngineVersion, newerCompileJob.EngineVersion); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 645edac147..f9ed0b926a 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -155,6 +155,7 @@ namespace Tgstation.Server.Tests.Live $"General:OpenDreamGitUrl={OpenDreamUrl}", $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", + "Telemetry:DisableVersionReporting=true", }; swarmArgs = new List(); diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index 3e20609e60..30e6b9e78e 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -216,7 +216,7 @@ namespace Tgstation.Server.Tests.Live }; var badClient = clientFactory.CreateFromToken(serverClient.Url, newToken); - await ApiAssert.ThrowsException(() => badClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => badClient.Administration.Read(false, cancellationToken)); await ApiAssert.ThrowsException(() => badClient.ServerInformation(cancellationToken)); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index bfc4538c68..b34b14f066 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -670,7 +670,7 @@ namespace Tgstation.Server.Tests.Live "asdfasdfasdfasdf"); await using var node1BadClient = clientFactory.CreateFromToken(node1.RootUrl, controllerUserClient.Token); - await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(false, cancellationToken)); // check instance info is not shared var controllerInstance = await controllerClient.Instances.CreateOrAttach( diff --git a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs index d104dbd6d3..585dd56999 100644 --- a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs +++ b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs @@ -39,7 +39,7 @@ namespace Tgstation.Server.Tests.Live }); var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of>(), mockOptions.Object); - RealClient = gitHubClientFactory.CreateClient(); + RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult(); } public static async Task InitializeAndInject(CancellationToken cancellationToken) diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index e73731de40..59b4783994 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests () => { }); const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a"; - await repo.CheckoutObject(StartSha, null, null, true, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); + await repo.CheckoutObject(StartSha, null, null, true, false, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); Assert.AreEqual(Host.Components.Repository.Repository.NoReference, repo.Reference); diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b385653486..c59f86fd19 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,7 +1,6 @@ // This program is minimal effort and should be sent to remedial school using System; -using System.Buffers.Text; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -60,8 +59,8 @@ namespace Tgstation.Server.ReleaseNotes var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - var ciCompletionCheck = versionString.Equals("--ci-completion-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); + var waitCodecov = versionString.Equals("--wait-codecov", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease @@ -69,8 +68,8 @@ namespace Tgstation.Server.ReleaseNotes && !shaCheck && !fullNotes && !nuget - && !ciCompletionCheck - && !genToken) + && !genToken + && !waitCodecov) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -111,20 +110,7 @@ namespace Tgstation.Server.ReleaseNotes break; } - const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; - var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); - if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) - { - Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); - return 3; - } - var client = new GitHubClient(new Octokit.ProductHeaderValue("tgs_release_notes")); - if (!String.IsNullOrWhiteSpace(githubToken)) - { - client.Credentials = new Credentials(githubToken); - } - try { if (ensureRelease) @@ -135,11 +121,47 @@ namespace Tgstation.Server.ReleaseNotes return 454233; } - await GenerateAppCredentials(client, args[1]); + await GenerateAppCredentials(client, args[1], false); return await EnsureRelease(client); } + if (genToken) + { + if (args.Length < 3) + { + Console.WriteLine("Missing output file path or PEM Base64 for app authentication!"); + return 33847; + } + + bool toSS13 = args.Length > 3 && args[3].Equals("--spacestation13", StringComparison.OrdinalIgnoreCase); + await GenerateAppCredentials(client, args[2], toSS13); + + var token = client.Credentials.GetToken(); + var destPath = args[1]; + Directory.CreateDirectory(Path.GetDirectoryName(destPath)); + await File.WriteAllTextAsync(destPath, token); + return 0; + } + + const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; + var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); + if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) + { + Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); + return 3; + } + + if (!String.IsNullOrWhiteSpace(githubToken)) + { + client.Credentials = new Credentials(githubToken); + } + + if (waitCodecov) + { + return await CodecovCheck(client, Int64.Parse(args[1])); + } + if (linkWinget) { if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) @@ -151,38 +173,9 @@ namespace Tgstation.Server.ReleaseNotes return await Winget(client, actionsUrl, null); } - if (ciCompletionCheck) - { - if (args.Length < 3) - { - Console.WriteLine("Missing SHA or PEM Base64 for creating check run!"); - return 4543; - } - - return await CICompletionCheck(client, args[1], args[2]); - } - - - if (genToken) - { - if (args.Length < 3) - { - Console.WriteLine("Missing output file path or PEM Base64 for app authentication!"); - return 33847; - } - - await GenerateAppCredentials(client, args[2]); - - var token = client.Credentials.GetToken(); - var destPath = args[1]; - Directory.CreateDirectory(Path.GetDirectoryName(destPath)); - await File.WriteAllTextAsync(destPath, token); - return 0; - } - if (shaCheck) { - if(args.Length < 2) + if (args.Length < 2) { Console.WriteLine("Missing SHA for PR template!"); return 32; @@ -764,7 +757,7 @@ namespace Tgstation.Server.ReleaseNotes } if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal) || trimmedLine.StartsWith("/🆑", StringComparison.Ordinal)) { - if(!Enum.TryParse(targetComponent, out var component)) + if (!Enum.TryParse(targetComponent, out var component)) component = targetComponent.ToUpperInvariant() switch { "**CONFIGURATION**" or "CONFIGURATION" or "CONFIG" => Component.Configuration, @@ -890,7 +883,7 @@ Note: `` is the directory's name containing the manifest you're submitting }); var prToModify = userPrsOnWingetRepo.Items.OrderByDescending(pr => pr.Number).FirstOrDefault(); - if(prToModify == null) + if (prToModify == null) { Console.WriteLine("Could not find open winget-pkgs PR!"); return 31; @@ -1013,7 +1006,7 @@ Note: `` is the directory's name containing the manifest you're submitting .GroupBy(kvp => kvp.Key) .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); - foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + foreach (var maxVersionKvp in prResults.SelectMany(x => x.Item1) .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) .GroupBy(x => x.Key) .Select(group => { @@ -1039,7 +1032,7 @@ Note: `` is the directory's name containing the manifest you're submitting var component = componentKvp.Key; var list = new List(); - foreach(var changelistDict in prResults.Select(x => x.Item1)) + foreach (var changelistDict in prResults.Select(x => x.Item1)) { if (!changelistDict.TryGetValue(component, out var changelist)) continue; @@ -1143,7 +1136,7 @@ Note: `` is the directory's name containing the manifest you're submitting return 0; } - static readonly HttpClient httpClient = new ( + static readonly HttpClient httpClient = new( new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate @@ -1192,7 +1185,8 @@ Note: `` is the directory's name containing the manifest you're submitting release => release.Id); var milestones = await TripleCheckGitHubPagination( - apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest + { State = ItemStateFilter.All }, apiOptions), milestone => milestone.Id); @@ -1400,7 +1394,7 @@ Note: `` is the directory's name containing the manifest you're submitting PrintChanges(newNotes, relevantChangelog); } - if(component == Component.DreamMakerApi) + if (component == Component.DreamMakerApi) { newNotes.AppendLine(); newNotes.AppendLine("#tgs-dmapi-release"); @@ -1450,7 +1444,7 @@ Note: `` is the directory's name containing the manifest you're submitting { Component.NugetClient, "Client" }, }; - foreach(var kvp in csprojNameMap) + foreach (var kvp in csprojNameMap) { var component = kvp.Key; var csprojPath = CsprojSubstitution.Replace("$PROJECT$", kvp.Value); @@ -1588,7 +1582,7 @@ package (version) distribution(s); urgency=urgency builder.AppendLine(); builder.Append(" * The following changes are for "); builder.Append(GetComponentDisplayName(kvp.Key, true)); - if(kvp.Key == Component.Configuration) + if (kvp.Key == Component.Configuration) { builder.Append(". You "); if (kvp.Value.Version.Minor == 0 && kvp.Value.Version.Build == 0) @@ -1640,7 +1634,7 @@ package (version) distribution(s); urgency=urgency return 0; } - static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64) + static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64, bool toSS13) { var pemBytes = Convert.FromBase64String(pemBase64); var pem = Encoding.UTF8.GetString(pemBytes); @@ -1665,27 +1659,16 @@ package (version) distribution(s); urgency=urgency gitHubClient.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); - var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent(RepoOwner, RepoName); + var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent( + toSS13 + ? "spacestation13" + : RepoOwner, + RepoName); var installToken = await gitHubClient.GitHubApps.CreateInstallationToken(installation.Id); gitHubClient.Credentials = new Credentials(installToken.Token); } - static async ValueTask CICompletionCheck(GitHubClient gitHubClient, string currentSha, string pemBase64) - { - await GenerateAppCredentials(gitHubClient, pemBase64); - - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Completion", currentSha) - { - CompletedAt = DateTime.UtcNow, - Conclusion = CheckConclusion.Success, - Output = new NewCheckRunOutput("CI Completion", "The CI Pipeline completed successfully"), - Status = CheckStatus.Completed, - }); - - return 0; - } - static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release @@ -1695,5 +1678,21 @@ package (version) distribution(s); urgency=urgency else Debug.Assert(condition); } + + static async ValueTask CodecovCheck(IGitHubClient client, long runId) + { + var currentRun = await client.Actions.Workflows.Runs.Get(RepoOwner, RepoName, runId); + + bool foundRun = false; + for(int i = 0; i < 15 && !foundRun; ++i) + { + var allRuns = await client.Check.Run.GetAllForReference(RepoOwner, RepoName, currentRun.HeadSha); + foundRun = allRuns.CheckRuns.Any(x => x.CheckSuite.Id == currentRun.Id && x.Name == "codecov/project"); + if (!foundRun && i != 14) + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + return foundRun ? 0 : 24398; + } } }