diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0ef110cffd6..e678497f5e6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -45,6 +45,7 @@ /code/modules/preferences_menu.dm @Mothblocks /code/modules/preferences_savefile.dm @Mothblocks /tgui/packages/tgui/interfaces/PreferencesMenu/ @Mothblocks +/tools/screenshot-test-comparison/ @Mothblocks # MrMelbert diff --git a/.github/workflows/ci_suite.yml b/.github/workflows/ci_suite.yml index edc81edcb13..86baaafacbd 100644 --- a/.github/workflows/ci_suite.yml +++ b/.github/workflows/ci_suite.yml @@ -132,6 +132,48 @@ jobs: run: | source $HOME/BYOND/byond/bin/byondsetup bash tools/ci/run_server.sh ${{ matrix.map }} + - name: Upload screenshot tests + if: always() + uses: actions/upload-artifact@v3 + with: + name: test_artifacts_${{ matrix.map }} + path: data/screenshots_new/ + retention-days: 1 + + compare_screenshots: + if: "!contains(github.event.head_commit.message, '[ci skip]') && always()" + needs: [run_all_tests] + name: Compare Screenshot Tests + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + # If we ever add more artifacts, this is going to break, but it'll be obvious. + - name: Download screenshot tests + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: ls -R + run: ls -R artifacts + - name: Setup screenshot comparison + run: npm i + working-directory: tools/screenshot-test-comparison + - name: Run screenshot comparison + run: node tools/screenshot-test-comparison/index.js artifacts code/modules/unit_tests/screenshots artifacts/screenshot_comparisons + # workflow_run does not give you the PR it ran on, + # even through the thing literally named "matching pull requests". + # However, in GraphQL, you can check if the check suite was ran + # by a specific PR, so trusting the (user controlled) action here is okay, + # as long as we check it later in show_screenshot_test_results + - name: Save PR ID + if: failure() && github.event.pull_request + run: | + echo ${{ github.event.pull_request.number }} > artifacts/screenshot_comparisons/pull_request_number.txt + - name: Upload bad screenshots + if: failure() + uses: actions/upload-artifact@v3 + with: + name: bad-screenshots + path: artifacts/screenshot_comparisons test_windows: if: "!contains(github.event.head_commit.message, '[ci skip]')" diff --git a/.github/workflows/show_screenshot_test_results.yml b/.github/workflows/show_screenshot_test_results.yml new file mode 100644 index 00000000000..3e9125510b7 --- /dev/null +++ b/.github/workflows/show_screenshot_test_results.yml @@ -0,0 +1,43 @@ +# This is a separate workflow so that it can access secrets, which are necessary +# because we need to be able to upload the images and post a comment. +# In the event this workflow fails, the screenshot test results are still +# available as an artifact of the screenshot test comparison workflow itself. +# This simply provides necessary quality of life. +name: Show Screenshot Test Results +on: + workflow_run: + workflows: [CI Suite] + types: + - completed +jobs: + show_screenshot_test_results: + if: "!contains(github.event.head_commit.message, '[ci skip]')" + name: Show Screenshot Test Results + runs-on: ubuntu-20.04 + steps: + - name: "Check for ARTIFACTS_FILE_HOUSE_KEY" + id: secrets_set + env: + ENABLER_SECRET: ${{ secrets.ARTIFACTS_FILE_HOUSE_KEY }} + run: | + unset SECRET_EXISTS + if [ -n "$ENABLER_SECRET" ]; then SECRET_EXISTS=true ; fi + echo "::set-output name=SECRETS_ENABLED::$SECRET_EXISTS" + - name: Checkout + if: steps.secrets_set.outputs.SECRETS_ENABLED + uses: actions/checkout@v3 + - name: Prepare module + if: steps.secrets_set.outputs.SECRETS_ENABLED + run: | + # This is needed because node-fetch needs import and doesn't work with require :/ + echo "{\"type\": \"module\"}" > package.json + npm install node-fetch + - name: Show screenshot test results + if: steps.secrets_set.outputs.SECRETS_ENABLED + uses: actions/github-script@v6 + env: + FILE_HOUSE_KEY: ${{ secrets.ARTIFACTS_FILE_HOUSE_KEY }} + with: + script: | + const { showScreenshotTestResults } = await import('${{ github.workspace }}/tools/ci/show_screenshot_test_results.js') + await showScreenshotTestResults({ github, context, exec }) diff --git a/.gitignore b/.gitignore index 44537f0a6b7..10cdcfb1349 100644 --- a/.gitignore +++ b/.gitignore @@ -197,5 +197,8 @@ Temporary Items /tools/LinuxOneShot/TGS_Instances /tools/LinuxOneShot/TGS_Logs -# Autowiki -/tools/autowiki/node_modules +# JavaScript tools +**/node_modules + +# Screenshot tests +/artifacts diff --git a/code/modules/client/preferences/middleware/antags.dm b/code/modules/client/preferences/middleware/antags.dm index b782397fd5b..08936bcd23e 100644 --- a/code/modules/client/preferences/middleware/antags.dm +++ b/code/modules/client/preferences/middleware/antags.dm @@ -110,6 +110,9 @@ early = TRUE cross_round_cachable = TRUE + /// Mapping of spritesheet keys -> icons + var/list/antag_icons = list() + /datum/asset/spritesheet/antagonists/create_spritesheets() // Antagonists that don't have a dynamic ruleset, but do have a preference var/static/list/non_ruleset_antagonists = list( @@ -128,7 +131,6 @@ antagonists[initial(ruleset.antag_flag)] = antagonist_type var/list/generated_icons = list() - var/list/to_insert = list() for (var/antag_flag in antagonists) var/datum/antagonist/antagonist_type = antagonists[antag_flag] @@ -137,7 +139,7 @@ var/spritesheet_key = serialize_antag_name(antag_flag) if (!isnull(generated_icons[antagonist_type])) - to_insert[spritesheet_key] = generated_icons[antagonist_type] + antag_icons[spritesheet_key] = generated_icons[antagonist_type] continue var/datum/antagonist/antagonist = new antagonist_type @@ -152,10 +154,10 @@ // If an icon is not prepared to be scaled to that size, it looks really ugly, and this // makes it harder to figure out what size it *actually* is. generated_icons[antagonist_type] = preview_icon - to_insert[spritesheet_key] = preview_icon + antag_icons[spritesheet_key] = preview_icon - for (var/spritesheet_key in to_insert) - Insert(spritesheet_key, to_insert[spritesheet_key]) + for (var/spritesheet_key in antag_icons) + Insert(spritesheet_key, antag_icons[spritesheet_key]) /// Serializes an antag name to be used for preferences UI /proc/serialize_antag_name(antag_name) diff --git a/code/modules/mob/living/carbon/human/dummy.dm b/code/modules/mob/living/carbon/human/dummy.dm index c8cc678f10d..ae1dd2b4c84 100644 --- a/code/modules/mob/living/carbon/human/dummy.dm +++ b/code/modules/mob/living/carbon/human/dummy.dm @@ -102,7 +102,8 @@ INITIALIZE_IMMEDIATE(/mob/living/carbon/human/dummy) dna.features["snout"] = "Round" dna.features["spines"] = "None" dna.features["tail_cat"] = "None" - dna.features["tail_lizard"] = "Light" + dna.features["tail_lizard"] = "Smooth" + dna.features["pod_hair"] = "Ivy" //Inefficient pooling/caching way. GLOBAL_LIST_EMPTY(human_dummy_list) diff --git a/code/modules/unit_tests/_unit_tests.dm b/code/modules/unit_tests/_unit_tests.dm index 2b3918a9531..699f215d0e0 100644 --- a/code/modules/unit_tests/_unit_tests.dm +++ b/code/modules/unit_tests/_unit_tests.dm @@ -129,6 +129,9 @@ #include "reagent_recipe_collisions.dm" #include "resist.dm" #include "say.dm" +#include "screenshot_antag_icons.dm" +#include "screenshot_basic.dm" +#include "screenshot_humanoids.dm" #include "security_officer_distribution.dm" #include "serving_tray.dm" #include "siunit.dm" diff --git a/code/modules/unit_tests/screenshot_antag_icons.dm b/code/modules/unit_tests/screenshot_antag_icons.dm new file mode 100644 index 00000000000..0a210ae31ed --- /dev/null +++ b/code/modules/unit_tests/screenshot_antag_icons.dm @@ -0,0 +1,12 @@ +/// A screenshot test to make sure every antag icon in the preferences menu is consistent +/datum/unit_test/screenshot_antag_icons + +/datum/unit_test/screenshot_antag_icons/Run() + var/datum/asset/spritesheet/antagonists/antagonists = get_asset_datum(/datum/asset/spritesheet/antagonists) + + for (var/antag_icon_key in antagonists.antag_icons) + var/icon/reference_icon = antagonists.antag_icons[antag_icon_key] + + var/icon/icon = new() + icon.Insert(reference_icon, null, SOUTH, 1) + test_screenshot(antag_icon_key, icon) diff --git a/code/modules/unit_tests/screenshot_basic.dm b/code/modules/unit_tests/screenshot_basic.dm new file mode 100644 index 00000000000..350514f007f --- /dev/null +++ b/code/modules/unit_tests/screenshot_basic.dm @@ -0,0 +1,8 @@ +/// This is an example for screenshot tests, and a meta-test to make sure they work in the success case. +/// It creates a picture that is red on the left side, green on the other. +/datum/unit_test/screenshot_basic + +/datum/unit_test/screenshot_basic/Run() + var/icon/red = icon('icons/blanks/32x32.dmi', "nothing") + red.Blend(COLOR_RED, ICON_OVERLAY) + test_screenshot("red", red) diff --git a/code/modules/unit_tests/screenshot_humanoids.dm b/code/modules/unit_tests/screenshot_humanoids.dm new file mode 100644 index 00000000000..196d946cb12 --- /dev/null +++ b/code/modules/unit_tests/screenshot_humanoids.dm @@ -0,0 +1,44 @@ +/// A screenshot test for every humanoid species with a handful of jobs. +/datum/unit_test/screenshot_humanoids + +/datum/unit_test/screenshot_humanoids/Run() + // Test lizards as their own thing so we can get more coverage on their features + var/mob/living/carbon/human/lizard = allocate(/mob/living/carbon/human/dummy/consistent) + lizard.dna.features["mcolor"] = "#099" + lizard.dna.features["tail_lizard"] = "Light Tiger" + lizard.dna.features["snout"] = "Sharp + Light" + lizard.dna.features["horns"] = "Simple" + lizard.dna.features["frills"] = "Aquatic" + lizard.dna.features["legs"] = "Normal Legs" + lizard.set_species(/datum/species/lizard) + lizard.equipOutfit(/datum/outfit/job/engineer) + test_screenshot("[/datum/species/lizard]", get_flat_icon_for_all_directions(lizard)) + + // let me have this + var/mob/living/carbon/human/moth = allocate(/mob/living/carbon/human/dummy/consistent) + moth.dna.features["moth_antennae"] = "Firewatch" + moth.dna.features["moth_markings"] = "None" + moth.dna.features["moth_wings"] = "Firewatch" + moth.set_species(/datum/species/moth) + moth.equipOutfit(/datum/outfit/job/cmo, visualsOnly = TRUE) + test_screenshot("[/datum/species/moth]", get_flat_icon_for_all_directions(moth)) + + // The rest of the species + for (var/datum/species/species_type as anything in subtypesof(/datum/species) - /datum/species/moth - /datum/species/lizard) + test_screenshot("[species_type]", get_flat_icon_for_all_directions(make_dummy(species_type, /datum/outfit/job/assistant/consistent))) + +/datum/unit_test/screenshot_humanoids/proc/get_flat_icon_for_all_directions(atom/thing) + var/icon/output = icon('icons/effects/effects.dmi', "nothing") + COMPILE_OVERLAYS(thing) + + for (var/direction in GLOB.cardinals) + var/icon/partial = getFlatIcon(thing, defdir = direction, no_anim = TRUE) + output.Insert(partial, dir = direction) + + return output + +/datum/unit_test/screenshot_humanoids/proc/make_dummy(species, job_outfit) + var/mob/living/carbon/human/dummy/consistent/dummy = allocate(/mob/living/carbon/human/dummy/consistent) + dummy.set_species(species) + dummy.equipOutfit(job_outfit, visualsOnly = TRUE) + return dummy diff --git a/code/modules/unit_tests/screenshots/README.md b/code/modules/unit_tests/screenshots/README.md new file mode 100644 index 00000000000..1d50a4e5612 --- /dev/null +++ b/code/modules/unit_tests/screenshots/README.md @@ -0,0 +1,18 @@ +This folder contains the results for screenshot tests. Screenshot tests make sure an icon looks the same as it did before a change to prevent regressions. + +You can create one by simply using the `test_screenshot` proc. + +This example test screenshots a red image and keeps it. + +```dm +/// This is an example for screenshot tests, and a meta-test to make sure they work in the success case. +/// It creates a picture that is red on the left side, green on the other. +/datum/unit_test/screenshot_basic + +/datum/unit_test/screenshot_basic/Run() + var/icon/red = icon('icons/blanks/32x32.dmi', "nothing") + red.Blend(COLOR_RED, ICON_OVERLAY) + test_screenshot("red", red) +``` + +Unfortunately, screenshot tests are sanest to test through a pull request directly, due to limitations with both DM and GitHub. diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_abductor.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_abductor.png new file mode 100644 index 00000000000..c0503326ddb Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_abductor.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_blob.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_blob.png new file mode 100644 index 00000000000..f21c6979c1a Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_blob.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_blobinfection.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_blobinfection.png new file mode 100644 index 00000000000..e3d7acbf5d8 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_blobinfection.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_bloodbrother.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_bloodbrother.png new file mode 100644 index 00000000000..6e604eecbc6 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_bloodbrother.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_changeling.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_changeling.png new file mode 100644 index 00000000000..de9743cf1ac Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_changeling.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_clownoperative.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_clownoperative.png new file mode 100644 index 00000000000..d61c10e7487 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_clownoperative.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_cultist.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_cultist.png new file mode 100644 index 00000000000..be8440660c9 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_cultist.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_familyheadaspirant.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_familyheadaspirant.png new file mode 100644 index 00000000000..5c3169087d0 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_familyheadaspirant.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_fugitive.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_fugitive.png new file mode 100644 index 00000000000..d1187a14d84 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_fugitive.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_gangster.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_gangster.png new file mode 100644 index 00000000000..5c3169087d0 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_gangster.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_headrevolutionary.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_headrevolutionary.png new file mode 100644 index 00000000000..7a1b6f6913d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_headrevolutionary.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_heretic.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_heretic.png new file mode 100644 index 00000000000..73bc5e02dd3 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_heretic.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_hereticsmuggler.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_hereticsmuggler.png new file mode 100644 index 00000000000..73bc5e02dd3 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_hereticsmuggler.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_loneoperative.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_loneoperative.png new file mode 100644 index 00000000000..d4be21a24a7 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_loneoperative.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfai.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfai.png new file mode 100644 index 00000000000..993fbc0b308 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfai.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfaimidround.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfaimidround.png new file mode 100644 index 00000000000..993fbc0b308 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_malfaimidround.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_nightmare.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_nightmare.png new file mode 100644 index 00000000000..3b723129ac8 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_nightmare.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_obsessed.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_obsessed.png new file mode 100644 index 00000000000..1d5f88bd840 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_obsessed.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_operative.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_operative.png new file mode 100644 index 00000000000..450f465ee80 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_operative.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_operativemidround.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_operativemidround.png new file mode 100644 index 00000000000..450f465ee80 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_operativemidround.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_opportunist.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_opportunist.png new file mode 100644 index 00000000000..63668193247 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_opportunist.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_provocateur.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_provocateur.png new file mode 100644 index 00000000000..7a1b6f6913d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_provocateur.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_revenant.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_revenant.png new file mode 100644 index 00000000000..eccedaabc01 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_revenant.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_sentientdisease.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_sentientdisease.png new file mode 100644 index 00000000000..e7e1cbd661f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_sentientdisease.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_spacedragon.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_spacedragon.png new file mode 100644 index 00000000000..f81dadfcf56 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_spacedragon.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_spaceninja.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_spaceninja.png new file mode 100644 index 00000000000..a54028e1be9 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_spaceninja.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicateinfiltrator.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicateinfiltrator.png new file mode 100644 index 00000000000..62e23dad5c7 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicateinfiltrator.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicatesleeperagent.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicatesleeperagent.png new file mode 100644 index 00000000000..62e23dad5c7 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_syndicatesleeperagent.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_thief.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_thief.png new file mode 100644 index 00000000000..63668193247 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_thief.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_traitor.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_traitor.png new file mode 100644 index 00000000000..62e23dad5c7 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_traitor.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizard.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizard.png new file mode 100644 index 00000000000..350266c8979 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizard.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizardmidround.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizardmidround.png new file mode 100644 index 00000000000..350266c8979 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_wizardmidround.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_antag_icons_xenomorph.png b/code/modules/unit_tests/screenshots/screenshot_antag_icons_xenomorph.png new file mode 100644 index 00000000000..fe089cfc4ec Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_antag_icons_xenomorph.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_basic_red.png b/code/modules/unit_tests/screenshots/screenshot_basic_red.png new file mode 100644 index 00000000000..280ec2883f8 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_basic_red.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_abductor.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_abductor.png new file mode 100644 index 00000000000..d4742750320 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_abductor.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_android.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_android.png new file mode 100644 index 00000000000..413fe56c47f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_android.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_dullahan.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_dullahan.png new file mode 100644 index 00000000000..12eec6affc3 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_dullahan.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_ethereal.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_ethereal.png new file mode 100644 index 00000000000..b38c363378c Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_ethereal.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_fly.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_fly.png new file mode 100644 index 00000000000..205d91bcef3 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_fly.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem.png new file mode 100644 index 00000000000..240e8270f34 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_adamantine.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_adamantine.png new file mode 100644 index 00000000000..78d706f1326 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_adamantine.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_alloy.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_alloy.png new file mode 100644 index 00000000000..e9199bfc305 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_alloy.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bananium.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bananium.png new file mode 100644 index 00000000000..28734ed1c17 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bananium.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bluespace.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bluespace.png new file mode 100644 index 00000000000..7a2b4b50700 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bluespace.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bone.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bone.png new file mode 100644 index 00000000000..11ffef30da3 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bone.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bronze.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bronze.png new file mode 100644 index 00000000000..0fb6bfa0597 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_bronze.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cardboard.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cardboard.png new file mode 100644 index 00000000000..cfc30dee8bd Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cardboard.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cloth.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cloth.png new file mode 100644 index 00000000000..86eb0a04904 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_cloth.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_diamond.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_diamond.png new file mode 100644 index 00000000000..8701d58ca5e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_diamond.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_durathread.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_durathread.png new file mode 100644 index 00000000000..c35eb550b5f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_durathread.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_glass.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_glass.png new file mode 100644 index 00000000000..b49db9e679d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_glass.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_gold.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_gold.png new file mode 100644 index 00000000000..e1353ea1968 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_gold.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_leather.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_leather.png new file mode 100644 index 00000000000..547484abd05 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_leather.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_mhydrogen.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_mhydrogen.png new file mode 100644 index 00000000000..b7ba888b71a Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_mhydrogen.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasma.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasma.png new file mode 100644 index 00000000000..8265865d458 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasma.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasteel.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasteel.png new file mode 100644 index 00000000000..da26728fe39 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plasteel.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastic.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastic.png new file mode 100644 index 00000000000..7e4c781f712 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastic.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastitanium.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastitanium.png new file mode 100644 index 00000000000..32e9549dc5b Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_plastitanium.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_runic.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_runic.png new file mode 100644 index 00000000000..e3a43f47b16 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_runic.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_sand.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_sand.png new file mode 100644 index 00000000000..d983ed55a0f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_sand.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_silver.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_silver.png new file mode 100644 index 00000000000..b7ba888b71a Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_silver.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_snow.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_snow.png new file mode 100644 index 00000000000..03e8f550f7f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_snow.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_titanium.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_titanium.png new file mode 100644 index 00000000000..7e4c781f712 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_titanium.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_uranium.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_uranium.png new file mode 100644 index 00000000000..7db2eb454fe Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_uranium.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_wood.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_wood.png new file mode 100644 index 00000000000..868ad85331c Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_golem_wood.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human.png new file mode 100644 index 00000000000..767a2ec704d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_felinid.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_felinid.png new file mode 100644 index 00000000000..27e6f575d7a Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_felinid.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_krokodil_addict.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_krokodil_addict.png new file mode 100644 index 00000000000..7f76aaf6d9d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_human_krokodil_addict.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly.png new file mode 100644 index 00000000000..8951c16232e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_luminescent.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_luminescent.png new file mode 100644 index 00000000000..690344a8ab5 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_luminescent.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_slime.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_slime.png new file mode 100644 index 00000000000..709d6d455fe Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_slime.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_stargazer.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_stargazer.png new file mode 100644 index 00000000000..8951c16232e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_jelly_stargazer.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard.png new file mode 100644 index 00000000000..955d413d50e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_ashwalker.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_ashwalker.png new file mode 100644 index 00000000000..24b9dc784bb Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_ashwalker.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_silverscale.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_silverscale.png new file mode 100644 index 00000000000..12ee5dc1c7e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_lizard_silverscale.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_monkey.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_monkey.png new file mode 100644 index 00000000000..6e2d1a6114c Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_monkey.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_moth.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_moth.png new file mode 100644 index 00000000000..87a567e7dc2 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_moth.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_mush.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_mush.png new file mode 100644 index 00000000000..c8527bfe56e Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_mush.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_plasmaman.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_plasmaman.png new file mode 100644 index 00000000000..94b05c68a8f Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_plasmaman.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_pod.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_pod.png new file mode 100644 index 00000000000..f77e21d1f52 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_pod.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow.png new file mode 100644 index 00000000000..0d232158466 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow_nightmare.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow_nightmare.png new file mode 100644 index 00000000000..e6b8dc6ef75 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_shadow_nightmare.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_skeleton.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_skeleton.png new file mode 100644 index 00000000000..bb5be6b7d5a Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_skeleton.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_snail.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_snail.png new file mode 100644 index 00000000000..716c6e1570d Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_snail.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_vampire.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_vampire.png new file mode 100644 index 00000000000..a29c38b292c Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_vampire.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie.png new file mode 100644 index 00000000000..7b0d5c736d5 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie.png differ diff --git a/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie_infectious.png b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie_infectious.png new file mode 100644 index 00000000000..7b0d5c736d5 Binary files /dev/null and b/code/modules/unit_tests/screenshots/screenshot_humanoids__datum_species_zombie_infectious.png differ diff --git a/code/modules/unit_tests/unit_test.dm b/code/modules/unit_tests/unit_test.dm index 4359d2f1de0..ea8e75c9e4a 100644 --- a/code/modules/unit_tests/unit_test.dm +++ b/code/modules/unit_tests/unit_test.dm @@ -84,6 +84,30 @@ GLOBAL_LIST_EMPTY(unit_test_mapping_logs) allocated += instance return instance +/datum/unit_test/proc/test_screenshot(name, icon/icon) + if (!istype(icon)) + TEST_FAIL("[icon] is not an icon.") + return + + var/path_prefix = replacetext(replacetext("[type]", "/datum/unit_test/", ""), "/", "_") + name = replacetext(name, "/", "_") + + var/filename = "code/modules/unit_tests/screenshots/[path_prefix]_[name].png" + + if (fexists(filename)) + var/data_filename = "data/screenshots/[path_prefix]_[name].png" + fcopy(icon, data_filename) + log_test("[path_prefix]_[name] was found, putting in data/screenshots") + else if (fexists("code")) + // We are probably running in a local build + fcopy(icon, filename) + TEST_FAIL("Screenshot for [name] did not exist. One has been created.") + else + // We are probably running in real CI, so just pretend it worked and move on + fcopy(icon, "data/screenshots_new/[path_prefix]_[name].png") + + log_test("[path_prefix]_[name] was put in data/screenshots_new") + /proc/RunUnitTest(test_path, list/test_results) var/datum/unit_test/test = new test_path diff --git a/tools/ci/run_server.sh b/tools/ci/run_server.sh index f9eaa31e566..baf172d8ec3 100644 --- a/tools/ci/run_server.sh +++ b/tools/ci/run_server.sh @@ -17,5 +17,10 @@ cp _maps/$MAP.json ci_test/data/next_map.json cd ci_test DreamDaemon tgstation.dmb -close -trusted -verbose -params "log-directory=ci" + cd .. + +mkdir -p data/screenshots_new +cp -r ci_test/data/screenshots_new data/screenshots_new + cat ci_test/data/logs/ci/clean_run.lk diff --git a/tools/ci/show_screenshot_test_results.js b/tools/ci/show_screenshot_test_results.js new file mode 100644 index 00000000000..607ad9854d6 --- /dev/null +++ b/tools/ci/show_screenshot_test_results.js @@ -0,0 +1,236 @@ +import fetch, { FormData, fileFrom } from "node-fetch"; +import fs from "fs"; +import path from "path"; +import process from "process"; + +const createComment = (screenshotFailures, zipFileUrl) => { + const formatScreenshotFailure = ({ directory, diffUrl, newUrl, oldUrl }) => { + const img = (url) => { + if (url) { + return `![](${url})`; + } else { + return "None produced."; + } + }; + + return `| ${directory} | ${img(oldUrl)} | ${img(newUrl)} | ${img(diffUrl)} |`; + }; + + return ` + Screenshot tests failed! + + ${zipFileUrl ? `[Download zip file of new screenshots.](${zipFileUrl})` : "No zip file could be produced, this is a bug!"} + + ## Diffs +
+ See snapshot diffs + + | Name | Expected image | Produced image | Diff | + | :--: | :------------: | :------------: | :--: | + ${screenshotFailures.map(formatScreenshotFailure).join("\n")} +
+ + ## Help +
+ What is this? + + Screenshot tests make sure that specific icons look the same as they did before. + This is important for elements that often mistakenly change, such as alien species. + + If the produced image looks broken, then it is possible your code caused a bug. + Make sure to test in game to see if you can fix it. +
+ +
+ I am changing sprites, it's supposed to look different. + + If the newly produced sprites are correct, then the tests should be updated. + + You can either: + + 1. Right-click the "produced image", and save it in \`code/modules/unit_tests/screenshots/NAME.png\`. + 2. Download and extract [this zip file](${zipFileUrl}) in the root of your repository, and commit. + + If you need help, you can ask maintainers either on Discord or on this pull request. +
+ +
+ This is a false positive. + + If you are sure your code did not cause this failure, especially if it's inconsistent, + then you may have found a false positive. + + Ask maintainers to rerun the test. + + If you need help, you can ask maintainers either on Discord or on this pull request. +
+ `.replace(/\t/g, ''); // If we keep tabs, it'll become a code block. +}; + +export async function showScreenshotTestResults({ github, context, exec }) { + const { FILE_HOUSE_KEY } = process.env; + + // Check if bad-screenshots is in the artifacts + const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + + const badScreenshots = artifacts.find(({ name }) => name === 'bad-screenshots'); + if (!badScreenshots) { + console.log("No bad screenshots found"); + return; + } + + // Download the screenshots from the artifacts + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: badScreenshots.id, + archive_format: "zip", + }); + + fs.writeFileSync("bad-screenshots.zip", Buffer.from(download.data)); + + await exec.exec("unzip bad-screenshots.zip -d bad-screenshots"); + + const prNumberFile = path.join("bad-screenshots", "pull_request_number.txt"); + + if (!fs.existsSync(prNumberFile)) { + console.log("No PR number found"); + return; + } + + const prNumber = parseInt(fs.readFileSync(prNumberFile, "utf8"), 10); + if (!prNumber) { + console.log("No PR number found"); + return; + } + + fs.rmSync(prNumberFile); + + // Validate the PR + const result = await github.graphql(`query($owner:String!, $repo:String!, $prNumber:Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $prNumber) { + commits(last: 1) { + nodes { + commit { + checkSuites(first: 10) { + nodes { + id + } + } + } + } + } + } + } + }`, { + owner: context.repo.owner, + repo: context.repo.repo, + prNumber, + }); + + const validPr = result + .repository + .pullRequest + .commits + .nodes[0] + .commit + .checkSuites + .nodes + .some(({ id }) => id === context.payload.workflow_run.check_suite_node_id); + + if (!validPr) { + console.log(`PR #${prNumber} is not valid (expected check suite ID ${context.payload.workflow_run.check_suite_node_id})`); + return; + } + + // Upload the screenshots + // 1. Loop over the bad-screenshots directory + // 2. Upload the screenshot + // 3. Save the URL + const uploadFile = async (filename) => { + if (!fs.existsSync(filename)) { + return; + } + + const formData = new FormData(); + + formData.set("key", FILE_HOUSE_KEY); + + formData.set("file", await fileFrom(filename), path.basename(filename)); + + return fetch("https://file.house/api/upload", { + method: "POST", + body: formData, + }) + .then(response => response.json()) + .then(response => { + console.log(response); + return response; + }) + .then(({ url }) => url); + }; + + const screenshotFailures = []; + + for (const directory of fs.readdirSync("bad-screenshots")) { + console.log(`Uploading screenshots for ${directory}`); + + let diffUrl; + let newUrl; + let oldUrl; + + await Promise.all([ + uploadFile(path.join("bad-screenshots", directory, "new.png")).then(url => newUrl = url), + uploadFile(path.join("bad-screenshots", directory, "old.png")).then(url => oldUrl = url), + uploadFile(path.join("bad-screenshots", directory, "diff.png")).then(url => diffUrl = url), + ]); + + console.log(`New URL (${directory}): ${newUrl}`); + console.log(`Old URL (${directory}): ${oldUrl}`); + console.log(`Diff URL (${directory}): ${diffUrl}`); + + screenshotFailures.push({ directory, diffUrl, newUrl, oldUrl }); + } + + if (screenshotFailures.length === 0) { + console.log("No screenshot failures found"); + return; + } + + // Upload zip file for quick fixes + const zipFilePath = path.join("data", "screenshot-update"); + const finalDestination = path.join( + zipFilePath, + "code", "modules", "unit_tests", "screenshots", + ) + + fs.mkdirSync(finalDestination, { recursive: true }); + + for (const { directory } of screenshotFailures) { + fs.copyFileSync( + path.join("bad-screenshots", directory, "new.png"), + path.join(finalDestination, `${directory}.png`), + ) + } + + await exec.exec("zip", ["-r", `../screenshot-update.zip`, "."], { + cwd: zipFilePath, + }); + + const zipUrl = await uploadFile(`${zipFilePath}.zip`); + + // Post the comment + const comment = createComment(screenshotFailures, zipUrl); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: comment, + }); +} diff --git a/tools/screenshot-test-comparison/index.js b/tools/screenshot-test-comparison/index.js new file mode 100644 index 00000000000..0a0f5fa4545 --- /dev/null +++ b/tools/screenshot-test-comparison/index.js @@ -0,0 +1,103 @@ +const fs = require("fs") +const path = require("path") +const pixelmatch = require("pixelmatch") +const process = require("process") +const PNG = require("pngjs").PNG + +const artifactsDirectory = process.argv[2] +if (!artifactsDirectory) { + console.error("Artifacts directory was not passed in") + process.exit(1) +} + +const screenshotsDirectory = process.argv[3] +if (!screenshotsDirectory) { + console.error("Screenshots directory was not passed in") + process.exit(1) +} + +const outputDirectory = process.argv[4] +if (!outputDirectory) { + console.error("Output directory was not passed in") + process.exit(1) +} + +const knownFailures = new Set() + +const fail = (screenshotName, newScreenshot, oldScreenshot, diff) => { + knownFailures.add(screenshotName) + + const outputPath = path.join(outputDirectory, path.parse(screenshotName).name) + fs.mkdirSync(outputPath, { + recursive: true + }) + + fs.copyFileSync(newScreenshot, path.join(outputPath, "new.png")) + + if (oldScreenshot) { + fs.copyFileSync(oldScreenshot, path.join(outputPath, "old.png")) + } + + if (diff) { + fs.writeFileSync(path.join(outputPath, "diff.png"), PNG.sync.write(diff)) + } +} + +for (const filename of fs.readdirSync(artifactsDirectory)) { + if (!filename.startsWith("test_artifacts")) { + continue + } + + const fullPath = path.join(artifactsDirectory, filename, "screenshots_new") + + const fullPathStat = fs.statSync(fullPath) + if (!fullPathStat.isDirectory()) { + continue + } + + for (const screenshotName of fs.readdirSync(fullPath)) { + if (knownFailures.has(screenshotName)) { + continue + } + + const fullPathScreenshotName = path.join(fullPath, screenshotName) + + const fullPathCompareScreenshot = path.join(screenshotsDirectory, screenshotName) + if (!fs.existsSync(fullPathCompareScreenshot)) { + fail(screenshotName, fullPathScreenshotName) + continue + } + + const screenshotNew = PNG.sync.read(fs.readFileSync(fullPathScreenshotName)) + const screenshotCompare = PNG.sync.read(fs.readFileSync(fullPathCompareScreenshot)) + + if ( + screenshotNew.width !== screenshotCompare.width + || screenshotNew.height !== screenshotCompare.height + ) { + console.error(`${screenshotName} has different dimensions from the known screenshot`) + fail(screenshotName, fullPathScreenshotName, fullPathCompareScreenshot) + continue + } + + const diff = new PNG({ width: screenshotNew.width, height: screenshotNew.height }) + const diffResult = pixelmatch( + screenshotNew.data, + screenshotCompare.data, + diff.data, + screenshotNew.width, + screenshotNew.height, + { threshold: 0.1 } + ) + + if (diffResult) { + console.error(`${screenshotName} differs from the known screenshot`) + fail(screenshotName, fullPathScreenshotName, fullPathCompareScreenshot, diff) + } + } +} + +if (knownFailures.size > 0) { + console.error(`${knownFailures.size} screenshots failed`) + process.exit(1) +} diff --git a/tools/screenshot-test-comparison/package-lock.json b/tools/screenshot-test-comparison/package-lock.json new file mode 100644 index 00000000000..4b24f319a79 --- /dev/null +++ b/tools/screenshot-test-comparison/package-lock.json @@ -0,0 +1,51 @@ +{ + "name": "screenshot-test-comparison", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "screenshot-test-comparison", + "version": "1.0.0", + "license": "AGPL-3.0-or-later", + "dependencies": { + "pixelmatch": "^5.3.0", + "pngjs": "^6.0.0" + } + }, + "node_modules/pixelmatch": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", + "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", + "dependencies": { + "pngjs": "^6.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "engines": { + "node": ">=12.13.0" + } + } + }, + "dependencies": { + "pixelmatch": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.3.0.tgz", + "integrity": "sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==", + "requires": { + "pngjs": "^6.0.0" + } + }, + "pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==" + } + } +} diff --git a/tools/screenshot-test-comparison/package.json b/tools/screenshot-test-comparison/package.json new file mode 100644 index 00000000000..8496bddb085 --- /dev/null +++ b/tools/screenshot-test-comparison/package.json @@ -0,0 +1,12 @@ +{ + "name": "screenshot-test-comparison", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "AGPL-3.0-or-later", + "dependencies": { + "pixelmatch": "^5.3.0", + "pngjs": "^6.0.0" + } +}