diff --git a/.github/workflows/ci_suite.yml b/.github/workflows/ci_suite.yml index 4f26fbbe432..d8313166a07 100644 --- a/.github/workflows/ci_suite.yml +++ b/.github/workflows/ci_suite.yml @@ -23,7 +23,7 @@ jobs: pip3 install setuptools bash tools/ci/install_build_tools.sh bash tools/ci/install_spaceman_dmm.sh dreamchecker - pip3 install -r tools/mapmerge2/requirements.txt + pip3 install -r tools/requirements.txt - name: Run Linters run: | bash tools/ci/check_filedirs.sh tgstation.dme @@ -32,7 +32,7 @@ jobs: find . -name "*.json" -not -path "*/node_modules/*" -print0 | xargs -0 python3 ./tools/json_verifier.py bash tools/ci/build_tgui.sh bash tools/ci/check_grep.sh - python3 tools/mapmerge2/dmi.py --test + tools/bootstrap/python -m dmi.test ~/dreamchecker > ${GITHUB_WORKSPACE}/output-annotations.txt 2>&1 - name: Annotate Lints uses: yogstation13/DreamAnnotate@v1 diff --git a/dependencies.sh b/dependencies.sh index e8709d10b1b..78192c3bc3d 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -1,13 +1,13 @@ -#!/bin/bash +#!/bin/sh #Project dependencies file #Final authority on what's required to fully build the project # byond version # Extracted from the Dockerfile. Change by editing Dockerfile's FROM command. -LIST=($(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile)) -export BYOND_MAJOR=${LIST[0]} -export BYOND_MINOR=${LIST[1]} +LIST="$(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile)" +export BYOND_MAJOR=${LIST% *} +export BYOND_MINOR=${LIST#* } unset LIST #rust_g git tag @@ -21,3 +21,6 @@ export SPACEMAN_DMM_VERSION=suite-1.6 # Extools git tag export EXTOOLS_VERSION=v0.0.6 + +# Python version for mapmerge and other tools +export PYTHON_VERSION=3.6.8 diff --git a/tools/HitboxExpander/.gitignore b/tools/HitboxExpander/.gitignore deleted file mode 100644 index ec7f5fd7c18..00000000000 --- a/tools/HitboxExpander/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -Imaging-1.1.7/ -zlib/ - diff --git a/tools/HitboxExpander/Hitbox Expander.bat b/tools/HitboxExpander/Hitbox Expander.bat new file mode 100644 index 00000000000..ac69e5505db --- /dev/null +++ b/tools/HitboxExpander/Hitbox Expander.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m HitboxExpander %* +@pause diff --git a/tools/HitboxExpander/README b/tools/HitboxExpander/README.txt similarity index 55% rename from tools/HitboxExpander/README rename to tools/HitboxExpander/README.txt index 1e1ef5f153c..67a5b8bf42d 100755 --- a/tools/HitboxExpander/README +++ b/tools/HitboxExpander/README.txt @@ -1,10 +1,4 @@ -Setup: Install python3 and run install.bat for windows, install.sh for unix. -Alternatively, you can manually install the Pillow package with -``` -pip install Pillow -``` - -Usage: python hitbox_expander.py +Usage: tools/bootstrap/python -m HitboxExpander This tool expands the hitbox of the given image by 1 pixel. Works by changing some of the fully-transparent pixels to alpha=1 black pixels. diff --git a/tools/HitboxExpander/hitbox_expander.py b/tools/HitboxExpander/__main__.py similarity index 92% rename from tools/HitboxExpander/hitbox_expander.py rename to tools/HitboxExpander/__main__.py index 36fb8085375..c06352440c7 100755 --- a/tools/HitboxExpander/hitbox_expander.py +++ b/tools/HitboxExpander/__main__.py @@ -74,7 +74,10 @@ icons_dir = os.path.join(root_dir, "icons") def Main(): if len(sys.argv) != 2: - print("Usage: hitbox_expander.py filename.dmi") + if os.name == 'nt': + print("Usage: drag-and-drop a .dmi onto `Hitbox Expander.bat`\n or") + with open(os.path.join(current_dir, "README.txt")) as f: + print(f.read()) return 0 try: diff --git a/tools/HitboxExpander/hitbox_expander.sh b/tools/HitboxExpander/hitbox_expander.sh new file mode 100755 index 00000000000..aca0ebf9e35 --- /dev/null +++ b/tools/HitboxExpander/hitbox_expander.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +exec "$(dirname "$0")/../bootstrap/python" -m HitboxExpander "$@" diff --git a/tools/HitboxExpander/install.bat b/tools/HitboxExpander/install.bat deleted file mode 100644 index 78c80cd7d30..00000000000 --- a/tools/HitboxExpander/install.bat +++ /dev/null @@ -1 +0,0 @@ -python3 -m pip install -r requirements.txt diff --git a/tools/HitboxExpander/install.sh b/tools/HitboxExpander/install.sh deleted file mode 100644 index 78c80cd7d30..00000000000 --- a/tools/HitboxExpander/install.sh +++ /dev/null @@ -1 +0,0 @@ -python3 -m pip install -r requirements.txt diff --git a/tools/HitboxExpander/requirements.txt b/tools/HitboxExpander/requirements.txt deleted file mode 100644 index 334e961ca1a..00000000000 --- a/tools/HitboxExpander/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow==7.2.0 diff --git a/tools/UpdatePaths/Update Paths.bat b/tools/UpdatePaths/Update Paths.bat new file mode 100644 index 00000000000..c126388de7e --- /dev/null +++ b/tools/UpdatePaths/Update Paths.bat @@ -0,0 +1,3 @@ +@echo off +call "%~dp0\..\bootstrap\python" -m UpdatePaths %* +pause diff --git a/tools/mapmerge2/update_paths.py b/tools/UpdatePaths/__main__.py similarity index 95% rename from tools/mapmerge2/update_paths.py rename to tools/UpdatePaths/__main__.py index 2c316e941de..ba2da1ea049 100644 --- a/tools/mapmerge2/update_paths.py +++ b/tools/UpdatePaths/__main__.py @@ -1,9 +1,10 @@ # A script and syntax for applying path updates to maps. import re import os +import sys import argparse -import frontend -from dmm import * +from mapmerge2 import frontend +from mapmerge2.dmm import * desc = """ Update dmm files given update file/string. @@ -167,7 +168,11 @@ def main(args): if __name__ == "__main__": - parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) + prog = __spec__.name.replace('.__main__', '') + if os.name == 'nt' and len(sys.argv) <= 1: + print("usage: drag-and-drop a path script .txt onto `Update Paths.bat`\n or") + + parser = argparse.ArgumentParser(prog=prog, description=desc, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("update_source", help="update file path / line of update notation") parser.add_argument("--map", "-m", help="path to update, defaults to all maps in maps directory") parser.add_argument("--directory", "-d", help="path to maps directory, defaults to _maps/") diff --git a/tools/mapmerge2/map_scripts/apc_pixel_offsets.txt b/tools/UpdatePaths/apc_pixel_offsets.txt similarity index 100% rename from tools/mapmerge2/map_scripts/apc_pixel_offsets.txt rename to tools/UpdatePaths/apc_pixel_offsets.txt diff --git a/tools/mapmerge2/map_scripts/clothingunderrepath.txt b/tools/UpdatePaths/clothingunderrepath.txt similarity index 100% rename from tools/mapmerge2/map_scripts/clothingunderrepath.txt rename to tools/UpdatePaths/clothingunderrepath.txt diff --git a/tools/mapmerge2/map_scripts/cornersfix.txt b/tools/UpdatePaths/cornersfix.txt similarity index 100% rename from tools/mapmerge2/map_scripts/cornersfix.txt rename to tools/UpdatePaths/cornersfix.txt diff --git a/tools/mapmerge2/map_scripts/turfs2decals.txt b/tools/UpdatePaths/turfs2decals.txt similarity index 100% rename from tools/mapmerge2/map_scripts/turfs2decals.txt rename to tools/UpdatePaths/turfs2decals.txt diff --git a/tools/bootstrap/python b/tools/bootstrap/python new file mode 100755 index 00000000000..f9ffc13edb6 --- /dev/null +++ b/tools/bootstrap/python @@ -0,0 +1,117 @@ +#!/bin/sh +# bootstrap/python +# +# Python-finding script for all `sh` environments, including Linux, MSYS2, +# Git for Windows, and GitHub Desktop. Invokable from CLI or automation. +# +# If a python.exe installed by `python_.ps1` is present, it will be used. +# Otherwise, this script requires a system `python3` and `pip` to be provided, +# and will create a standard virtualenv in which to install `requirements.txt`. +set -e + +# Convenience variables +Bootstrap="$(dirname "$0")" +Sdk="$(dirname "$Bootstrap")" +Cache="$Bootstrap/.cache" +if [ "$TG_BOOTSTRAP_CACHE" ]; then + Cache="$TG_BOOTSTRAP_CACHE" +fi +OldPWD="$PWD" +cd "$Bootstrap/../.." +. ./dependencies.sh # sets PYTHON_VERSION +cd "$OldPWD" +PythonVersion="$PYTHON_VERSION" +PythonDir="$Cache/python-$PythonVersion" +PythonExe="$PythonDir/python.exe" +Log="$Cache/last-command.log" + +# If a portable Python for Windows is not present, search on $PATH. +if [ "$(uname)" = "Linux" ] || [ ! -f "$PythonExe" ]; then + # Strip the "App Execution Aliases" from $PATH. Even if the user installed + # Python using the Windows Store on purpose, these aliases always generate + # "Permission denied" errors when sh.exe tries to invoke them. + PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":") + + # Try to find a Python executable. + if command -v python3 >/dev/null 2>&1; then + PythonExe=python3 + elif command -v python >/dev/null 2>&1; then + PythonExe=python + elif command -v py >/dev/null 2>&1; then + PythonExe="py -3" + else + echo + if command -v apt-get >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo apt-get install python3 python3-pip" + elif [ "$(uname -o)" = "Msys" ]; then + echo "Please run tools/bootstrap/python.bat instead of tools/bootstrap/python once to" + echo "install Python automatically, or install it from https://www.python.org/downloads/" + # TODO: give MSYS pacman advice? + elif command -v pacman >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo pacman -S python python-pip" + else + echo "Please install Python from https://www.python.org/downloads/ or using your system's package manager." + fi + echo + exit 1 + fi + + # Create a venv and activate it + PythonDir="$Cache/venv" + if [ ! -d "$PythonDir" ]; then + echo "Creating virtualenv..." + "$PythonExe" -m venv "$PythonDir" + fi + if [ -f "$PythonDir/bin/python" ]; then + PythonExe="$PythonDir/bin/python" + elif [ -f "$PythonDir/scripts/python3.exe" ]; then + PythonExe="$PythonDir/scripts/python3.exe"; + else + echo "bootstrap/python failed to find the python executable inside its virtualenv" + exit 1 + fi +fi + +# Use pip to install our requirements +if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then + echo "Updating dependencies..." + "$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt" + cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt" + echo "---" +fi + +# Verify version and deduce the path separator +PythonMajor=${PythonVersion%%.*} +PythonMinor=${PythonVersion#*.} +PythonMinor=${PythonMinor%.*} +PATHSEP=$("$PythonExe" - "$PythonMajor" "$PythonMinor" <<'EOF' +import sys, os +if sys.version_info.major != int(sys.argv[1]) or sys.version_info.minor < int(sys.argv[2]): + print("Error: Python ", sys.argv[1], ".", sys.argv[2], " or later is required, but you have:\n", sys.version, sep="", file=sys.stderr) + exit(1) +print(os.pathsep) +EOF +) + +# Cheap shell function if tee.exe is not available +if ! command -v tee >/dev/null 2>&1; then + tee() { + # Fudge: assume $1 is always "-a" + while read -r line; do + echo "$line" >> "$2" + echo "$line" + done + } +fi + +# Invoke python with all command-line arguments +export PYTHONPATH="$Sdk$PATHSEP${PYTHONPATH:-}" +mkdir -p "$Cache" +printf '%s\n' "$PythonExe" "$@" > "$Log" +printf -- '---\n' >> "$Log" +exec 4>&1 +exitstatus=$({ { set +e; "$PythonExe" -u "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1) +exec 4>&- +exit "$exitstatus" diff --git a/tools/bootstrap/python.bat b/tools/bootstrap/python.bat new file mode 100644 index 00000000000..3b20c2d436b --- /dev/null +++ b/tools/bootstrap/python.bat @@ -0,0 +1 @@ +@call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\python_.ps1" %* diff --git a/tools/bootstrap/python36._pth b/tools/bootstrap/python36._pth new file mode 100644 index 00000000000..2bd2e8a7d6e --- /dev/null +++ b/tools/bootstrap/python36._pth @@ -0,0 +1,6 @@ +python36.zip +. +..\..\.. + +# Uncomment to run site.main() automatically +import site diff --git a/tools/bootstrap/python_.ps1 b/tools/bootstrap/python_.ps1 new file mode 100644 index 00000000000..c9e39156137 --- /dev/null +++ b/tools/bootstrap/python_.ps1 @@ -0,0 +1,103 @@ +# bootstrap/python_.ps1 +# +# Python bootstrapping script for Windows. +# +# Automatically downloads a portable edition of a pinned Python version to +# a cache directory, installs Pip, installs `requirements.txt`, and then invokes +# Python. +# +# The underscore in the name is so that typing `bootstrap/python` into +# PowerShell finds the `.bat` file first, which ensures this script executes +# regardless of ExecutionPolicy. +$host.ui.RawUI.WindowTitle = "starting :: python $args" +$ErrorActionPreference = "Stop" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function ExtractVersion { + param([string] $Path, [string] $Key) + foreach ($Line in Get-Content $Path) { + if ($Line.StartsWith("export $Key=")) { + return $Line.Substring("export $Key=".Length) + } + } + throw "Couldn't find value for $Key in $Path" +} + +# Convenience variables +$Bootstrap = Split-Path $script:MyInvocation.MyCommand.Path +$Tools = Split-Path $Bootstrap +$Cache = "$Bootstrap/.cache" +if ($Env:TG_BOOTSTRAP_CACHE) { + $Cache = $Env:TG_BOOTSTRAP_CACHE +} +$PythonVersion = ExtractVersion -Path "$Bootstrap/../../dependencies.sh" -Key "PYTHON_VERSION" +$PythonDir = "$Cache/python-$PythonVersion" +$PythonExe = "$PythonDir/python.exe" +$Log = "$Cache/last-command.log" + +# Download and unzip a portable version of Python +if (!(Test-Path $PythonExe -PathType Leaf)) { + $host.ui.RawUI.WindowTitle = "Downloading Python $PythonVersion..." + New-Item $Cache -ItemType Directory -ErrorAction silentlyContinue | Out-Null + + $Archive = "$Cache/python-$PythonVersion-embed.zip" + Invoke-WebRequest ` + "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" ` + -OutFile $Archive ` + -ErrorAction Stop + + [System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir) + + # Copy a ._pth file without "import site" commented, so pip will work + Copy-Item "$Bootstrap/python36._pth" $PythonDir ` + -ErrorAction Stop + + Remove-Item $Archive +} + +# Install pip +if (!(Test-Path "$PythonDir/Scripts/pip.exe")) { + $host.ui.RawUI.WindowTitle = "Downloading Pip..." + + Invoke-WebRequest "https://bootstrap.pypa.io/get-pip.py" ` + -OutFile "$Cache/get-pip.py" ` + -ErrorAction Stop + + & $PythonExe "$Cache/get-pip.py" --no-warn-script-location + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Remove-Item "$Cache/get-pip.py" ` + -ErrorAction Stop +} + +# Use pip to install our requirements +if (!(Test-Path "$PythonDir/requirements.txt") -or ((Get-FileHash "$Tools/requirements.txt").hash -ne (Get-FileHash "$PythonDir/requirements.txt").hash)) { + $host.ui.RawUI.WindowTitle = "Updating dependencies..." + + & $PythonExe -m pip install -U pip -r "$Tools/requirements.txt" + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Copy-Item "$Tools/requirements.txt" "$PythonDir/requirements.txt" + Write-Output "`n---`n" +} + +# Invoke python with all command-line arguments +Write-Output $PythonExe | Out-File -Encoding utf8 $Log +[System.String]::Join([System.Environment]::NewLine, $args) | Out-File -Encoding utf8 -Append $Log +Write-Output "---" | Out-File -Encoding utf8 -Append $Log +$host.ui.RawUI.WindowTitle = "python $args" +$ErrorActionPreference = "Continue" +& $PythonExe -u $args 2>&1 | ForEach-Object { + $str = "$_" + if ($_.GetType() -eq [System.Management.Automation.ErrorRecord]) { + $str = $str.TrimEnd("`r`n") + } + $str | Out-File -Encoding utf8 -Append $Log + $str | Out-Host +} +exit $LastExitCode diff --git a/tools/dmi/Resolve Icon Conflicts.bat b/tools/dmi/Resolve Icon Conflicts.bat new file mode 100644 index 00000000000..8def88442c6 --- /dev/null +++ b/tools/dmi/Resolve Icon Conflicts.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python.bat" -m dmi.merge_driver --posthoc %* +@pause diff --git a/tools/mapmerge2/dmi.py b/tools/dmi/__init__.py similarity index 87% rename from tools/mapmerge2/dmi.py rename to tools/dmi/__init__.py index e87cf507b99..cdf28463bcd 100644 --- a/tools/mapmerge2/dmi.py +++ b/tools/dmi/__init__.py @@ -12,10 +12,10 @@ NORTH = 1 SOUTH = 2 EAST = 4 WEST = 8 -SOUTHEAST = SOUTH|EAST -SOUTHWEST = SOUTH|WEST -NORTHEAST = NORTH|EAST -NORTHWEST = NORTH|WEST +SOUTHEAST = SOUTH | EAST +SOUTHWEST = SOUTH | WEST +NORTHEAST = NORTH | EAST +NORTHWEST = NORTH | WEST CARDINALS = [NORTH, SOUTH, EAST, WEST] DIR_ORDER = [SOUTH, NORTH, EAST, WEST, SOUTHEAST, SOUTHWEST, NORTHEAST, NORTHWEST] @@ -34,6 +34,7 @@ DIR_NAMES = { None: SOUTH, } + class Dmi: version = "4.0" @@ -134,7 +135,7 @@ class Dmi: comment += f"state = {escape(state.name)}\n" comment += f"\tdirs = {state.dirs}\n" comment += f"\tframes = {state.framecount}\n" - if state.framecount > 1 and len(state.delays): #any(x != 1 for x in state.delays): + if state.framecount > 1 and len(state.delays): # any(x != 1 for x in state.delays): comment += "\tdelay = " + ",".join(map(str, state.delays)) + "\n" if state.loop != 0: comment += f"\tloop = {state.loop}\n" @@ -175,6 +176,7 @@ class Dmi: output = output.convert('P') output.save(filename, 'png', optimize=True, pnginfo=pnginfo) + class State: def __init__(self, dmi, name, *, loop=LOOP_UNLIMITED, rewind=False, movement=False, dirs=1): self.dmi = dmi @@ -216,11 +218,13 @@ class State: def get_frame(self, *args, **kwargs): return self.frames[self._frame_index(*args, **kwargs)] + def escape(text): text = text.replace('\\', '\\\\') text = text.replace('"', '\\"') return f'"{text}"' + def unescape(text, quote='"'): if text == 'null': return None @@ -231,51 +235,14 @@ def unescape(text, quote='"'): text = text.replace('\\\\', '\\') return text + def parse_num(value): if '.' in value: return float(value) return int(value) + def parse_bool(value): if value not in ('0', '1'): raise ValueError(value) return value == '1' - -def _self_test(): - # test: can we load every DMI in the tree - import os - - count = 0 - for dirpath, dirnames, filenames in os.walk('.'): - if '.git' in dirnames: - dirnames.remove('.git') - for filename in filenames: - if filename.endswith('.dmi'): - fullpath = os.path.join(dirpath, filename) - try: - Dmi.from_file(fullpath) - except: - print('Failed on:', fullpath) - raise - count += 1 - - print(f"Successfully parsed {count} dmi files") - -def _usage(): - import sys - print(f"Usage:") - print(f" {sys.argv[0]} --test") - exit(1) - -def _main(): - import sys - if len(sys.argv) < 2: - return _usage() - - if sys.argv[1] == '--test': - return _self_test() - - return _usage() - -if __name__ == '__main__': - _main() diff --git a/tools/mapmerge2/merge_driver_dmi.py b/tools/dmi/merge_driver.py similarity index 81% rename from tools/mapmerge2/merge_driver_dmi.py rename to tools/dmi/merge_driver.py index 37c0abd9543..75c3daacb07 100644 --- a/tools/mapmerge2/merge_driver_dmi.py +++ b/tools/dmi/merge_driver.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import sys import dmi +from hooks.merge_frontend import MergeDriver + def images_equal(left, right): if left.size != right.size: @@ -15,6 +17,7 @@ def images_equal(left, right): return False return True + def states_equal(left, right): result = True @@ -31,9 +34,11 @@ def states_equal(left, right): return result + def key_of(state): return (state.name, state.movement) + def dictify(sheet): result = {} for state in sheet.states: @@ -43,6 +48,7 @@ def dictify(sheet): result[k] = state return result + def three_way_merge(base, left, right): base_dims = base.width, base.height if base_dims != (left.width, left.height) or base_dims != (right.width, right.height): @@ -145,33 +151,31 @@ def three_way_merge(base, left, right): merged.states = final_states return len(conflicts), merged -def main(path, original, left, right): - print(f"Merging icon: {path}") - icon_orig = dmi.Dmi.from_file(original) - icon_left = dmi.Dmi.from_file(left) - icon_right = dmi.Dmi.from_file(right) +class DmiDriver(MergeDriver): + driver_id = 'dmi' + + def merge(self, base, left, right): + icon_base = dmi.Dmi.from_file(base) + icon_left = dmi.Dmi.from_file(left) + icon_right = dmi.Dmi.from_file(right) + trouble, merge_result = three_way_merge(icon_base, icon_left, icon_right) + return not trouble, merge_result + + def to_file(self, outfile, merge_result): + merge_result.to_file(outfile) + + def post_announce(self, success, merge_result): + if not success: + print("!!! Manual merge required!") + if merge_result: + print(" A best-effort merge was performed. You must edit the icon and remove all") + print(" icon states marked with !CONFLICT!, leaving only the desired icon.") + else: + print(" The icon was totally unable to be merged, you must start with one version") + print(" or the other and manually resolve the conflict.") + print(" Information about which states conflicted is listed above.") - trouble, merged = three_way_merge(icon_orig, icon_left, icon_right) - if merged: - merged.to_file(left) - if trouble: - print("!!! Manual merge required!") - if merged: - print(" A best-effort merge was performed. You must edit the icon and remove all") - print(" icon states marked with !CONFLICT!, leaving only the desired icon.") - else: - print(" The icon was totally unable to be merged, you must start with one version") - print(" or the other and manually resolve the conflict.") - print(" Information about which states conflicted is listed above.") - return trouble if __name__ == '__main__': - if len(sys.argv) != 6: - print("DMI merge driver called with wrong number of arguments") - print(" usage: merge-driver-dmi %P %O %A %B %L") - exit(1) - - # "left" is also the file that ought to be overwritten - _, path, original, left, right, conflict_size_marker = sys.argv - exit(main(path, original, left, right)) + exit(DmiDriver().main()) diff --git a/tools/dmi/test.py b/tools/dmi/test.py new file mode 100644 index 00000000000..057e47c0a36 --- /dev/null +++ b/tools/dmi/test.py @@ -0,0 +1,39 @@ +import os +import sys +from dmi import * + + +def _self_test(): + # test: can we load every DMI in the tree + count = 0 + for dirpath, dirnames, filenames in os.walk('.'): + if '.git' in dirnames: + dirnames.remove('.git') + for filename in filenames: + if filename.endswith('.dmi'): + fullpath = os.path.join(dirpath, filename) + try: + Dmi.from_file(fullpath) + except Exception: + print('Failed on:', fullpath) + raise + count += 1 + + print(f"Successfully parsed {count} dmi files") + + +def _usage(): + print(f"Usage:") + print(f" tools{os.sep}bootstrap{os.sep}python -m {__spec__.name}") + exit(1) + + +def _main(): + if len(sys.argv) == 1: + return _self_test() + + return _usage() + + +if __name__ == '__main__': + _main() diff --git a/tools/hooks/Install.bat b/tools/hooks/Install.bat new file mode 100644 index 00000000000..1bfc1b50d5c --- /dev/null +++ b/tools/hooks/Install.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m hooks.install %* +@pause diff --git a/tools/hooks/README.md b/tools/hooks/README.md index b15fb493d8d..058ef04d61d 100644 --- a/tools/hooks/README.md +++ b/tools/hooks/README.md @@ -5,7 +5,7 @@ Use of these hooks and drivers is optional and they must be installed explicitly before they take effect. To install the current set of hooks, or update if new hooks are added, run -`install.bat` (Windows) or `install.sh` (Unix-like) as appropriate. +`Install.bat` (Windows) or `tools/hooks/install` (Unix-like) as appropriate. Hooks expect a Unix-like environment on the backend. Usually this is handled automatically by GUI tools like TortoiseGit and GitHub for Windows, but diff --git a/tools/hooks/Uninstall.bat b/tools/hooks/Uninstall.bat new file mode 100644 index 00000000000..862cbaf8bc4 --- /dev/null +++ b/tools/hooks/Uninstall.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m hooks.install --uninstall %* +@pause diff --git a/tools/hooks/dmi.merge b/tools/hooks/dmi.merge index 7fd9f171bf0..d82ac06be86 100755 --- a/tools/hooks/dmi.merge +++ b/tools/hooks/dmi.merge @@ -1,2 +1,2 @@ #!/bin/sh -exec tools/hooks/python.sh -m merge_driver_dmi "$@" +exec tools/bootstrap/python -m dmi.merge_driver "$@" diff --git a/tools/hooks/install.bat b/tools/hooks/install.bat deleted file mode 100644 index 7a11129a2a2..00000000000 --- a/tools/hooks/install.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -cd %~dp0 -for %%f in (*.hook) do ( - echo Installing hook: %%~nf - copy %%f ..\..\.git\hooks\%%~nf >nul -) -for %%f in (*.merge) do ( - echo Installing merge driver: %%~nf - echo [merge "%%~nf"]^ - - driver = tools/hooks/%%f %%P %%O %%A %%B %%L >> ..\..\.git\config -) -echo Installing Python dependencies -python -m pip install -r ..\mapmerge2\requirements.txt -echo Done -pause diff --git a/tools/hooks/install.py b/tools/hooks/install.py new file mode 100644 index 00000000000..0736b29681e --- /dev/null +++ b/tools/hooks/install.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# hooks/install.py +# +# This script is configured by adding `*.hook` and `*.merge` files in the same +# directory. Such files should be `#!/bin/sh` scripts, usually invoking Python. +# This installer will have to be re-run any time a hook or merge file is added +# or removed, but not when they are changed. +# +# Merge drivers will also need a corresponding entry in the `.gitattributes` +# file. + +import os +import stat +import glob +import re +import pygit2 +import shlex + + +def write_hook(fname, command): + with open(fname, 'w', encoding='utf-8', newline='\n') as f: + print("#!/bin/sh", file=f) + print("exec", command, file=f) + + # chmod +x + st = os.stat(fname) + if not hasattr(st, 'st_file_attributes'): + os.chmod(fname, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def _find_stuff(target=None): + repo_dir = pygit2.discover_repository(target or os.getcwd()) + repo = pygit2.Repository(repo_dir) + # Strips any active worktree to find the hooks directory. + root_repo_dir = re.sub(r'/.git/worktrees/[^/]+/', '/.git/', repo_dir) + hooks_dir = os.path.join(root_repo_dir, 'hooks') + return repo, hooks_dir + + +def uninstall(target=None, keep=()): + repo, hooks_dir = _find_stuff(target) + + # Remove hooks + for fname in glob.glob(os.path.join(hooks_dir, '*')): + _, shortname = os.path.split(fname) + if not fname.endswith('.sample') and f"{shortname}.hook" not in keep: + print('Removing hook:', shortname) + os.unlink(fname) + + # Remove merge driver configuration + for entry in repo.config: + match = re.match(r'^merge\.([^.]+)\.driver$', entry.name) + if match and f"{match.group(1)}.merge" not in keep: + print('Removing merge driver:', match.group(1)) + del repo.config[entry.name] + + +def install(target=None): + repo, hooks_dir = _find_stuff(target) + tools_hooks = os.path.split(__file__)[0] + + keep = set() + for full_path in glob.glob(os.path.join(tools_hooks, '*.hook')): + _, fname = os.path.split(full_path) + name, _ = os.path.splitext(fname) + print('Installing hook:', name) + keep.add(fname) + relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/')) + write_hook(os.path.join(hooks_dir, name), f'{relative_path} "$@"') + + # Use libgit2 config manipulation to set the merge driver config. + for full_path in glob.glob(os.path.join(tools_hooks, '*.merge')): + # Merge drivers are documented here: https://git-scm.com/docs/gitattributes + _, fname = os.path.split(full_path) + name, _ = os.path.splitext(fname) + print('Installing merge driver:', name) + keep.add(fname) + # %P: "real" path of the file, should not usually be read or modified + # %O: ancestor's version + # %A: current version, and also the output path + # %B: other branches' version + # %L: conflict marker size + relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/')) + repo.config[f"merge.{name}.driver"] = f'{relative_path} %P %O %A %B %L' + + uninstall(target, keep=keep) + + +def main(argv): + if len(argv) <= 1: + return install() + elif argv[1] == '--uninstall': + return uninstall() + else: + print("Usage: python -m hooks.install [--uninstall]") + return 1 + + +if __name__ == '__main__': + import sys + exit(main(sys.argv)) diff --git a/tools/hooks/install.sh b/tools/hooks/install.sh index 899b85b941a..d0e995ca097 100755 --- a/tools/hooks/install.sh +++ b/tools/hooks/install.sh @@ -1,20 +1,2 @@ -#!/bin/bash -set -e -shopt -s nullglob -cd "$(dirname "$0")" -for f in *.hook; do - echo Installing hook: ${f%.hook} - cp $f ../../.git/hooks/${f%.hook} -done -for f in *.merge; do - echo Installing merge driver: ${f%.merge} - git config --replace-all merge.${f%.merge}.driver "tools/hooks/$f %P %O %A %B %L" -done - -echo "Installing tgui hooks" -../../tgui/bin/tgui --install-git-hooks - -echo "Installing Python dependencies" -./python.sh -m pip install -r ../mapmerge2/requirements.txt - -echo "Done" +#!/bin/sh +exec "$(dirname "$0")/../bootstrap/python" -m hooks.install "$@" diff --git a/tools/hooks/merge_frontend.py b/tools/hooks/merge_frontend.py new file mode 100644 index 00000000000..b735d1e3a5c --- /dev/null +++ b/tools/hooks/merge_frontend.py @@ -0,0 +1,169 @@ +# merge_frontend.py +import sys +import io +import os +import pygit2 +import collections +import typing + + +ENCODING = 'utf-8' + + +class MergeReturn(typing.NamedTuple): + success: bool + merge_result: typing.Optional[object] + + +class MergeDriver: + driver_id: typing.Optional[str] = None + + def pre_announce(self, path: str): + """ + Called before merge() is called, with a human-friendly path for output. + """ + print(f"Merging {self.driver_id}: {path}") + + def merge(self, base: typing.BinaryIO, left: typing.BinaryIO, right: typing.BinaryIO) -> MergeReturn: + """ + Read from three BinaryIOs: base (common ancestor), left (ours), and + right (theirs). Perform the actual three-way merge operation. Leave + conflict markers if necessary. + + Return (False, None) to indicate the merge driver totally failed. + Return (False, merge_result) if the result contains conflict markers. + Return (True, merge_result) if everything went smoothly. + """ + raise NotImplementedError + + def to_file(self, output: typing.BinaryIO, merge_result: object): + """ + Save the merge() result to the given output stream. + Override this if the merge() result is not bytes or str. + """ + if isinstance(merge_result, bytes): + output.write(merge_result) + elif isinstance(merge_result, str): + with io.TextIOWrapper(output, ENCODING) as f: + f.write(merge_result) + else: + raise NotImplementedError + + def post_announce(self, success: bool, merge_result: object): + """ + Called after merge() is called, to warn the user if action is needed. + """ + if not success: + print("!!! Manual merge required") + if merge_result: + print(" A best-effort merge was performed. You must finish the job yourself.") + else: + print(" No merge was possible. You must resolve the conflict yourself.") + + def main(self, args: typing.List[str] = None): + return _main(self, args or sys.argv[1:]) + + +def _main(driver: MergeDriver, args: typing.List[str]): + if len(args) > 0 and args[0] == '--posthoc': + return _posthoc_main(driver, args[1:]) + else: + return _driver_main(driver, args) + + +def _driver_main(driver: MergeDriver, args: typing.List[str]): + """ + Act like a normal Git merge driver, called by Git during a merge. + """ + if len(args) != 5: + print("merge driver called with wrong number of arguments") + print(" usage: %P %O %A %B %L") + return 1 + + path, path_base, path_left, path_right, _ = args + driver.pre_announce(path) + + with open(path_base, 'rb') as io_base: + with open(path_left, 'rb') as io_left: + with open(path_right, 'rb') as io_right: + success, merge_result = driver.merge(io_base, io_left, io_right) + + if merge_result: + # If we got anything, write it to the working directory. + with open(path_left, 'wb') as io_output: + driver.to_file(io_output, merge_result) + + driver.post_announce(success, merge_result) + if not success: + # If we were not successful, do not mark the conflict as resolved. + return 1 + + +def _posthoc_main(driver: MergeDriver, args: typing.List[str]): + """ + Apply merge driver logic to a repository which is already in a conflicted + state, running the driver on any conflicted files. + """ + repo_dir = pygit2.discover_repository(os.getcwd()) + repo = pygit2.Repository(repo_dir) + conflicts = repo.index.conflicts + if not conflicts: + print("There are no unresolved conflicts.") + return 0 + + all_success = True + index_changed = False + any_attempted = False + for base, left, right in list(conflicts): + if not base or not left or not right: + # (not left) or (not right): deleted in one branch, modified in the other. + # (not base): added differently in both branches. + # In either case, there's nothing we can do for now. + continue + + path = left.path + if not _applies_to(repo, driver, path): + # Skip the file if it's not the right extension. + continue + + any_attempted = True + driver.pre_announce(path) + io_base = io.BytesIO(repo[base.id].data) + io_left = io.BytesIO(repo[left.id].data) + io_right = io.BytesIO(repo[right.id].data) + success, merge_result = driver.merge(io_base, io_left, io_right) + if merge_result: + # If we got anything, write it to the working directory. + with open(os.path.join(repo.workdir, path), 'wb') as io_output: + driver.to_file(io_output, merge_result) + + if success: + # If we were successful, mark the conflict as resolved. + with open(os.path.join(repo.workdir, path), 'rb') as io_readback: + contents = io_readback.read() + merged_id = repo.create_blob(contents) + repo.index.add(pygit2.IndexEntry(path, merged_id, left.mode)) + del conflicts[path] + index_changed = True + if not success: + all_success = False + driver.post_announce(success, merge_result) + + if index_changed: + repo.index.write() + + if not any_attempted: + print("There are no unresolved", driver.driver_id, "conflicts.") + + if not all_success: + # Not usually observed, but indicate the failure just in case. + return 1 + + +def _applies_to(repo: pygit2.Repository, driver: MergeDriver, path: str): + """ + Check if the current merge driver is a candidate to handle a given path. + """ + if not driver.driver_id: + raise ValueError('Driver must have ID to perform post-hoc merge') + return repo.get_attr(path, 'merge') == driver.driver_id diff --git a/tools/hooks/pre-commit.hook b/tools/hooks/pre-commit.hook index 970be47a44b..8ad5ba32a45 100755 --- a/tools/hooks/pre-commit.hook +++ b/tools/hooks/pre-commit.hook @@ -1,3 +1,2 @@ #!/bin/sh -# `sh` must be used here instead of `bash` to support GitHub Desktop. -exec tools/hooks/python.sh -m precommit +exec tools/bootstrap/python -m mapmerge2.precommit diff --git a/tools/hooks/python.sh b/tools/hooks/python.sh index fc1804d8bc3..60a5efd700d 100755 --- a/tools/hooks/python.sh +++ b/tools/hooks/python.sh @@ -1,32 +1,17 @@ #!/bin/sh -# `sh` must be used here instead of `bash` to support GitHub Desktop. set -e - -# Strip the "App Execution Aliases" from $PATH. Even if the user installed -# Python using the Windows Store on purpose, these aliases always generate -# "Permission denied" errors when sh.exe tries to invoke them. -PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":") - -# Try to find a Python executable. -if command -v python3 >/dev/null 2>&1; then - PY=python3 -elif command -v python >/dev/null 2>&1; then - PY=python -elif command -v py >/dev/null 2>&1; then - PY="py -3" +if [ "$*" = "-m precommit" ]; then + echo "Hooks are being updated..." + echo "Details: https://github.com/tgstation/tgstation/pull/55658" + if [ "$(uname -o)" = "Msys" ]; then + tools/hooks/Install.bat + else + tools/hooks/install.sh + fi + echo "---------------" + exec tools/hooks/pre-commit.hook else - echo "Please install Python from https://www.python.org/downloads/" + echo "tools/hooks/python.sh is replaced by tools/bootstrap/python" + echo "Details: https://github.com/tgstation/tgstation/pull/55658" exit 1 fi - -# Deduce the path separator and add the mapmerge package to the search path. -PATHSEP=$($PY - <<'EOF' -import sys, os -if sys.version_info.major != 3 or sys.version_info.minor < 6: - sys.stderr.write("Python 3.6 or later is required, but you have:\n" + sys.version + "\n") - exit(1) -print(os.pathsep) -EOF -) -export PYTHONPATH=tools/mapmerge2/${PATHSEP}${PYTHONPATH} -exec $PY "$@" diff --git a/tools/makeChangelog.bat b/tools/makeChangelog.bat index f0646caeca0..acdd638b0e2 100644 --- a/tools/makeChangelog.bat +++ b/tools/makeChangelog.bat @@ -1,4 +1,4 @@ @echo off rem Cheridan asked for this. - N3X -call python ss13_genchangelog.py ../html/changelog.html ../html/changelogs -pause \ No newline at end of file +call "%~dp0\bootstrap\python" ss13_genchangelog.py ../html/changelog.html ../html/changelogs +pause diff --git a/tools/mapmerge2/README.md b/tools/mapmerge2/README.md index 0ff4d21ac2e..cbe4c30c4ee 100644 --- a/tools/mapmerge2/README.md +++ b/tools/mapmerge2/README.md @@ -15,11 +15,13 @@ contains the desired changes. ## Installation -To install Python dependencies, run `requirements-install.bat`, or run -`python -m pip install -r requirements.txt` directly. See the [Git hooks] -documentation to install the Git pre-commit hook which runs the map merger -automatically, or use `tools/mapmerge/Prepare Maps.bat` to save backups before -running `mapmerge.bat`. +To install the Git hooks, open the `tools/hooks/` folder and double-click +`Install.bat` (Linux users run `tools/hooks/install`). + +To use Map Merge manually, such as when using a Git GUI which is incompatible +with some of the hooks, double-click the `.bat` files at the appropriate time. + +A private copy of Python and any dependencies will be installed automatically. For up-to-date installation and detailed troubleshooting instructions, visit the [Map Merger] wiki article. diff --git a/tools/mapmerge2/convert.py b/tools/mapmerge2/convert.py index 35e5dda4433..35fa606834c 100644 --- a/tools/mapmerge2/convert.py +++ b/tools/mapmerge2/convert.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import frontend -import dmm +from . import frontend, dmm if __name__ == '__main__': settings = frontend.read_settings() diff --git a/tools/mapmerge2/dmm2tgm.bat b/tools/mapmerge2/dmm2tgm.bat index bcf6150c2ea..76084281a13 100644 --- a/tools/mapmerge2/dmm2tgm.bat +++ b/tools/mapmerge2/dmm2tgm.bat @@ -1,5 +1,5 @@ @echo off -set MAPROOT=../../_maps/ +set MAPROOT=%~dp0/../../_maps/ set TGM=1 -python convert.py +call "%~dp0\..\bootstrap\python" -m mapmerge2.convert %* pause diff --git a/tools/mapmerge2/mapmerge.bat b/tools/mapmerge2/mapmerge.bat index 5a066226b3c..0b3dc2cf31c 100644 --- a/tools/mapmerge2/mapmerge.bat +++ b/tools/mapmerge2/mapmerge.bat @@ -1,5 +1,5 @@ @echo off -set MAPROOT=../../_maps/ +set MAPROOT=%~dp0/../../_maps/ set TGM=1 -python mapmerge.py +call "%~dp0\..\bootstrap\python" -m mapmerge2.mapmerge %* pause diff --git a/tools/mapmerge2/mapmerge.py b/tools/mapmerge2/mapmerge.py index f449bd948f0..44d01ce3489 100644 --- a/tools/mapmerge2/mapmerge.py +++ b/tools/mapmerge2/mapmerge.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -import frontend import shutil -from dmm import * from collections import defaultdict +from . import frontend +from .dmm import * def merge_map(new_map, old_map, delete_unused=False): if new_map.key_length != old_map.key_length: diff --git a/tools/mapmerge2/precommit.py b/tools/mapmerge2/precommit.py index f5ea49a50ea..7350c403292 100644 --- a/tools/mapmerge2/precommit.py +++ b/tools/mapmerge2/precommit.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import os import pygit2 -import dmm -from mapmerge import merge_map +from . import dmm +from .mapmerge import merge_map def main(repo): if repo.index.conflicts: diff --git a/tools/mapmerge2/requirements-install.bat b/tools/mapmerge2/requirements-install.bat deleted file mode 100644 index 71341af8792..00000000000 --- a/tools/mapmerge2/requirements-install.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -python -m pip install -r requirements.txt -pause diff --git a/tools/mapmerge2/requirements.txt b/tools/mapmerge2/requirements.txt deleted file mode 100644 index cb06e94ee55..00000000000 --- a/tools/mapmerge2/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pygit2==1.0.1 -bidict==0.13.1 -Pillow==7.2.0 diff --git a/tools/mapmerge2/tgm2dmm.bat b/tools/mapmerge2/tgm2dmm.bat index 2748533febf..b446002bcb1 100644 --- a/tools/mapmerge2/tgm2dmm.bat +++ b/tools/mapmerge2/tgm2dmm.bat @@ -1,5 +1,5 @@ @echo off -set MAPROOT=../../_maps/ +set MAPROOT=%~dp0/../../_maps/ set TGM=0 -python convert.py +call "%~dp0\..\bootstrap\python" -m mapmerge2.convert %* pause diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..cf8552635d1 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,7 @@ +pygit2==1.0.1 +bidict==0.13.1 +Pillow==7.2.0 + +# changelogs +PyYaml==5.3.1 +beautifulsoup4==4.9.3