From bea83b29996bc2104b4fd50f0fface335cecb1cf Mon Sep 17 00:00:00 2001 From: Alex 'Avunia' Takiya Date: Thu, 14 Jan 2021 16:52:24 +0100 Subject: [PATCH] [MANUAL MIRROR] Use portable Python for map merge hooks, other tools / Add .dmm merge driver (#2681) * Use portable Python for map merge hooks, other tools (#55658) * Add .dmm merge driver (#55699) This is kind of a prototype. It only fully handles a few situations, and doesn't produce particularly easy-to-read conflict markers when it fails. I hope that it can be useful at least some of the time, can be improved over time, and that the lessons learned can influence a future interactive GUI conflict resolver (integrated into StrongDMM?). In the worst case, one can fall back to the tried and true "manually re-do one side's changes" strategy. **Automatic use**: In `tools/hooks/`, run `Install.bat` **Manual use**, for Git GUIs that don't run merge drivers: while a merge is in progress, in `tools/mapmerge2/`, double-click `Resolve Map Conflicts.bat` This PR also removes the error-prone "Prepare Maps.bat" / "mapmerge.bat" workflow. Those who aren't using the hooks should instead use `Run Before Committing.bat` before committing. First-time contributors who opened a PR without map merging can be advised to run `I Forgot To Map Merge.bat`. * Fix loose double-quot in tradership_faction.dmm Co-authored-by: Tad Hardesty --- .github/workflows/ci_suite.yml | 5 +- dependencies.sh | 11 +- .../maps/shuttles/tradership_faction.dmm | 2 +- tools/HitboxExpander/.gitignore | 3 - tools/HitboxExpander/Hitbox Expander.bat | 2 + tools/HitboxExpander/{README => README.txt} | 8 +- .../{hitbox_expander.py => __main__.py} | 5 +- tools/HitboxExpander/hitbox_expander.sh | 3 + tools/HitboxExpander/install.bat | 1 - tools/HitboxExpander/install.sh | 1 - tools/HitboxExpander/requirements.txt | 1 - tools/UpdatePaths/Update Paths.bat | 3 + .../__main__.py} | 11 +- .../apc_pixel_offsets.txt | 0 .../clothingunderrepath.txt | 0 .../cornersfix.txt | 0 .../replacemonkeys.txt | 0 .../turfs2decals.txt | 0 tools/bootstrap/python | 117 ++++++++++++ tools/bootstrap/python.bat | 1 + tools/bootstrap/python36._pth | 6 + tools/bootstrap/python_.ps1 | 103 +++++++++++ tools/dmi/Resolve Icon Conflicts.bat | 2 + tools/{mapmerge2/dmi.py => dmi/__init__.py} | 55 ++---- .../merge_driver.py} | 56 +++--- tools/dmi/test.py | 39 ++++ tools/hooks/Install.bat | 2 + tools/hooks/README.md | 39 ++-- tools/hooks/Uninstall.bat | 2 + tools/hooks/dmi.merge | 2 +- tools/hooks/dmm.merge | 2 + tools/hooks/install.bat | 16 -- tools/hooks/install.py | 101 +++++++++++ tools/hooks/install.sh | 22 +-- tools/hooks/merge_frontend.py | 169 ++++++++++++++++++ tools/hooks/pre-commit.hook | 3 +- tools/hooks/python.sh | 39 ++-- tools/makeChangelog.bat | 4 +- tools/mapmerge2/I Forgot To Map Merge.bat | 7 + tools/mapmerge2/Prepare Maps.bat | 12 -- tools/mapmerge2/README.md | 39 ++-- tools/mapmerge2/Resolve Map Conflicts.bat | 2 + tools/mapmerge2/Run Before Committing.bat | 2 + tools/mapmerge2/convert.py | 3 +- tools/mapmerge2/dmm.py | 51 ++++-- tools/mapmerge2/dmm2tgm.bat | 5 - tools/mapmerge2/dmm_test.py | 39 ++++ tools/mapmerge2/fixup.py | 132 ++++++++++++++ tools/mapmerge2/mapmerge.bat | 5 - tools/mapmerge2/mapmerge.py | 6 +- tools/mapmerge2/merge_driver.py | 124 +++++++++++++ tools/mapmerge2/precommit.py | 30 +++- tools/mapmerge2/requirements-install.bat | 3 - tools/mapmerge2/requirements.txt | 3 - tools/mapmerge2/tgm2dmm.bat | 5 - tools/requirements.txt | 7 + 56 files changed, 1055 insertions(+), 256 deletions(-) delete mode 100644 tools/HitboxExpander/.gitignore create mode 100644 tools/HitboxExpander/Hitbox Expander.bat rename tools/HitboxExpander/{README => README.txt} (55%) rename tools/HitboxExpander/{hitbox_expander.py => __main__.py} (92%) create mode 100755 tools/HitboxExpander/hitbox_expander.sh delete mode 100644 tools/HitboxExpander/install.bat delete mode 100644 tools/HitboxExpander/install.sh delete mode 100644 tools/HitboxExpander/requirements.txt create mode 100644 tools/UpdatePaths/Update Paths.bat rename tools/{mapmerge2/update_paths.py => UpdatePaths/__main__.py} (95%) rename tools/{mapmerge2/map_scripts => UpdatePaths}/apc_pixel_offsets.txt (100%) rename tools/{mapmerge2/map_scripts => UpdatePaths}/clothingunderrepath.txt (100%) rename tools/{mapmerge2/map_scripts => UpdatePaths}/cornersfix.txt (100%) rename tools/{mapmerge2/map_scripts => UpdatePaths}/replacemonkeys.txt (100%) rename tools/{mapmerge2/map_scripts => UpdatePaths}/turfs2decals.txt (100%) create mode 100755 tools/bootstrap/python create mode 100644 tools/bootstrap/python.bat create mode 100644 tools/bootstrap/python36._pth create mode 100644 tools/bootstrap/python_.ps1 create mode 100644 tools/dmi/Resolve Icon Conflicts.bat rename tools/{mapmerge2/dmi.py => dmi/__init__.py} (87%) rename tools/{mapmerge2/merge_driver_dmi.py => dmi/merge_driver.py} (81%) create mode 100644 tools/dmi/test.py create mode 100644 tools/hooks/Install.bat create mode 100644 tools/hooks/Uninstall.bat create mode 100755 tools/hooks/dmm.merge delete mode 100644 tools/hooks/install.bat create mode 100644 tools/hooks/install.py create mode 100644 tools/hooks/merge_frontend.py create mode 100644 tools/mapmerge2/I Forgot To Map Merge.bat delete mode 100644 tools/mapmerge2/Prepare Maps.bat create mode 100644 tools/mapmerge2/Resolve Map Conflicts.bat create mode 100644 tools/mapmerge2/Run Before Committing.bat delete mode 100644 tools/mapmerge2/dmm2tgm.bat create mode 100644 tools/mapmerge2/dmm_test.py create mode 100644 tools/mapmerge2/fixup.py delete mode 100644 tools/mapmerge2/mapmerge.bat create mode 100644 tools/mapmerge2/merge_driver.py delete mode 100644 tools/mapmerge2/requirements-install.bat delete mode 100644 tools/mapmerge2/requirements.txt delete mode 100644 tools/mapmerge2/tgm2dmm.bat create mode 100644 tools/requirements.txt diff --git a/.github/workflows/ci_suite.yml b/.github/workflows/ci_suite.yml index aaa87fd6d3b..a5bf602c4ec 100644 --- a/.github/workflows/ci_suite.yml +++ b/.github/workflows/ci_suite.yml @@ -22,7 +22,7 @@ jobs: pip3 install setuptools bash tools/ci/install_build_tools.sh bash tools/ci/install_spaceman_dmm.sh dreamchecker - pip3 install -r tools/mapmerge2/requirements.txt + pip3 install -r tools/requirements.txt - name: Run Linters run: | bash tools/ci/check_filedirs.sh tgstation.dme @@ -31,7 +31,8 @@ jobs: find . -name "*.json" -not -path "*/node_modules/*" -print0 | xargs -0 python3 ./tools/json_verifier.py bash tools/ci/build_tgui.sh bash tools/ci/check_grep.sh - python3 tools/mapmerge2/dmi.py --test + tools/bootstrap/python -m dmi.test + tools/bootstrap/python -m mapmerge2.dmm_test ~/dreamchecker > ${GITHUB_WORKSPACE}/output-annotations.txt 2>&1 - name: Annotate Lints uses: yogstation13/DreamAnnotate@v1 diff --git a/dependencies.sh b/dependencies.sh index efe08331339..e6ea4396b60 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -1,13 +1,13 @@ -#!/bin/bash +#!/bin/sh #Project dependencies file #Final authority on what's required to fully build the project # byond version # Extracted from the Dockerfile. Change by editing Dockerfile's FROM command. -LIST=($(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile)) -export BYOND_MAJOR=${LIST[0]} -export BYOND_MINOR=${LIST[1]} +LIST="$(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile)" +export BYOND_MAJOR=${LIST% *} +export BYOND_MINOR=${LIST#* } unset LIST #rust_g git tag @@ -21,3 +21,6 @@ export SPACEMAN_DMM_VERSION=suite-1.6 # Extools git tag export EXTOOLS_VERSION=v0.0.7 + +# Python version for mapmerge and other tools +export PYTHON_VERSION=3.6.8 diff --git a/modular_skyrat/modules/faction/maps/shuttles/tradership_faction.dmm b/modular_skyrat/modules/faction/maps/shuttles/tradership_faction.dmm index ccf9c8d2fa3..47ebf4ded6f 100644 --- a/modular_skyrat/modules/faction/maps/shuttles/tradership_faction.dmm +++ b/modular_skyrat/modules/faction/maps/shuttles/tradership_faction.dmm @@ -2648,7 +2648,7 @@ name = "Starboard Blast Door Control"; pixel_x = -24; pixel_y = -5; - req_access_txt = "403"" + req_access_txt = "403" }, /obj/structure/closet/crate/secure/tradership_cargo_valuable{ used_preset = 2 diff --git a/tools/HitboxExpander/.gitignore b/tools/HitboxExpander/.gitignore deleted file mode 100644 index ec7f5fd7c18..00000000000 --- a/tools/HitboxExpander/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -Imaging-1.1.7/ -zlib/ - diff --git a/tools/HitboxExpander/Hitbox Expander.bat b/tools/HitboxExpander/Hitbox Expander.bat new file mode 100644 index 00000000000..ac69e5505db --- /dev/null +++ b/tools/HitboxExpander/Hitbox Expander.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m HitboxExpander %* +@pause diff --git a/tools/HitboxExpander/README b/tools/HitboxExpander/README.txt similarity index 55% rename from tools/HitboxExpander/README rename to tools/HitboxExpander/README.txt index 1e1ef5f153c..67a5b8bf42d 100755 --- a/tools/HitboxExpander/README +++ b/tools/HitboxExpander/README.txt @@ -1,10 +1,4 @@ -Setup: Install python3 and run install.bat for windows, install.sh for unix. -Alternatively, you can manually install the Pillow package with -``` -pip install Pillow -``` - -Usage: python hitbox_expander.py +Usage: tools/bootstrap/python -m HitboxExpander This tool expands the hitbox of the given image by 1 pixel. Works by changing some of the fully-transparent pixels to alpha=1 black pixels. diff --git a/tools/HitboxExpander/hitbox_expander.py b/tools/HitboxExpander/__main__.py similarity index 92% rename from tools/HitboxExpander/hitbox_expander.py rename to tools/HitboxExpander/__main__.py index 36fb8085375..c06352440c7 100755 --- a/tools/HitboxExpander/hitbox_expander.py +++ b/tools/HitboxExpander/__main__.py @@ -74,7 +74,10 @@ icons_dir = os.path.join(root_dir, "icons") def Main(): if len(sys.argv) != 2: - print("Usage: hitbox_expander.py filename.dmi") + if os.name == 'nt': + print("Usage: drag-and-drop a .dmi onto `Hitbox Expander.bat`\n or") + with open(os.path.join(current_dir, "README.txt")) as f: + print(f.read()) return 0 try: diff --git a/tools/HitboxExpander/hitbox_expander.sh b/tools/HitboxExpander/hitbox_expander.sh new file mode 100755 index 00000000000..aca0ebf9e35 --- /dev/null +++ b/tools/HitboxExpander/hitbox_expander.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +exec "$(dirname "$0")/../bootstrap/python" -m HitboxExpander "$@" diff --git a/tools/HitboxExpander/install.bat b/tools/HitboxExpander/install.bat deleted file mode 100644 index 78c80cd7d30..00000000000 --- a/tools/HitboxExpander/install.bat +++ /dev/null @@ -1 +0,0 @@ -python3 -m pip install -r requirements.txt diff --git a/tools/HitboxExpander/install.sh b/tools/HitboxExpander/install.sh deleted file mode 100644 index 78c80cd7d30..00000000000 --- a/tools/HitboxExpander/install.sh +++ /dev/null @@ -1 +0,0 @@ -python3 -m pip install -r requirements.txt diff --git a/tools/HitboxExpander/requirements.txt b/tools/HitboxExpander/requirements.txt deleted file mode 100644 index 334e961ca1a..00000000000 --- a/tools/HitboxExpander/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Pillow==7.2.0 diff --git a/tools/UpdatePaths/Update Paths.bat b/tools/UpdatePaths/Update Paths.bat new file mode 100644 index 00000000000..c126388de7e --- /dev/null +++ b/tools/UpdatePaths/Update Paths.bat @@ -0,0 +1,3 @@ +@echo off +call "%~dp0\..\bootstrap\python" -m UpdatePaths %* +pause diff --git a/tools/mapmerge2/update_paths.py b/tools/UpdatePaths/__main__.py similarity index 95% rename from tools/mapmerge2/update_paths.py rename to tools/UpdatePaths/__main__.py index 2c316e941de..ba2da1ea049 100644 --- a/tools/mapmerge2/update_paths.py +++ b/tools/UpdatePaths/__main__.py @@ -1,9 +1,10 @@ # A script and syntax for applying path updates to maps. import re import os +import sys import argparse -import frontend -from dmm import * +from mapmerge2 import frontend +from mapmerge2.dmm import * desc = """ Update dmm files given update file/string. @@ -167,7 +168,11 @@ def main(args): if __name__ == "__main__": - parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) + prog = __spec__.name.replace('.__main__', '') + if os.name == 'nt' and len(sys.argv) <= 1: + print("usage: drag-and-drop a path script .txt onto `Update Paths.bat`\n or") + + parser = argparse.ArgumentParser(prog=prog, description=desc, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("update_source", help="update file path / line of update notation") parser.add_argument("--map", "-m", help="path to update, defaults to all maps in maps directory") parser.add_argument("--directory", "-d", help="path to maps directory, defaults to _maps/") diff --git a/tools/mapmerge2/map_scripts/apc_pixel_offsets.txt b/tools/UpdatePaths/apc_pixel_offsets.txt similarity index 100% rename from tools/mapmerge2/map_scripts/apc_pixel_offsets.txt rename to tools/UpdatePaths/apc_pixel_offsets.txt diff --git a/tools/mapmerge2/map_scripts/clothingunderrepath.txt b/tools/UpdatePaths/clothingunderrepath.txt similarity index 100% rename from tools/mapmerge2/map_scripts/clothingunderrepath.txt rename to tools/UpdatePaths/clothingunderrepath.txt diff --git a/tools/mapmerge2/map_scripts/cornersfix.txt b/tools/UpdatePaths/cornersfix.txt similarity index 100% rename from tools/mapmerge2/map_scripts/cornersfix.txt rename to tools/UpdatePaths/cornersfix.txt diff --git a/tools/mapmerge2/map_scripts/replacemonkeys.txt b/tools/UpdatePaths/replacemonkeys.txt similarity index 100% rename from tools/mapmerge2/map_scripts/replacemonkeys.txt rename to tools/UpdatePaths/replacemonkeys.txt diff --git a/tools/mapmerge2/map_scripts/turfs2decals.txt b/tools/UpdatePaths/turfs2decals.txt similarity index 100% rename from tools/mapmerge2/map_scripts/turfs2decals.txt rename to tools/UpdatePaths/turfs2decals.txt diff --git a/tools/bootstrap/python b/tools/bootstrap/python new file mode 100755 index 00000000000..f9ffc13edb6 --- /dev/null +++ b/tools/bootstrap/python @@ -0,0 +1,117 @@ +#!/bin/sh +# bootstrap/python +# +# Python-finding script for all `sh` environments, including Linux, MSYS2, +# Git for Windows, and GitHub Desktop. Invokable from CLI or automation. +# +# If a python.exe installed by `python_.ps1` is present, it will be used. +# Otherwise, this script requires a system `python3` and `pip` to be provided, +# and will create a standard virtualenv in which to install `requirements.txt`. +set -e + +# Convenience variables +Bootstrap="$(dirname "$0")" +Sdk="$(dirname "$Bootstrap")" +Cache="$Bootstrap/.cache" +if [ "$TG_BOOTSTRAP_CACHE" ]; then + Cache="$TG_BOOTSTRAP_CACHE" +fi +OldPWD="$PWD" +cd "$Bootstrap/../.." +. ./dependencies.sh # sets PYTHON_VERSION +cd "$OldPWD" +PythonVersion="$PYTHON_VERSION" +PythonDir="$Cache/python-$PythonVersion" +PythonExe="$PythonDir/python.exe" +Log="$Cache/last-command.log" + +# If a portable Python for Windows is not present, search on $PATH. +if [ "$(uname)" = "Linux" ] || [ ! -f "$PythonExe" ]; then + # Strip the "App Execution Aliases" from $PATH. Even if the user installed + # Python using the Windows Store on purpose, these aliases always generate + # "Permission denied" errors when sh.exe tries to invoke them. + PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":") + + # Try to find a Python executable. + if command -v python3 >/dev/null 2>&1; then + PythonExe=python3 + elif command -v python >/dev/null 2>&1; then + PythonExe=python + elif command -v py >/dev/null 2>&1; then + PythonExe="py -3" + else + echo + if command -v apt-get >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo apt-get install python3 python3-pip" + elif [ "$(uname -o)" = "Msys" ]; then + echo "Please run tools/bootstrap/python.bat instead of tools/bootstrap/python once to" + echo "install Python automatically, or install it from https://www.python.org/downloads/" + # TODO: give MSYS pacman advice? + elif command -v pacman >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo pacman -S python python-pip" + else + echo "Please install Python from https://www.python.org/downloads/ or using your system's package manager." + fi + echo + exit 1 + fi + + # Create a venv and activate it + PythonDir="$Cache/venv" + if [ ! -d "$PythonDir" ]; then + echo "Creating virtualenv..." + "$PythonExe" -m venv "$PythonDir" + fi + if [ -f "$PythonDir/bin/python" ]; then + PythonExe="$PythonDir/bin/python" + elif [ -f "$PythonDir/scripts/python3.exe" ]; then + PythonExe="$PythonDir/scripts/python3.exe"; + else + echo "bootstrap/python failed to find the python executable inside its virtualenv" + exit 1 + fi +fi + +# Use pip to install our requirements +if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then + echo "Updating dependencies..." + "$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt" + cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt" + echo "---" +fi + +# Verify version and deduce the path separator +PythonMajor=${PythonVersion%%.*} +PythonMinor=${PythonVersion#*.} +PythonMinor=${PythonMinor%.*} +PATHSEP=$("$PythonExe" - "$PythonMajor" "$PythonMinor" <<'EOF' +import sys, os +if sys.version_info.major != int(sys.argv[1]) or sys.version_info.minor < int(sys.argv[2]): + print("Error: Python ", sys.argv[1], ".", sys.argv[2], " or later is required, but you have:\n", sys.version, sep="", file=sys.stderr) + exit(1) +print(os.pathsep) +EOF +) + +# Cheap shell function if tee.exe is not available +if ! command -v tee >/dev/null 2>&1; then + tee() { + # Fudge: assume $1 is always "-a" + while read -r line; do + echo "$line" >> "$2" + echo "$line" + done + } +fi + +# Invoke python with all command-line arguments +export PYTHONPATH="$Sdk$PATHSEP${PYTHONPATH:-}" +mkdir -p "$Cache" +printf '%s\n' "$PythonExe" "$@" > "$Log" +printf -- '---\n' >> "$Log" +exec 4>&1 +exitstatus=$({ { set +e; "$PythonExe" -u "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1) +exec 4>&- +exit "$exitstatus" diff --git a/tools/bootstrap/python.bat b/tools/bootstrap/python.bat new file mode 100644 index 00000000000..3b20c2d436b --- /dev/null +++ b/tools/bootstrap/python.bat @@ -0,0 +1 @@ +@call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\python_.ps1" %* diff --git a/tools/bootstrap/python36._pth b/tools/bootstrap/python36._pth new file mode 100644 index 00000000000..2bd2e8a7d6e --- /dev/null +++ b/tools/bootstrap/python36._pth @@ -0,0 +1,6 @@ +python36.zip +. +..\..\.. + +# Uncomment to run site.main() automatically +import site diff --git a/tools/bootstrap/python_.ps1 b/tools/bootstrap/python_.ps1 new file mode 100644 index 00000000000..c9e39156137 --- /dev/null +++ b/tools/bootstrap/python_.ps1 @@ -0,0 +1,103 @@ +# bootstrap/python_.ps1 +# +# Python bootstrapping script for Windows. +# +# Automatically downloads a portable edition of a pinned Python version to +# a cache directory, installs Pip, installs `requirements.txt`, and then invokes +# Python. +# +# The underscore in the name is so that typing `bootstrap/python` into +# PowerShell finds the `.bat` file first, which ensures this script executes +# regardless of ExecutionPolicy. +$host.ui.RawUI.WindowTitle = "starting :: python $args" +$ErrorActionPreference = "Stop" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function ExtractVersion { + param([string] $Path, [string] $Key) + foreach ($Line in Get-Content $Path) { + if ($Line.StartsWith("export $Key=")) { + return $Line.Substring("export $Key=".Length) + } + } + throw "Couldn't find value for $Key in $Path" +} + +# Convenience variables +$Bootstrap = Split-Path $script:MyInvocation.MyCommand.Path +$Tools = Split-Path $Bootstrap +$Cache = "$Bootstrap/.cache" +if ($Env:TG_BOOTSTRAP_CACHE) { + $Cache = $Env:TG_BOOTSTRAP_CACHE +} +$PythonVersion = ExtractVersion -Path "$Bootstrap/../../dependencies.sh" -Key "PYTHON_VERSION" +$PythonDir = "$Cache/python-$PythonVersion" +$PythonExe = "$PythonDir/python.exe" +$Log = "$Cache/last-command.log" + +# Download and unzip a portable version of Python +if (!(Test-Path $PythonExe -PathType Leaf)) { + $host.ui.RawUI.WindowTitle = "Downloading Python $PythonVersion..." + New-Item $Cache -ItemType Directory -ErrorAction silentlyContinue | Out-Null + + $Archive = "$Cache/python-$PythonVersion-embed.zip" + Invoke-WebRequest ` + "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" ` + -OutFile $Archive ` + -ErrorAction Stop + + [System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir) + + # Copy a ._pth file without "import site" commented, so pip will work + Copy-Item "$Bootstrap/python36._pth" $PythonDir ` + -ErrorAction Stop + + Remove-Item $Archive +} + +# Install pip +if (!(Test-Path "$PythonDir/Scripts/pip.exe")) { + $host.ui.RawUI.WindowTitle = "Downloading Pip..." + + Invoke-WebRequest "https://bootstrap.pypa.io/get-pip.py" ` + -OutFile "$Cache/get-pip.py" ` + -ErrorAction Stop + + & $PythonExe "$Cache/get-pip.py" --no-warn-script-location + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Remove-Item "$Cache/get-pip.py" ` + -ErrorAction Stop +} + +# Use pip to install our requirements +if (!(Test-Path "$PythonDir/requirements.txt") -or ((Get-FileHash "$Tools/requirements.txt").hash -ne (Get-FileHash "$PythonDir/requirements.txt").hash)) { + $host.ui.RawUI.WindowTitle = "Updating dependencies..." + + & $PythonExe -m pip install -U pip -r "$Tools/requirements.txt" + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Copy-Item "$Tools/requirements.txt" "$PythonDir/requirements.txt" + Write-Output "`n---`n" +} + +# Invoke python with all command-line arguments +Write-Output $PythonExe | Out-File -Encoding utf8 $Log +[System.String]::Join([System.Environment]::NewLine, $args) | Out-File -Encoding utf8 -Append $Log +Write-Output "---" | Out-File -Encoding utf8 -Append $Log +$host.ui.RawUI.WindowTitle = "python $args" +$ErrorActionPreference = "Continue" +& $PythonExe -u $args 2>&1 | ForEach-Object { + $str = "$_" + if ($_.GetType() -eq [System.Management.Automation.ErrorRecord]) { + $str = $str.TrimEnd("`r`n") + } + $str | Out-File -Encoding utf8 -Append $Log + $str | Out-Host +} +exit $LastExitCode diff --git a/tools/dmi/Resolve Icon Conflicts.bat b/tools/dmi/Resolve Icon Conflicts.bat new file mode 100644 index 00000000000..8def88442c6 --- /dev/null +++ b/tools/dmi/Resolve Icon Conflicts.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python.bat" -m dmi.merge_driver --posthoc %* +@pause diff --git a/tools/mapmerge2/dmi.py b/tools/dmi/__init__.py similarity index 87% rename from tools/mapmerge2/dmi.py rename to tools/dmi/__init__.py index e87cf507b99..cdf28463bcd 100644 --- a/tools/mapmerge2/dmi.py +++ b/tools/dmi/__init__.py @@ -12,10 +12,10 @@ NORTH = 1 SOUTH = 2 EAST = 4 WEST = 8 -SOUTHEAST = SOUTH|EAST -SOUTHWEST = SOUTH|WEST -NORTHEAST = NORTH|EAST -NORTHWEST = NORTH|WEST +SOUTHEAST = SOUTH | EAST +SOUTHWEST = SOUTH | WEST +NORTHEAST = NORTH | EAST +NORTHWEST = NORTH | WEST CARDINALS = [NORTH, SOUTH, EAST, WEST] DIR_ORDER = [SOUTH, NORTH, EAST, WEST, SOUTHEAST, SOUTHWEST, NORTHEAST, NORTHWEST] @@ -34,6 +34,7 @@ DIR_NAMES = { None: SOUTH, } + class Dmi: version = "4.0" @@ -134,7 +135,7 @@ class Dmi: comment += f"state = {escape(state.name)}\n" comment += f"\tdirs = {state.dirs}\n" comment += f"\tframes = {state.framecount}\n" - if state.framecount > 1 and len(state.delays): #any(x != 1 for x in state.delays): + if state.framecount > 1 and len(state.delays): # any(x != 1 for x in state.delays): comment += "\tdelay = " + ",".join(map(str, state.delays)) + "\n" if state.loop != 0: comment += f"\tloop = {state.loop}\n" @@ -175,6 +176,7 @@ class Dmi: output = output.convert('P') output.save(filename, 'png', optimize=True, pnginfo=pnginfo) + class State: def __init__(self, dmi, name, *, loop=LOOP_UNLIMITED, rewind=False, movement=False, dirs=1): self.dmi = dmi @@ -216,11 +218,13 @@ class State: def get_frame(self, *args, **kwargs): return self.frames[self._frame_index(*args, **kwargs)] + def escape(text): text = text.replace('\\', '\\\\') text = text.replace('"', '\\"') return f'"{text}"' + def unescape(text, quote='"'): if text == 'null': return None @@ -231,51 +235,14 @@ def unescape(text, quote='"'): text = text.replace('\\\\', '\\') return text + def parse_num(value): if '.' in value: return float(value) return int(value) + def parse_bool(value): if value not in ('0', '1'): raise ValueError(value) return value == '1' - -def _self_test(): - # test: can we load every DMI in the tree - import os - - count = 0 - for dirpath, dirnames, filenames in os.walk('.'): - if '.git' in dirnames: - dirnames.remove('.git') - for filename in filenames: - if filename.endswith('.dmi'): - fullpath = os.path.join(dirpath, filename) - try: - Dmi.from_file(fullpath) - except: - print('Failed on:', fullpath) - raise - count += 1 - - print(f"Successfully parsed {count} dmi files") - -def _usage(): - import sys - print(f"Usage:") - print(f" {sys.argv[0]} --test") - exit(1) - -def _main(): - import sys - if len(sys.argv) < 2: - return _usage() - - if sys.argv[1] == '--test': - return _self_test() - - return _usage() - -if __name__ == '__main__': - _main() diff --git a/tools/mapmerge2/merge_driver_dmi.py b/tools/dmi/merge_driver.py similarity index 81% rename from tools/mapmerge2/merge_driver_dmi.py rename to tools/dmi/merge_driver.py index 37c0abd9543..75c3daacb07 100644 --- a/tools/mapmerge2/merge_driver_dmi.py +++ b/tools/dmi/merge_driver.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import sys import dmi +from hooks.merge_frontend import MergeDriver + def images_equal(left, right): if left.size != right.size: @@ -15,6 +17,7 @@ def images_equal(left, right): return False return True + def states_equal(left, right): result = True @@ -31,9 +34,11 @@ def states_equal(left, right): return result + def key_of(state): return (state.name, state.movement) + def dictify(sheet): result = {} for state in sheet.states: @@ -43,6 +48,7 @@ def dictify(sheet): result[k] = state return result + def three_way_merge(base, left, right): base_dims = base.width, base.height if base_dims != (left.width, left.height) or base_dims != (right.width, right.height): @@ -145,33 +151,31 @@ def three_way_merge(base, left, right): merged.states = final_states return len(conflicts), merged -def main(path, original, left, right): - print(f"Merging icon: {path}") - icon_orig = dmi.Dmi.from_file(original) - icon_left = dmi.Dmi.from_file(left) - icon_right = dmi.Dmi.from_file(right) +class DmiDriver(MergeDriver): + driver_id = 'dmi' + + def merge(self, base, left, right): + icon_base = dmi.Dmi.from_file(base) + icon_left = dmi.Dmi.from_file(left) + icon_right = dmi.Dmi.from_file(right) + trouble, merge_result = three_way_merge(icon_base, icon_left, icon_right) + return not trouble, merge_result + + def to_file(self, outfile, merge_result): + merge_result.to_file(outfile) + + def post_announce(self, success, merge_result): + if not success: + print("!!! Manual merge required!") + if merge_result: + print(" A best-effort merge was performed. You must edit the icon and remove all") + print(" icon states marked with !CONFLICT!, leaving only the desired icon.") + else: + print(" The icon was totally unable to be merged, you must start with one version") + print(" or the other and manually resolve the conflict.") + print(" Information about which states conflicted is listed above.") - trouble, merged = three_way_merge(icon_orig, icon_left, icon_right) - if merged: - merged.to_file(left) - if trouble: - print("!!! Manual merge required!") - if merged: - print(" A best-effort merge was performed. You must edit the icon and remove all") - print(" icon states marked with !CONFLICT!, leaving only the desired icon.") - else: - print(" The icon was totally unable to be merged, you must start with one version") - print(" or the other and manually resolve the conflict.") - print(" Information about which states conflicted is listed above.") - return trouble if __name__ == '__main__': - if len(sys.argv) != 6: - print("DMI merge driver called with wrong number of arguments") - print(" usage: merge-driver-dmi %P %O %A %B %L") - exit(1) - - # "left" is also the file that ought to be overwritten - _, path, original, left, right, conflict_size_marker = sys.argv - exit(main(path, original, left, right)) + exit(DmiDriver().main()) diff --git a/tools/dmi/test.py b/tools/dmi/test.py new file mode 100644 index 00000000000..09629927d88 --- /dev/null +++ b/tools/dmi/test.py @@ -0,0 +1,39 @@ +import os +import sys +from dmi import * + + +def _self_test(): + # test: can we load every DMI in the tree + count = 0 + for dirpath, dirnames, filenames in os.walk('.'): + if '.git' in dirnames: + dirnames.remove('.git') + for filename in filenames: + if filename.endswith('.dmi'): + fullpath = os.path.join(dirpath, filename) + try: + Dmi.from_file(fullpath) + except Exception: + print('Failed on:', fullpath) + raise + count += 1 + + print(f"{os.path.relpath(__file__)}: successfully parsed {count} .dmi files") + + +def _usage(): + print(f"Usage:") + print(f" tools{os.sep}bootstrap{os.sep}python -m {__spec__.name}") + exit(1) + + +def _main(): + if len(sys.argv) == 1: + return _self_test() + + return _usage() + + +if __name__ == '__main__': + _main() diff --git a/tools/hooks/Install.bat b/tools/hooks/Install.bat new file mode 100644 index 00000000000..1bfc1b50d5c --- /dev/null +++ b/tools/hooks/Install.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m hooks.install %* +@pause diff --git a/tools/hooks/README.md b/tools/hooks/README.md index b15fb493d8d..c7105903972 100644 --- a/tools/hooks/README.md +++ b/tools/hooks/README.md @@ -1,41 +1,36 @@ # Git Integration Hooks -This folder contains installable scripts for [Git hooks] and [merge drivers]. +This folder contains installable scripts for Git [hooks] and [merge drivers]. Use of these hooks and drivers is optional and they must be installed explicitly before they take effect. To install the current set of hooks, or update if new hooks are added, run -`install.bat` (Windows) or `install.sh` (Unix-like) as appropriate. +`Install.bat` (Windows) or `tools/hooks/install.sh` (Unix-like) as appropriate. -Hooks expect a Unix-like environment on the backend. Usually this is handled -automatically by GUI tools like TortoiseGit and GitHub for Windows, but -[Git for Windows] is an option if you prefer to use a CLI even on Windows. +If your Git GUI does not support a given hook, there is usually a `.bat` file +or other script you can run instead - see the links below for details. -## Current Hooks +## Hooks -* **Pre-commit**: Runs [mapmerge2] on changed maps, if any. -* **DMI merger**: Attempts to [fix icon conflicts] when performing a git merge. - If it succeeds, the file is marked merged. If it fails, it logs what states - are still in conflict and adds them to the .dmi file, where the desired - resolution can be chosen. +* **Pre-commit**: Runs [mapmerge2] to reduce the diff on any changed maps. +* **DMI merger**: Attempts to [fix icon conflicts] when performing a Git merge. +* **DMM merger**: Attempts to [fix map conflicts] when performing a Git merge. ## Adding New Hooks -New [Git hooks] may be added by creating a file named `.hook` in +New Git [hooks] may be added by creating a file named `.hook` in this directory. Git determines what hooks are available and what their names -are. The install script copies the `.hook` file into `.git/hooks`, so editing -the `.hook` file will require a reinstall. +are. -New [merge drivers] may be added by adding a shell script named `.merge` +New Git [merge drivers] may be added by adding a shell script named `.merge` and updating `.gitattributes` in the root of the repository to include the line -`*. merge=`. The install script will set up the merge driver to point -to the `.merge` file directly, and editing it will not require a reinstall. +`*. merge=`. -`tools/hooks/python.sh` may be used as a trampoline to ensure that the correct -version of Python is found. +Adding or removing hooks or merge drivers requires running the install script +again, but modifying them does not. See existing `.hook` and `.merge` files for examples. -[Git hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks +[hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks [merge drivers]: https://git-scm.com/docs/gitattributes#_performing_a_three_way_merge -[Git for Windows]: https://gitforwindows.org/ [mapmerge2]: ../mapmerge2/README.md -[fix icon conflicts]: ../mapmerge2/merge_driver_dmi.py +[fix icon conflicts]: https://tgstation13.org/wiki/Resolving_icon_conflicts +[fix map conflicts]: https://tgstation13.org/wiki/Map_Merger diff --git a/tools/hooks/Uninstall.bat b/tools/hooks/Uninstall.bat new file mode 100644 index 00000000000..862cbaf8bc4 --- /dev/null +++ b/tools/hooks/Uninstall.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m hooks.install --uninstall %* +@pause diff --git a/tools/hooks/dmi.merge b/tools/hooks/dmi.merge index 7fd9f171bf0..d82ac06be86 100755 --- a/tools/hooks/dmi.merge +++ b/tools/hooks/dmi.merge @@ -1,2 +1,2 @@ #!/bin/sh -exec tools/hooks/python.sh -m merge_driver_dmi "$@" +exec tools/bootstrap/python -m dmi.merge_driver "$@" diff --git a/tools/hooks/dmm.merge b/tools/hooks/dmm.merge new file mode 100755 index 00000000000..64fa16b5020 --- /dev/null +++ b/tools/hooks/dmm.merge @@ -0,0 +1,2 @@ +#!/bin/sh +exec tools/bootstrap/python -m mapmerge2.merge_driver "$@" diff --git a/tools/hooks/install.bat b/tools/hooks/install.bat deleted file mode 100644 index 7a11129a2a2..00000000000 --- a/tools/hooks/install.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -cd %~dp0 -for %%f in (*.hook) do ( - echo Installing hook: %%~nf - copy %%f ..\..\.git\hooks\%%~nf >nul -) -for %%f in (*.merge) do ( - echo Installing merge driver: %%~nf - echo [merge "%%~nf"]^ - - driver = tools/hooks/%%f %%P %%O %%A %%B %%L >> ..\..\.git\config -) -echo Installing Python dependencies -python -m pip install -r ..\mapmerge2\requirements.txt -echo Done -pause diff --git a/tools/hooks/install.py b/tools/hooks/install.py new file mode 100644 index 00000000000..0736b29681e --- /dev/null +++ b/tools/hooks/install.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# hooks/install.py +# +# This script is configured by adding `*.hook` and `*.merge` files in the same +# directory. Such files should be `#!/bin/sh` scripts, usually invoking Python. +# This installer will have to be re-run any time a hook or merge file is added +# or removed, but not when they are changed. +# +# Merge drivers will also need a corresponding entry in the `.gitattributes` +# file. + +import os +import stat +import glob +import re +import pygit2 +import shlex + + +def write_hook(fname, command): + with open(fname, 'w', encoding='utf-8', newline='\n') as f: + print("#!/bin/sh", file=f) + print("exec", command, file=f) + + # chmod +x + st = os.stat(fname) + if not hasattr(st, 'st_file_attributes'): + os.chmod(fname, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def _find_stuff(target=None): + repo_dir = pygit2.discover_repository(target or os.getcwd()) + repo = pygit2.Repository(repo_dir) + # Strips any active worktree to find the hooks directory. + root_repo_dir = re.sub(r'/.git/worktrees/[^/]+/', '/.git/', repo_dir) + hooks_dir = os.path.join(root_repo_dir, 'hooks') + return repo, hooks_dir + + +def uninstall(target=None, keep=()): + repo, hooks_dir = _find_stuff(target) + + # Remove hooks + for fname in glob.glob(os.path.join(hooks_dir, '*')): + _, shortname = os.path.split(fname) + if not fname.endswith('.sample') and f"{shortname}.hook" not in keep: + print('Removing hook:', shortname) + os.unlink(fname) + + # Remove merge driver configuration + for entry in repo.config: + match = re.match(r'^merge\.([^.]+)\.driver$', entry.name) + if match and f"{match.group(1)}.merge" not in keep: + print('Removing merge driver:', match.group(1)) + del repo.config[entry.name] + + +def install(target=None): + repo, hooks_dir = _find_stuff(target) + tools_hooks = os.path.split(__file__)[0] + + keep = set() + for full_path in glob.glob(os.path.join(tools_hooks, '*.hook')): + _, fname = os.path.split(full_path) + name, _ = os.path.splitext(fname) + print('Installing hook:', name) + keep.add(fname) + relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/')) + write_hook(os.path.join(hooks_dir, name), f'{relative_path} "$@"') + + # Use libgit2 config manipulation to set the merge driver config. + for full_path in glob.glob(os.path.join(tools_hooks, '*.merge')): + # Merge drivers are documented here: https://git-scm.com/docs/gitattributes + _, fname = os.path.split(full_path) + name, _ = os.path.splitext(fname) + print('Installing merge driver:', name) + keep.add(fname) + # %P: "real" path of the file, should not usually be read or modified + # %O: ancestor's version + # %A: current version, and also the output path + # %B: other branches' version + # %L: conflict marker size + relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/')) + repo.config[f"merge.{name}.driver"] = f'{relative_path} %P %O %A %B %L' + + uninstall(target, keep=keep) + + +def main(argv): + if len(argv) <= 1: + return install() + elif argv[1] == '--uninstall': + return uninstall() + else: + print("Usage: python -m hooks.install [--uninstall]") + return 1 + + +if __name__ == '__main__': + import sys + exit(main(sys.argv)) diff --git a/tools/hooks/install.sh b/tools/hooks/install.sh index 899b85b941a..d0e995ca097 100755 --- a/tools/hooks/install.sh +++ b/tools/hooks/install.sh @@ -1,20 +1,2 @@ -#!/bin/bash -set -e -shopt -s nullglob -cd "$(dirname "$0")" -for f in *.hook; do - echo Installing hook: ${f%.hook} - cp $f ../../.git/hooks/${f%.hook} -done -for f in *.merge; do - echo Installing merge driver: ${f%.merge} - git config --replace-all merge.${f%.merge}.driver "tools/hooks/$f %P %O %A %B %L" -done - -echo "Installing tgui hooks" -../../tgui/bin/tgui --install-git-hooks - -echo "Installing Python dependencies" -./python.sh -m pip install -r ../mapmerge2/requirements.txt - -echo "Done" +#!/bin/sh +exec "$(dirname "$0")/../bootstrap/python" -m hooks.install "$@" diff --git a/tools/hooks/merge_frontend.py b/tools/hooks/merge_frontend.py new file mode 100644 index 00000000000..b735d1e3a5c --- /dev/null +++ b/tools/hooks/merge_frontend.py @@ -0,0 +1,169 @@ +# merge_frontend.py +import sys +import io +import os +import pygit2 +import collections +import typing + + +ENCODING = 'utf-8' + + +class MergeReturn(typing.NamedTuple): + success: bool + merge_result: typing.Optional[object] + + +class MergeDriver: + driver_id: typing.Optional[str] = None + + def pre_announce(self, path: str): + """ + Called before merge() is called, with a human-friendly path for output. + """ + print(f"Merging {self.driver_id}: {path}") + + def merge(self, base: typing.BinaryIO, left: typing.BinaryIO, right: typing.BinaryIO) -> MergeReturn: + """ + Read from three BinaryIOs: base (common ancestor), left (ours), and + right (theirs). Perform the actual three-way merge operation. Leave + conflict markers if necessary. + + Return (False, None) to indicate the merge driver totally failed. + Return (False, merge_result) if the result contains conflict markers. + Return (True, merge_result) if everything went smoothly. + """ + raise NotImplementedError + + def to_file(self, output: typing.BinaryIO, merge_result: object): + """ + Save the merge() result to the given output stream. + Override this if the merge() result is not bytes or str. + """ + if isinstance(merge_result, bytes): + output.write(merge_result) + elif isinstance(merge_result, str): + with io.TextIOWrapper(output, ENCODING) as f: + f.write(merge_result) + else: + raise NotImplementedError + + def post_announce(self, success: bool, merge_result: object): + """ + Called after merge() is called, to warn the user if action is needed. + """ + if not success: + print("!!! Manual merge required") + if merge_result: + print(" A best-effort merge was performed. You must finish the job yourself.") + else: + print(" No merge was possible. You must resolve the conflict yourself.") + + def main(self, args: typing.List[str] = None): + return _main(self, args or sys.argv[1:]) + + +def _main(driver: MergeDriver, args: typing.List[str]): + if len(args) > 0 and args[0] == '--posthoc': + return _posthoc_main(driver, args[1:]) + else: + return _driver_main(driver, args) + + +def _driver_main(driver: MergeDriver, args: typing.List[str]): + """ + Act like a normal Git merge driver, called by Git during a merge. + """ + if len(args) != 5: + print("merge driver called with wrong number of arguments") + print(" usage: %P %O %A %B %L") + return 1 + + path, path_base, path_left, path_right, _ = args + driver.pre_announce(path) + + with open(path_base, 'rb') as io_base: + with open(path_left, 'rb') as io_left: + with open(path_right, 'rb') as io_right: + success, merge_result = driver.merge(io_base, io_left, io_right) + + if merge_result: + # If we got anything, write it to the working directory. + with open(path_left, 'wb') as io_output: + driver.to_file(io_output, merge_result) + + driver.post_announce(success, merge_result) + if not success: + # If we were not successful, do not mark the conflict as resolved. + return 1 + + +def _posthoc_main(driver: MergeDriver, args: typing.List[str]): + """ + Apply merge driver logic to a repository which is already in a conflicted + state, running the driver on any conflicted files. + """ + repo_dir = pygit2.discover_repository(os.getcwd()) + repo = pygit2.Repository(repo_dir) + conflicts = repo.index.conflicts + if not conflicts: + print("There are no unresolved conflicts.") + return 0 + + all_success = True + index_changed = False + any_attempted = False + for base, left, right in list(conflicts): + if not base or not left or not right: + # (not left) or (not right): deleted in one branch, modified in the other. + # (not base): added differently in both branches. + # In either case, there's nothing we can do for now. + continue + + path = left.path + if not _applies_to(repo, driver, path): + # Skip the file if it's not the right extension. + continue + + any_attempted = True + driver.pre_announce(path) + io_base = io.BytesIO(repo[base.id].data) + io_left = io.BytesIO(repo[left.id].data) + io_right = io.BytesIO(repo[right.id].data) + success, merge_result = driver.merge(io_base, io_left, io_right) + if merge_result: + # If we got anything, write it to the working directory. + with open(os.path.join(repo.workdir, path), 'wb') as io_output: + driver.to_file(io_output, merge_result) + + if success: + # If we were successful, mark the conflict as resolved. + with open(os.path.join(repo.workdir, path), 'rb') as io_readback: + contents = io_readback.read() + merged_id = repo.create_blob(contents) + repo.index.add(pygit2.IndexEntry(path, merged_id, left.mode)) + del conflicts[path] + index_changed = True + if not success: + all_success = False + driver.post_announce(success, merge_result) + + if index_changed: + repo.index.write() + + if not any_attempted: + print("There are no unresolved", driver.driver_id, "conflicts.") + + if not all_success: + # Not usually observed, but indicate the failure just in case. + return 1 + + +def _applies_to(repo: pygit2.Repository, driver: MergeDriver, path: str): + """ + Check if the current merge driver is a candidate to handle a given path. + """ + if not driver.driver_id: + raise ValueError('Driver must have ID to perform post-hoc merge') + return repo.get_attr(path, 'merge') == driver.driver_id diff --git a/tools/hooks/pre-commit.hook b/tools/hooks/pre-commit.hook index 970be47a44b..8ad5ba32a45 100755 --- a/tools/hooks/pre-commit.hook +++ b/tools/hooks/pre-commit.hook @@ -1,3 +1,2 @@ #!/bin/sh -# `sh` must be used here instead of `bash` to support GitHub Desktop. -exec tools/hooks/python.sh -m precommit +exec tools/bootstrap/python -m mapmerge2.precommit diff --git a/tools/hooks/python.sh b/tools/hooks/python.sh index fc1804d8bc3..60a5efd700d 100755 --- a/tools/hooks/python.sh +++ b/tools/hooks/python.sh @@ -1,32 +1,17 @@ #!/bin/sh -# `sh` must be used here instead of `bash` to support GitHub Desktop. set -e - -# Strip the "App Execution Aliases" from $PATH. Even if the user installed -# Python using the Windows Store on purpose, these aliases always generate -# "Permission denied" errors when sh.exe tries to invoke them. -PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":") - -# Try to find a Python executable. -if command -v python3 >/dev/null 2>&1; then - PY=python3 -elif command -v python >/dev/null 2>&1; then - PY=python -elif command -v py >/dev/null 2>&1; then - PY="py -3" +if [ "$*" = "-m precommit" ]; then + echo "Hooks are being updated..." + echo "Details: https://github.com/tgstation/tgstation/pull/55658" + if [ "$(uname -o)" = "Msys" ]; then + tools/hooks/Install.bat + else + tools/hooks/install.sh + fi + echo "---------------" + exec tools/hooks/pre-commit.hook else - echo "Please install Python from https://www.python.org/downloads/" + echo "tools/hooks/python.sh is replaced by tools/bootstrap/python" + echo "Details: https://github.com/tgstation/tgstation/pull/55658" exit 1 fi - -# Deduce the path separator and add the mapmerge package to the search path. -PATHSEP=$($PY - <<'EOF' -import sys, os -if sys.version_info.major != 3 or sys.version_info.minor < 6: - sys.stderr.write("Python 3.6 or later is required, but you have:\n" + sys.version + "\n") - exit(1) -print(os.pathsep) -EOF -) -export PYTHONPATH=tools/mapmerge2/${PATHSEP}${PYTHONPATH} -exec $PY "$@" diff --git a/tools/makeChangelog.bat b/tools/makeChangelog.bat index f0646caeca0..acdd638b0e2 100644 --- a/tools/makeChangelog.bat +++ b/tools/makeChangelog.bat @@ -1,4 +1,4 @@ @echo off rem Cheridan asked for this. - N3X -call python ss13_genchangelog.py ../html/changelog.html ../html/changelogs -pause \ No newline at end of file +call "%~dp0\bootstrap\python" ss13_genchangelog.py ../html/changelog.html ../html/changelogs +pause diff --git a/tools/mapmerge2/I Forgot To Map Merge.bat b/tools/mapmerge2/I Forgot To Map Merge.bat new file mode 100644 index 00000000000..952b9fe2550 --- /dev/null +++ b/tools/mapmerge2/I Forgot To Map Merge.bat @@ -0,0 +1,7 @@ +@echo off +echo Installing hooks for next time... +call "%~dp0\..\bootstrap\python.bat" -m hooks.install +echo. +echo Fixing things up... +call "%~dp0\..\bootstrap\python.bat" -m mapmerge2.fixup +pause diff --git a/tools/mapmerge2/Prepare Maps.bat b/tools/mapmerge2/Prepare Maps.bat deleted file mode 100644 index 6955e9c877c..00000000000 --- a/tools/mapmerge2/Prepare Maps.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off -cd ../../_maps/ - -for /R %%f in (*.dmm) do copy "%%f" "%%f.backup" - -cls -echo All dmm files in _maps directories have been backed up -echo Now you can make your changes... -echo --- -echo Remember to run mapmerge.bat just before you commit your changes! -echo --- -pause diff --git a/tools/mapmerge2/README.md b/tools/mapmerge2/README.md index 0ff4d21ac2e..71164bdb71f 100644 --- a/tools/mapmerge2/README.md +++ b/tools/mapmerge2/README.md @@ -1,8 +1,27 @@ -# Map Merge 2 +# Map Merger -**Map Merge 2** is an improvement over previous map merging scripts, with -better merge-conflict prevention, multi-Z support, and automatic handling of -key overflow. For up-to-date tips and tricks, also visit the [Map Merger] wiki article. +The **Map Merger** is a collection of scripts that keep this repository's maps +in a format which is easier to track in Git and less likely to cause merge +conflicts. When merge conflicts do occur, it can sometimes resolve them. + +For detailed troubleshooting instructions and other tips, visit the +[Map Merger] wiki article. + +## Installation + +To install the [Git hooks], open the `tools/hooks/` folder and double-click +`Install.bat`. Linux users run `tools/hooks/install.sh`. + +## Manual Use + +If using a Git GUI which is not compatible with the hooks: + +* Before committing, double-click `Run Before Committing.bat` +* When a merge has map conflicts, double-click `Resolve Map Conflicts.bat` + +The console will show whether the operation succeeded. + +For more details, see the [Map Merger] wiki article. ## What Map Merging Is @@ -13,16 +32,8 @@ version of the map while maintaining all the actual changes. It requires an old version of the map to use as a reference and a new version of the map which contains the desired changes. -## Installation - -To install Python dependencies, run `requirements-install.bat`, or run -`python -m pip install -r requirements.txt` directly. See the [Git hooks] -documentation to install the Git pre-commit hook which runs the map merger -automatically, or use `tools/mapmerge/Prepare Maps.bat` to save backups before -running `mapmerge.bat`. - -For up-to-date installation and detailed troubleshooting instructions, visit -the [Map Merger] wiki article. +Map Merge 2 adds multi-Z support, automatic handling of key overflow, better +merge conflict prevention, and a real merge conflict resolver. ## Code Structure diff --git a/tools/mapmerge2/Resolve Map Conflicts.bat b/tools/mapmerge2/Resolve Map Conflicts.bat new file mode 100644 index 00000000000..cd4b95834b2 --- /dev/null +++ b/tools/mapmerge2/Resolve Map Conflicts.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python.bat" -m mapmerge2.merge_driver --posthoc %* +@pause diff --git a/tools/mapmerge2/Run Before Committing.bat b/tools/mapmerge2/Run Before Committing.bat new file mode 100644 index 00000000000..e989edb7b5b --- /dev/null +++ b/tools/mapmerge2/Run Before Committing.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m mapmerge2.precommit --use-workdir %* +@pause diff --git a/tools/mapmerge2/convert.py b/tools/mapmerge2/convert.py index 35e5dda4433..35fa606834c 100644 --- a/tools/mapmerge2/convert.py +++ b/tools/mapmerge2/convert.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import frontend -import dmm +from . import frontend, dmm if __name__ == '__main__': settings = frontend.read_settings() diff --git a/tools/mapmerge2/dmm.py b/tools/mapmerge2/dmm.py index 20b45215208..2d87cfc1724 100644 --- a/tools/mapmerge2/dmm.py +++ b/tools/mapmerge2/dmm.py @@ -22,20 +22,19 @@ class DMM: @staticmethod def from_file(fname): - # stream the file rather than forcing all its contents to memory with open(fname, 'r', encoding=ENCODING) as f: - return _parse(iter(lambda: f.read(1), '')) + return _parse(f.read()) @staticmethod def from_bytes(bytes): return _parse(bytes.decode(ENCODING)) - def to_file(self, fname, tgm = True): + def to_file(self, fname, *, tgm = True): self._presave_checks() with open(fname, 'w', newline='\n', encoding=ENCODING) as f: (save_tgm if tgm else save_dmm)(self, f) - def to_bytes(self, tgm = True): + def to_bytes(self, *, tgm = True): self._presave_checks() bio = io.BytesIO() with io.TextIOWrapper(bio, newline='\n', encoding=ENCODING) as f: @@ -43,6 +42,21 @@ class DMM: f.flush() return bio.getvalue() + def get_or_generate_key(self, tile): + try: + return self.dictionary.inv[tile] + except KeyError: + key = self.generate_new_key() + self.dictionary[key] = tile + return key + + def get_tile(self, coord): + return self.dictionary[self.grid[coord]] + + def set_tile(self, coord, tile): + tile = tuple(tile) + self.grid[coord] = self.get_or_generate_key(tile) + def generate_new_key(self): free_keys = self._ensure_free_keys(1) max_key = max_key_for(self.key_length) @@ -119,6 +133,9 @@ class DMM: for x in range(1, self.size.x + 1): yield (y, x) + def __repr__(self): + return f"DMM(size={self.size}, key_length={self.key_length}, dictionary_size={len(self.dictionary)})" + # ---------- # key handling @@ -221,10 +238,8 @@ def is_bad_atom_ordering(key, atoms): print(f"Warning: key '{key}' is missing either a turf or area") return can_fix -def fix_atom_ordering(atoms): - movables = [] - turfs = [] - areas = [] +def split_atom_groups(atoms): + movables, turfs, areas = [], [], [] for each in atoms: if each.startswith('/turf'): turfs.append(each) @@ -232,6 +247,10 @@ def fix_atom_ordering(atoms): areas.append(each) else: movables.append(each) + return movables, turfs, areas + +def fix_atom_ordering(atoms): + movables, turfs, areas = split_atom_groups(atoms) movables.extend(turfs) movables.extend(areas) return movables @@ -281,7 +300,7 @@ def save_tgm(dmm, output): output.write("\n") for x in range(1, max_x + 1): output.write(f"({x},{1},{z}) = {{\"\n") - for y in range(1, max_y + 1): + for y in range(max_y, 0, -1): output.write(f"{num_to_key(dmm.grid[x, y, z], dmm.key_length)}\n") output.write("\"}\n") @@ -303,7 +322,7 @@ def save_dmm(dmm, output): for z in range(1, max_z + 1): output.write(f"(1,1,{z}) = {{\"\n") - for y in range(1, max_y + 1): + for y in range(max_y, 0, -1): for x in range(1, max_x + 1): try: output.write(num_to_key(dmm.grid[x, y, z], dmm.key_length)) @@ -536,7 +555,17 @@ def _parse(map_raw_text): if curr_y > maxy: maxy = curr_y + if not grid: + # Usually caused by unbalanced quotes. + max_key = num_to_key(max(dictionary.keys()), key_length, True) + raise ValueError(f"dmm failed to parse, check for a syntax error near or after key {max_key!r}") + + # Convert from raw .dmm coordinates to DM/BYOND coordinates by flipping Y + grid2 = dict() + for (x, y, z), tile in grid.items(): + grid2[x, maxy + 1 - y, z] = tile + data = DMM(key_length, Coordinate(maxx, maxy, maxz)) data.dictionary = dictionary - data.grid = grid + data.grid = grid2 return data diff --git a/tools/mapmerge2/dmm2tgm.bat b/tools/mapmerge2/dmm2tgm.bat deleted file mode 100644 index bcf6150c2ea..00000000000 --- a/tools/mapmerge2/dmm2tgm.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -set MAPROOT=../../_maps/ -set TGM=1 -python convert.py -pause diff --git a/tools/mapmerge2/dmm_test.py b/tools/mapmerge2/dmm_test.py new file mode 100644 index 00000000000..02f5383f2ab --- /dev/null +++ b/tools/mapmerge2/dmm_test.py @@ -0,0 +1,39 @@ +import os +import sys +from .dmm import * + + +def _self_test(): + # test: can we load every DMM in the tree + count = 0 + for dirpath, dirnames, filenames in os.walk('.'): + if '.git' in dirnames: + dirnames.remove('.git') + for filename in filenames: + if filename.endswith('.dmm'): + fullpath = os.path.join(dirpath, filename) + try: + DMM.from_file(fullpath) + except Exception: + print('Failed on:', fullpath) + raise + count += 1 + + print(f"{os.path.relpath(__file__)}: successfully parsed {count} .dmm files") + + +def _usage(): + print(f"Usage:") + print(f" tools{os.sep}bootstrap{os.sep}python -m {__spec__.name}") + exit(1) + + +def _main(): + if len(sys.argv) == 1: + return _self_test() + + return _usage() + + +if __name__ == '__main__': + _main() diff --git a/tools/mapmerge2/fixup.py b/tools/mapmerge2/fixup.py new file mode 100644 index 00000000000..f3133d1d730 --- /dev/null +++ b/tools/mapmerge2/fixup.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +import os +import pygit2 +from . import dmm +from .mapmerge import merge_map + + +STATUS_INDEX = (pygit2.GIT_STATUS_INDEX_NEW + | pygit2.GIT_STATUS_INDEX_MODIFIED + | pygit2.GIT_STATUS_INDEX_DELETED + | pygit2.GIT_STATUS_INDEX_RENAMED + | pygit2.GIT_STATUS_INDEX_TYPECHANGE +) +STATUS_WT = (pygit2.GIT_STATUS_WT_NEW + | pygit2.GIT_STATUS_WT_MODIFIED + | pygit2.GIT_STATUS_WT_DELETED + | pygit2.GIT_STATUS_WT_RENAMED + | pygit2.GIT_STATUS_WT_TYPECHANGE +) +ABBREV_LEN = 12 +TGM_HEADER = dmm.TGM_HEADER.encode(dmm.ENCODING) + + +def walk_tree(tree, *, _prefix=''): + for child in tree: + if isinstance(child, pygit2.Tree): + yield from walk_tree(child, _prefix=f'{_prefix}{child.name}/') + else: + yield f'{_prefix}{child.name}', child + + +def insert_into_tree(repo, tree_builder, path, blob_oid): + try: + first, rest = path.split('/', 1) + except ValueError: + tree_builder.insert(path, blob_oid, pygit2.GIT_FILEMODE_BLOB) + else: + inner = repo.TreeBuilder(tree_builder.get(first)) + insert_into_tree(repo, inner, rest, blob_oid) + tree_builder.insert(first, inner.write(), pygit2.GIT_FILEMODE_TREE) + + +def main(repo): + if repo.index.conflicts: + print("You need to resolve merge conflicts first.") + return 1 + + # Ensure the index is clean. + for path, status in repo.status().items(): + if status & pygit2.GIT_STATUS_IGNORED: + continue + if status & STATUS_INDEX: + print("You have changes staged for commit. Commit them or unstage them first.") + print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.") + return 1 + if path.endswith(".dmm") and (status & STATUS_WT): + print("You have modified maps. Commit them first.") + print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.") + return 1 + + # Read the HEAD commit. + head_commit = repo[repo.head.target] + head_files = {} + for path, blob in walk_tree(head_commit.tree): + if path.endswith(".dmm"): + data = blob.read_raw() + if not data.startswith(TGM_HEADER): + head_files[path] = dmm.DMM.from_bytes(data) + + if not head_files: + print("All committed maps appear to be in the correct format.") + print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.") + return 1 + + # Work backwards to find a base for each map, converting as found. + converted = {} + if len(head_commit.parents) != 1: + print("Unable to automatically fix anything because HEAD is a merge commit.") + return 1 + commit_message_lines = [] + working_commit = head_commit.parents[0] + while len(converted) < len(head_files): + for path in head_files.keys() - converted.keys(): + try: + blob = working_commit.tree[path] + except KeyError: + commit_message_lines.append(f"{'new':{ABBREV_LEN}}: {path}") + print(f"Converting new map: {path}") + converted[path] = head_files[path] + else: + data = blob.read_raw() + if data.startswith(TGM_HEADER): + str_id = str(working_commit.id)[:ABBREV_LEN] + commit_message_lines.append(f"{str_id}: {path}") + print(f"Converting map: {path}") + converted[path] = merge_map(head_files[path], dmm.DMM.from_bytes(data)) + if len(working_commit.parents) != 1: + print("A merge commit was encountered before good versions of these maps were found:") + print("\n".join(f" {x}" for x in head_files.keys() - base_files.keys())) + return 1 + working_commit = working_commit.parents[0] + + # Okay, do the actual work. + tree_builder = repo.TreeBuilder(head_commit.tree) + for path, merged_map in converted.items(): + blob_oid = repo.create_blob(merged_map.to_bytes()) + insert_into_tree(repo, tree_builder, path, blob_oid) + repo.index.add(pygit2.IndexEntry(path, blob_oid, repo.index[path].mode)) + merged_map.to_file(os.path.join(repo.workdir, path)) + + # Save the index. + repo.index.write() + + # Commit the index to the current branch. + signature = pygit2.Signature(repo.config['user.name'], repo.config['user.email']) + joined = "\n".join(commit_message_lines) + repo.create_commit( + repo.head.name, + signature, # author + signature, # committer + f'Convert maps to TGM\n\n{joined}\n\nAutomatically commited by: {os.path.relpath(__file__, repo.workdir)}', + tree_builder.write(), + [head_commit.id], + ) + + # Success. + print("Successfully committed a fixup. Push as needed.") + return 0 + + +if __name__ == '__main__': + exit(main(pygit2.Repository(pygit2.discover_repository(os.getcwd())))) diff --git a/tools/mapmerge2/mapmerge.bat b/tools/mapmerge2/mapmerge.bat deleted file mode 100644 index 5a066226b3c..00000000000 --- a/tools/mapmerge2/mapmerge.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -set MAPROOT=../../_maps/ -set TGM=1 -python mapmerge.py -pause diff --git a/tools/mapmerge2/mapmerge.py b/tools/mapmerge2/mapmerge.py index f449bd948f0..d9cdea24cfb 100644 --- a/tools/mapmerge2/mapmerge.py +++ b/tools/mapmerge2/mapmerge.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -import frontend import shutil -from dmm import * from collections import defaultdict +from . import frontend +from .dmm import * def merge_map(new_map, old_map, delete_unused=False): if new_map.key_length != old_map.key_length: @@ -66,7 +66,7 @@ def merge_map(new_map, old_map, delete_unused=False): # step two: delete unused keys if unused_keys: - print(f"Notice: Trimming {len(unused_keys)} unused dictionary keys.") + #print(f"Notice: Trimming {len(unused_keys)} unused dictionary keys.") for key in unused_keys: del merged.dictionary[key] diff --git a/tools/mapmerge2/merge_driver.py b/tools/mapmerge2/merge_driver.py new file mode 100644 index 00000000000..39bc1f723e3 --- /dev/null +++ b/tools/mapmerge2/merge_driver.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +import sys +import collections +from . import dmm, mapmerge +from hooks.merge_frontend import MergeDriver + + +debug_stats = collections.defaultdict(int) + + +def select(base, left, right, *, debug=None): + if left == right: + # whether or not it's in the base, both sides agree + if debug: + debug_stats[f"select {debug} both"] += 1 + return left + elif base == left: + # base == left, but right is different: accept right + if debug: + debug_stats[f"select {debug} right"] += 1 + return right + elif base == right: + # base == right, but left is different: accept left + if debug: + debug_stats[f"select {debug} left"] += 1 + return left + else: + # all three versions are different + if debug: + debug_stats[f"select {debug} fail"] += 1 + return None + + +def three_way_merge(base, left, right): + if base.size != left.size or base.size != right.size: + print("Dimensions have changed:") + print(f" Base: {base.size}") + print(f" Ours: {left.size}") + print(f" Theirs: {right.size}") + return True, None + + trouble = False + merged = dmm.DMM(base.key_length, base.size) + merged.dictionary = base.dictionary.copy() + + for (z, y, x) in base.coords_zyx: + coord = x, y, z + base_tile = base.get_tile(coord) + left_tile = left.get_tile(coord) + right_tile = right.get_tile(coord) + + # try to merge the whole tiles + whole_tile_merge = select(base_tile, left_tile, right_tile, debug='tile') + if whole_tile_merge is not None: + merged.set_tile(coord, whole_tile_merge) + continue + + # try to merge each group independently (movables, turfs, areas) + base_movables, base_turfs, base_areas = dmm.split_atom_groups(base_tile) + left_movables, left_turfs, left_areas = dmm.split_atom_groups(left_tile) + right_movables, right_turfs, right_areas = dmm.split_atom_groups(right_tile) + + merged_movables = select(base_movables, left_movables, right_movables, debug='movable') + merged_turfs = select(base_turfs, left_turfs, right_turfs, debug='turf') + merged_areas = select(base_areas, left_areas, right_areas, debug='area') + + if merged_movables is not None and merged_turfs is not None and merged_areas is not None: + merged.set_tile(coord, merged_movables + merged_turfs + merged_areas) + continue + + # TODO: more advanced strategies? + + # fall back to requiring manual conflict resolution + trouble = True + print(f" C: Both sides touch the tile at {coord}") + + if merged_movables is None: + obj_name = "---Merge conflict marker---" + merged_movables = left_movables + [f'/obj{{name = "{obj_name}"}}'] + right_movables + print(f" Left and right movable groups are split by an `/obj` named \"{obj_name}\"") + if merged_turfs is None: + merged_turfs = left_turfs + print(f" Saving turf: {', '.join(left_turfs)}") + print(f" Alternative: {', '.join(right_turfs)}") + print(f" Original: {', '.join(base_turfs)}") + if merged_areas is None: + merged_areas = left_areas + print(f" Saving area: {', '.join(left_areas)}") + print(f" Alternative: {', '.join(right_areas)}") + print(f" Original: {', '.join(base_areas)}") + + merged.set_tile(coord, merged_movables + merged_turfs + merged_areas) + + merged = mapmerge.merge_map(merged, base) + return trouble, merged + + +class DmmDriver(MergeDriver): + driver_id = 'dmm' + + def merge(self, base, left, right): + map_base = dmm.DMM.from_bytes(base.read()) + map_left = dmm.DMM.from_bytes(left.read()) + map_right = dmm.DMM.from_bytes(right.read()) + trouble, merge_result = three_way_merge(map_base, map_left, map_right) + return not trouble, merge_result + + def to_file(self, outfile, merge_result): + outfile.write(merge_result.to_bytes()) + + def post_announce(self, success, merge_result): + if not success: + print("!!! Manual merge required!") + if merge_result: + print(" A best-effort merge was performed. You must edit the map and confirm") + print(" that all coordinates mentioned above are as desired.") + else: + print(" The map was totally unable to be merged; you must start with one version") + print(" or the other and manually resolve the conflict. Information about the") + print(" conflicting tiles is listed above.") + + +if __name__ == '__main__': + exit(DmmDriver().main()) diff --git a/tools/mapmerge2/precommit.py b/tools/mapmerge2/precommit.py index f5ea49a50ea..3d22917f92c 100644 --- a/tools/mapmerge2/precommit.py +++ b/tools/mapmerge2/precommit.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import os +import sys import pygit2 -import dmm -from mapmerge import merge_map +from . import dmm +from .mapmerge import merge_map -def main(repo): + +def main(repo, *, use_workdir=False): if repo.index.conflicts: print("You need to resolve merge conflicts first.") return 1 @@ -16,12 +18,21 @@ def main(repo): except KeyError: pass + target_statuses = pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_INDEX_NEW + skip_to_file_statuses = pygit2.GIT_STATUS_WT_DELETED | pygit2.GIT_STATUS_WT_MODIFIED + if use_workdir: + target_statuses |= pygit2.GIT_STATUS_WT_MODIFIED | pygit2.GIT_STATUS_WT_NEW + skip_to_file_statuses &= ~pygit2.GIT_STATUS_WT_MODIFIED + changed = 0 for path, status in repo.status().items(): - if path.endswith(".dmm") and (status & (pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_INDEX_NEW)): + if path.endswith(".dmm") and (status & target_statuses): # read the index index_entry = repo.index[path] - index_map = dmm.DMM.from_bytes(repo[index_entry.id].read_raw()) + if use_workdir: + index_map = dmm.DMM.from_file(os.path.join(repo.workdir, path)) + else: + index_map = dmm.DMM.from_bytes(repo[index_entry.id].read_raw()) try: head_blob = repo[repo[repo.head.target].tree[path].id] @@ -32,7 +43,7 @@ def main(repo): merged_map = index_map else: # Entry in HEAD, merge the index over it - print(f"Merging map: {path}", flush=True) + print(f"Converting map: {path}", flush=True) assert not (status & pygit2.GIT_STATUS_INDEX_NEW) head_map = dmm.DMM.from_bytes(head_blob.read_raw()) merged_map = merge_map(index_map, head_map) @@ -43,15 +54,16 @@ def main(repo): changed += 1 # write to the working directory if that's clean - if status & (pygit2.GIT_STATUS_WT_DELETED | pygit2.GIT_STATUS_WT_MODIFIED): + if status & skip_to_file_statuses: print(f"Warning: {path} has unindexed changes, not overwriting them") else: merged_map.to_file(os.path.join(repo.workdir, path)) if changed: repo.index.write() - print(f"Merged {changed} maps.") return 0 + if __name__ == '__main__': - exit(main(pygit2.Repository(pygit2.discover_repository(os.getcwd())))) + repo = pygit2.Repository(pygit2.discover_repository(os.getcwd())) + exit(main(repo, use_workdir='--use-workdir' in sys.argv)) diff --git a/tools/mapmerge2/requirements-install.bat b/tools/mapmerge2/requirements-install.bat deleted file mode 100644 index 71341af8792..00000000000 --- a/tools/mapmerge2/requirements-install.bat +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -python -m pip install -r requirements.txt -pause diff --git a/tools/mapmerge2/requirements.txt b/tools/mapmerge2/requirements.txt deleted file mode 100644 index cb06e94ee55..00000000000 --- a/tools/mapmerge2/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pygit2==1.0.1 -bidict==0.13.1 -Pillow==7.2.0 diff --git a/tools/mapmerge2/tgm2dmm.bat b/tools/mapmerge2/tgm2dmm.bat deleted file mode 100644 index 2748533febf..00000000000 --- a/tools/mapmerge2/tgm2dmm.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -set MAPROOT=../../_maps/ -set TGM=0 -python convert.py -pause diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..cf8552635d1 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,7 @@ +pygit2==1.0.1 +bidict==0.13.1 +Pillow==7.2.0 + +# changelogs +PyYaml==5.3.1 +beautifulsoup4==4.9.3