[MANUAL MIRROR] Use portable Python for map merge hooks, other tools / Add .dmm merge driver (#2681)

* Use portable Python for map merge hooks, other tools (#55658)

* Add .dmm merge driver (#55699)

This is kind of a prototype. It only fully handles a few situations, 
and doesn't produce particularly easy-to-read conflict markers when it 
fails. I hope that it can be useful at least some of the time, can be 
improved over time, and that the lessons learned can influence a future 
interactive GUI conflict resolver (integrated into StrongDMM?). In the 
worst case, one can fall back to the tried and true "manually re-do one 
side's changes" strategy. 

**Automatic use**: In `tools/hooks/`, run `Install.bat`

**Manual use**, for Git GUIs that don't run merge drivers: while a 
merge is in progress, in `tools/mapmerge2/`, double-click `Resolve Map 
Conflicts.bat`

This PR also removes the error-prone "Prepare Maps.bat" / 
"mapmerge.bat" workflow. Those who aren't using the hooks should 
instead use `Run Before Committing.bat` before committing. First-time 
contributors who opened a PR without map merging can be advised to run 
`I Forgot To Map Merge.bat`.

* Fix loose double-quot in tradership_faction.dmm

Co-authored-by: Tad Hardesty <tad@platymuus.com>

.github/workflows/ci_suite.yml

@@ -22,7 +22,7 @@ jobs:
           pip3 install setuptools
           bash tools/ci/install_build_tools.sh
           bash tools/ci/install_spaceman_dmm.sh dreamchecker
-          pip3 install -r tools/mapmerge2/requirements.txt
+          pip3 install -r tools/requirements.txt
       - name: Run Linters
         run: |
           bash tools/ci/check_filedirs.sh tgstation.dme
@@ -31,7 +31,8 @@ jobs:
           find . -name "*.json" -not -path "*/node_modules/*" -print0 | xargs -0 python3 ./tools/json_verifier.py
           bash tools/ci/build_tgui.sh
           bash tools/ci/check_grep.sh
-          python3 tools/mapmerge2/dmi.py --test
+          tools/bootstrap/python -m dmi.test
+          tools/bootstrap/python -m mapmerge2.dmm_test
           ~/dreamchecker > ${GITHUB_WORKSPACE}/output-annotations.txt 2>&1
       - name: Annotate Lints
         uses: yogstation13/DreamAnnotate@v1

dependencies.sh

@@ -1,13 +1,13 @@
-#!/bin/bash
+#!/bin/sh
 
 #Project dependencies file
 #Final authority on what's required to fully build the project
 
 # byond version
 # Extracted from the Dockerfile. Change by editing Dockerfile's FROM command.
-LIST=($(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile))
-export BYOND_MAJOR=${LIST[0]}
-export BYOND_MINOR=${LIST[1]}
+LIST="$(sed -n 's/.*byond:\([0-9]\+\)\.\([0-9]\+\).*/\1 \2/p' Dockerfile)"
+export BYOND_MAJOR=${LIST% *}
+export BYOND_MINOR=${LIST#* }
 unset LIST
 
 #rust_g git tag
@@ -21,3 +21,6 @@ export SPACEMAN_DMM_VERSION=suite-1.6
 
 # Extools git tag
 export EXTOOLS_VERSION=v0.0.7
+
+# Python version for mapmerge and other tools
+export PYTHON_VERSION=3.6.8

modular_skyrat/modules/faction/maps/shuttles/tradership_faction.dmm

@@ -2648,7 +2648,7 @@
 	name = "Starboard Blast Door Control";
 	pixel_x = -24;
 	pixel_y = -5;
-	req_access_txt = "403""
+	req_access_txt = "403"
 	},
 /obj/structure/closet/crate/secure/tradership_cargo_valuable{
 	used_preset = 2

tools/HitboxExpander/.gitignore

@@ -1,3 +0,0 @@
-Imaging-1.1.7/
-zlib/
-

tools/HitboxExpander/Hitbox Expander.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python" -m HitboxExpander %*
+@pause

tools/HitboxExpander/README.txt

@@ -1,10 +1,4 @@
-Setup: Install python3 and run install.bat for windows, install.sh for unix.
-Alternatively, you can manually install the Pillow package with
-```
-pip install Pillow
-```
-
-Usage: python hitbox_expander.py <path_to_file.dmi or png>
+Usage: tools/bootstrap/python -m HitboxExpander <path_to_file.dmi or png>
 
 This tool expands the hitbox of the given image by 1 pixel.
 Works by changing some of the fully-transparent pixels to alpha=1 black pixels.

tools/HitboxExpander/__main__.py

@@ -74,7 +74,10 @@ icons_dir = os.path.join(root_dir, "icons")
 
 def Main():
   if len(sys.argv) != 2:
-    print("Usage: hitbox_expander.py filename.dmi")
+    if os.name == 'nt':
+      print("Usage: drag-and-drop a .dmi onto `Hitbox Expander.bat`\n  or")
+    with open(os.path.join(current_dir, "README.txt")) as f:
+      print(f.read())
     return 0
 
   try:

tools/HitboxExpander/hitbox_expander.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -e
+exec "$(dirname "$0")/../bootstrap/python" -m HitboxExpander "$@"

tools/HitboxExpander/install.bat

@@ -1 +0,0 @@
-python3 -m pip install -r requirements.txt

tools/HitboxExpander/install.sh

@@ -1 +0,0 @@
-python3 -m pip install -r requirements.txt

tools/HitboxExpander/requirements.txt

@@ -1 +0,0 @@
-Pillow==7.2.0

tools/UpdatePaths/Update Paths.bat

@@ -0,0 +1,3 @@
+@echo off
+call "%~dp0\..\bootstrap\python" -m UpdatePaths %*
+pause

tools/UpdatePaths/__main__.py

@@ -1,9 +1,10 @@
 # A script and syntax for applying path updates to maps.
 import re
 import os
+import sys
 import argparse
-import frontend
-from dmm import *
+from mapmerge2 import frontend
+from mapmerge2.dmm import *
 
 desc = """
 Update dmm files given update file/string.
@@ -167,7 +168,11 @@ def main(args):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter)
+    prog = __spec__.name.replace('.__main__', '')
+    if os.name == 'nt' and len(sys.argv) <= 1:
+        print("usage: drag-and-drop a path script .txt onto `Update Paths.bat`\n  or")
+
+    parser = argparse.ArgumentParser(prog=prog, description=desc, formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument("update_source", help="update file path / line of update notation")
     parser.add_argument("--map", "-m", help="path to update, defaults to all maps in maps directory")
     parser.add_argument("--directory", "-d", help="path to maps directory, defaults to _maps/")

tools/bootstrap/python

@@ -0,0 +1,117 @@
+#!/bin/sh
+# bootstrap/python
+#
+# Python-finding script for all `sh` environments, including Linux, MSYS2,
+# Git for Windows, and GitHub Desktop. Invokable from CLI or automation.
+#
+# If a python.exe installed by `python_.ps1` is present, it will be used.
+# Otherwise, this script requires a system `python3` and `pip` to be provided,
+# and will create a standard virtualenv in which to install `requirements.txt`.
+set -e
+
+# Convenience variables
+Bootstrap="$(dirname "$0")"
+Sdk="$(dirname "$Bootstrap")"
+Cache="$Bootstrap/.cache"
+if [ "$TG_BOOTSTRAP_CACHE" ]; then
+	Cache="$TG_BOOTSTRAP_CACHE"
+fi
+OldPWD="$PWD"
+cd "$Bootstrap/../.."
+. ./dependencies.sh  # sets PYTHON_VERSION
+cd "$OldPWD"
+PythonVersion="$PYTHON_VERSION"
+PythonDir="$Cache/python-$PythonVersion"
+PythonExe="$PythonDir/python.exe"
+Log="$Cache/last-command.log"
+
+# If a portable Python for Windows is not present, search on $PATH.
+if [ "$(uname)" = "Linux" ] || [ ! -f "$PythonExe" ]; then
+	# Strip the "App Execution Aliases" from $PATH. Even if the user installed
+	# Python using the Windows Store on purpose, these aliases always generate
+	# "Permission denied" errors when sh.exe tries to invoke them.
+	PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":")
+
+	# Try to find a Python executable.
+	if command -v python3 >/dev/null 2>&1; then
+		PythonExe=python3
+	elif command -v python >/dev/null 2>&1; then
+		PythonExe=python
+	elif command -v py >/dev/null 2>&1; then
+		PythonExe="py -3"
+	else
+		echo
+		if command -v apt-get >/dev/null 2>&1; then
+			echo "Please install Python using your system's package manager:"
+			echo "    sudo apt-get install python3 python3-pip"
+		elif [ "$(uname -o)" = "Msys" ]; then
+			echo "Please run tools/bootstrap/python.bat instead of tools/bootstrap/python once to"
+			echo "install Python automatically, or install it from https://www.python.org/downloads/"
+			# TODO: give MSYS pacman advice?
+		elif command -v pacman >/dev/null 2>&1; then
+			echo "Please install Python using your system's package manager:"
+			echo "    sudo pacman -S python python-pip"
+		else
+			echo "Please install Python from https://www.python.org/downloads/ or using your system's package manager."
+		fi
+		echo
+		exit 1
+	fi
+
+	# Create a venv and activate it
+	PythonDir="$Cache/venv"
+	if [ ! -d "$PythonDir" ]; then
+		echo "Creating virtualenv..."
+		"$PythonExe" -m venv "$PythonDir"
+	fi
+	if [ -f "$PythonDir/bin/python" ]; then
+		PythonExe="$PythonDir/bin/python"
+	elif [ -f "$PythonDir/scripts/python3.exe" ]; then
+		PythonExe="$PythonDir/scripts/python3.exe";
+	else
+		echo "bootstrap/python failed to find the python executable inside its virtualenv"
+		exit 1
+	fi
+fi
+
+# Use pip to install our requirements
+if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then
+	echo "Updating dependencies..."
+	"$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt"
+	cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt"
+	echo "---"
+fi
+
+# Verify version and deduce the path separator
+PythonMajor=${PythonVersion%%.*}
+PythonMinor=${PythonVersion#*.}
+PythonMinor=${PythonMinor%.*}
+PATHSEP=$("$PythonExe" - "$PythonMajor" "$PythonMinor" <<'EOF'
+import sys, os
+if sys.version_info.major != int(sys.argv[1]) or sys.version_info.minor < int(sys.argv[2]):
+	print("Error: Python ", sys.argv[1], ".", sys.argv[2], " or later is required, but you have:\n", sys.version, sep="", file=sys.stderr)
+	exit(1)
+print(os.pathsep)
+EOF
+)
+
+# Cheap shell function if tee.exe is not available
+if ! command -v tee >/dev/null 2>&1; then
+	tee() {
+		# Fudge: assume $1 is always "-a"
+		while read -r line; do
+			echo "$line" >> "$2"
+			echo "$line"
+		done
+	}
+fi
+
+# Invoke python with all command-line arguments
+export PYTHONPATH="$Sdk$PATHSEP${PYTHONPATH:-}"
+mkdir -p "$Cache"
+printf '%s\n' "$PythonExe" "$@" > "$Log"
+printf -- '---\n' >> "$Log"
+exec 4>&1
+exitstatus=$({ { set +e; "$PythonExe" -u "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1)
+exec 4>&-
+exit "$exitstatus"

tools/bootstrap/python.bat

@@ -0,0 +1 @@
+@call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\python_.ps1" %*

tools/bootstrap/python36._pth

@@ -0,0 +1,6 @@
+python36.zip
+.
+..\..\..
+
+# Uncomment to run site.main() automatically
+import site

tools/bootstrap/python_.ps1

@@ -0,0 +1,103 @@
+# bootstrap/python_.ps1
+#
+# Python bootstrapping script for Windows.
+#
+# Automatically downloads a portable edition of a pinned Python version to
+# a cache directory, installs Pip, installs `requirements.txt`, and then invokes
+# Python.
+#
+# The underscore in the name is so that typing `bootstrap/python` into
+# PowerShell finds the `.bat` file first, which ensures this script executes
+# regardless of ExecutionPolicy.
+$host.ui.RawUI.WindowTitle = "starting :: python $args"
+$ErrorActionPreference = "Stop"
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+function ExtractVersion {
+	param([string] $Path, [string] $Key)
+	foreach ($Line in Get-Content $Path) {
+		if ($Line.StartsWith("export $Key=")) {
+			return $Line.Substring("export $Key=".Length)
+		}
+	}
+	throw "Couldn't find value for $Key in $Path"
+}
+
+# Convenience variables
+$Bootstrap = Split-Path $script:MyInvocation.MyCommand.Path
+$Tools = Split-Path $Bootstrap
+$Cache = "$Bootstrap/.cache"
+if ($Env:TG_BOOTSTRAP_CACHE) {
+	$Cache = $Env:TG_BOOTSTRAP_CACHE
+}
+$PythonVersion = ExtractVersion -Path "$Bootstrap/../../dependencies.sh" -Key "PYTHON_VERSION"
+$PythonDir = "$Cache/python-$PythonVersion"
+$PythonExe = "$PythonDir/python.exe"
+$Log = "$Cache/last-command.log"
+
+# Download and unzip a portable version of Python
+if (!(Test-Path $PythonExe -PathType Leaf)) {
+	$host.ui.RawUI.WindowTitle = "Downloading Python $PythonVersion..."
+	New-Item $Cache -ItemType Directory -ErrorAction silentlyContinue | Out-Null
+
+	$Archive = "$Cache/python-$PythonVersion-embed.zip"
+	Invoke-WebRequest `
+		"https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" `
+		-OutFile $Archive `
+		-ErrorAction Stop
+
+	[System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir)
+
+	# Copy a ._pth file without "import site" commented, so pip will work
+	Copy-Item "$Bootstrap/python36._pth" $PythonDir `
+		-ErrorAction Stop
+
+	Remove-Item $Archive
+}
+
+# Install pip
+if (!(Test-Path "$PythonDir/Scripts/pip.exe")) {
+	$host.ui.RawUI.WindowTitle = "Downloading Pip..."
+
+	Invoke-WebRequest "https://bootstrap.pypa.io/get-pip.py" `
+		-OutFile "$Cache/get-pip.py" `
+		-ErrorAction Stop
+
+	& $PythonExe "$Cache/get-pip.py" --no-warn-script-location
+	if ($LASTEXITCODE -ne 0) {
+		exit $LASTEXITCODE
+	}
+
+	Remove-Item "$Cache/get-pip.py" `
+		-ErrorAction Stop
+}
+
+# Use pip to install our requirements
+if (!(Test-Path "$PythonDir/requirements.txt") -or ((Get-FileHash "$Tools/requirements.txt").hash -ne (Get-FileHash "$PythonDir/requirements.txt").hash)) {
+	$host.ui.RawUI.WindowTitle = "Updating dependencies..."
+
+	& $PythonExe -m pip install -U pip -r "$Tools/requirements.txt"
+	if ($LASTEXITCODE -ne 0) {
+		exit $LASTEXITCODE
+	}
+
+	Copy-Item "$Tools/requirements.txt" "$PythonDir/requirements.txt"
+	Write-Output "`n---`n"
+}
+
+# Invoke python with all command-line arguments
+Write-Output $PythonExe | Out-File -Encoding utf8 $Log
+[System.String]::Join([System.Environment]::NewLine, $args) | Out-File -Encoding utf8 -Append $Log
+Write-Output "---" | Out-File -Encoding utf8 -Append $Log
+$host.ui.RawUI.WindowTitle = "python $args"
+$ErrorActionPreference = "Continue"
+& $PythonExe -u $args 2>&1 | ForEach-Object {
+	$str = "$_"
+	if ($_.GetType() -eq [System.Management.Automation.ErrorRecord]) {
+		$str = $str.TrimEnd("`r`n")
+	}
+	$str | Out-File -Encoding utf8 -Append $Log
+	$str | Out-Host
+}
+exit $LastExitCode

tools/dmi/Resolve Icon Conflicts.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python.bat" -m dmi.merge_driver --posthoc %*
+@pause

tools/dmi/__init__.py

@@ -12,10 +12,10 @@ NORTH = 1
 SOUTH = 2
 EAST = 4
 WEST = 8
-SOUTHEAST = SOUTH|EAST
-SOUTHWEST = SOUTH|WEST
-NORTHEAST = NORTH|EAST
-NORTHWEST = NORTH|WEST
+SOUTHEAST = SOUTH | EAST
+SOUTHWEST = SOUTH | WEST
+NORTHEAST = NORTH | EAST
+NORTHWEST = NORTH | WEST
 
 CARDINALS = [NORTH, SOUTH, EAST, WEST]
 DIR_ORDER = [SOUTH, NORTH, EAST, WEST, SOUTHEAST, SOUTHWEST, NORTHEAST, NORTHWEST]
@@ -34,6 +34,7 @@ DIR_NAMES = {
     None: SOUTH,
 }
 
+
 class Dmi:
     version = "4.0"
 
@@ -134,7 +135,7 @@ class Dmi:
             comment += f"state = {escape(state.name)}\n"
             comment += f"\tdirs = {state.dirs}\n"
             comment += f"\tframes = {state.framecount}\n"
-            if state.framecount > 1 and len(state.delays): #any(x != 1 for x in state.delays):
+            if state.framecount > 1 and len(state.delays):  # any(x != 1 for x in state.delays):
                 comment += "\tdelay = " + ",".join(map(str, state.delays)) + "\n"
             if state.loop != 0:
                 comment += f"\tloop = {state.loop}\n"
@@ -175,6 +176,7 @@ class Dmi:
             output = output.convert('P')
         output.save(filename, 'png', optimize=True, pnginfo=pnginfo)
 
+
 class State:
     def __init__(self, dmi, name, *, loop=LOOP_UNLIMITED, rewind=False, movement=False, dirs=1):
         self.dmi = dmi
@@ -216,11 +218,13 @@ class State:
     def get_frame(self, *args, **kwargs):
         return self.frames[self._frame_index(*args, **kwargs)]
 
+
 def escape(text):
     text = text.replace('\\', '\\\\')
     text = text.replace('"', '\\"')
     return f'"{text}"'
 
+
 def unescape(text, quote='"'):
     if text == 'null':
         return None
@@ -231,51 +235,14 @@ def unescape(text, quote='"'):
     text = text.replace('\\\\', '\\')
     return text
 
+
 def parse_num(value):
     if '.' in value:
         return float(value)
     return int(value)
 
+
 def parse_bool(value):
     if value not in ('0', '1'):
         raise ValueError(value)
     return value == '1'
-
-def _self_test():
-    # test: can we load every DMI in the tree
-    import os
-
-    count = 0
-    for dirpath, dirnames, filenames in os.walk('.'):
-        if '.git' in dirnames:
-            dirnames.remove('.git')
-        for filename in filenames:
-            if filename.endswith('.dmi'):
-                fullpath = os.path.join(dirpath, filename)
-                try:
-                    Dmi.from_file(fullpath)
-                except:
-                    print('Failed on:', fullpath)
-                    raise
-                count += 1
-
-    print(f"Successfully parsed {count} dmi files")
-
-def _usage():
-    import sys
-    print(f"Usage:")
-    print(f"    {sys.argv[0]} --test")
-    exit(1)
-
-def _main():
-    import sys
-    if len(sys.argv) < 2:
-        return _usage()
-
-    if sys.argv[1] == '--test':
-        return _self_test()
-
-    return _usage()
-
-if __name__ == '__main__':
-    _main()

tools/dmi/merge_driver.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import sys
 import dmi
+from hooks.merge_frontend import MergeDriver
+
 
 def images_equal(left, right):
     if left.size != right.size:
@@ -15,6 +17,7 @@ def images_equal(left, right):
                 return False
     return True
 
+
 def states_equal(left, right):
     result = True
 
@@ -31,9 +34,11 @@ def states_equal(left, right):
 
     return result
 
+
 def key_of(state):
     return (state.name, state.movement)
 
+
 def dictify(sheet):
     result = {}
     for state in sheet.states:
@@ -43,6 +48,7 @@ def dictify(sheet):
         result[k] = state
     return result
 
+
 def three_way_merge(base, left, right):
     base_dims = base.width, base.height
     if base_dims != (left.width, left.height) or base_dims != (right.width, right.height):
@@ -145,33 +151,31 @@ def three_way_merge(base, left, right):
     merged.states = final_states
     return len(conflicts), merged
 
-def main(path, original, left, right):
-    print(f"Merging icon: {path}")
 
-    icon_orig = dmi.Dmi.from_file(original)
-    icon_left = dmi.Dmi.from_file(left)
-    icon_right = dmi.Dmi.from_file(right)
+class DmiDriver(MergeDriver):
+    driver_id = 'dmi'
+
+    def merge(self, base, left, right):
+        icon_base = dmi.Dmi.from_file(base)
+        icon_left = dmi.Dmi.from_file(left)
+        icon_right = dmi.Dmi.from_file(right)
+        trouble, merge_result = three_way_merge(icon_base, icon_left, icon_right)
+        return not trouble, merge_result
+
+    def to_file(self, outfile, merge_result):
+        merge_result.to_file(outfile)
+
+    def post_announce(self, success, merge_result):
+        if not success:
+            print("!!! Manual merge required!")
+            if merge_result:
+                print("    A best-effort merge was performed. You must edit the icon and remove all")
+                print("    icon states marked with !CONFLICT!, leaving only the desired icon.")
+            else:
+                print("    The icon was totally unable to be merged, you must start with one version")
+                print("    or the other and manually resolve the conflict.")
+            print("    Information about which states conflicted is listed above.")
 
-    trouble, merged = three_way_merge(icon_orig, icon_left, icon_right)
-    if merged:
-        merged.to_file(left)
-    if trouble:
-        print("!!! Manual merge required!")
-        if merged:
-            print("    A best-effort merge was performed. You must edit the icon and remove all")
-            print("    icon states marked with !CONFLICT!, leaving only the desired icon.")
-        else:
-            print("    The icon was totally unable to be merged, you must start with one version")
-            print("    or the other and manually resolve the conflict.")
-        print("    Information about which states conflicted is listed above.")
-    return trouble
 
 if __name__ == '__main__':
-    if len(sys.argv) != 6:
-        print("DMI merge driver called with wrong number of arguments")
-        print("    usage: merge-driver-dmi %P %O %A %B %L")
-        exit(1)
-
-    # "left" is also the file that ought to be overwritten
-    _, path, original, left, right, conflict_size_marker = sys.argv
-    exit(main(path, original, left, right))
+    exit(DmiDriver().main())

tools/dmi/test.py

@@ -0,0 +1,39 @@
+import os
+import sys
+from dmi import *
+
+
+def _self_test():
+    # test: can we load every DMI in the tree
+    count = 0
+    for dirpath, dirnames, filenames in os.walk('.'):
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+        for filename in filenames:
+            if filename.endswith('.dmi'):
+                fullpath = os.path.join(dirpath, filename)
+                try:
+                    Dmi.from_file(fullpath)
+                except Exception:
+                    print('Failed on:', fullpath)
+                    raise
+                count += 1
+
+    print(f"{os.path.relpath(__file__)}: successfully parsed {count} .dmi files")
+
+
+def _usage():
+    print(f"Usage:")
+    print(f"    tools{os.sep}bootstrap{os.sep}python -m {__spec__.name}")
+    exit(1)
+
+
+def _main():
+    if len(sys.argv) == 1:
+        return _self_test()
+
+    return _usage()
+
+
+if __name__ == '__main__':
+    _main()

tools/hooks/Install.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python" -m hooks.install %*
+@pause

tools/hooks/README.md

@@ -1,41 +1,36 @@
 # Git Integration Hooks
 
-This folder contains installable scripts for [Git hooks] and [merge drivers].
+This folder contains installable scripts for Git [hooks] and [merge drivers].
 Use of these hooks and drivers is optional and they must be installed
 explicitly before they take effect.
 
 To install the current set of hooks, or update if new hooks are added, run
-`install.bat` (Windows) or `install.sh` (Unix-like) as appropriate.
+`Install.bat` (Windows) or `tools/hooks/install.sh` (Unix-like) as appropriate.
 
-Hooks expect a Unix-like environment on the backend. Usually this is handled
-automatically by GUI tools like TortoiseGit and GitHub for Windows, but
-[Git for Windows] is an option if you prefer to use a CLI even on Windows.
+If your Git GUI does not support a given hook, there is usually a `.bat` file
+or other script you can run instead - see the links below for details.
 
-## Current Hooks
+## Hooks
 
-* **Pre-commit**: Runs [mapmerge2] on changed maps, if any.
-* **DMI merger**: Attempts to [fix icon conflicts] when performing a git merge.
-  If it succeeds, the file is marked merged. If it fails, it logs what states
-  are still in conflict and adds them to the .dmi file, where the desired
-  resolution can be chosen.
+* **Pre-commit**: Runs [mapmerge2] to reduce the diff on any changed maps.
+* **DMI merger**: Attempts to [fix icon conflicts] when performing a Git merge.
+* **DMM merger**: Attempts to [fix map conflicts] when performing a Git merge.
 
 ## Adding New Hooks
 
-New [Git hooks] may be added by creating a file named `<hook-name>.hook` in
+New Git [hooks] may be added by creating a file named `<hook-name>.hook` in
 this directory. Git determines what hooks are available and what their names
-are. The install script copies the `.hook` file into `.git/hooks`, so editing
-the `.hook` file will require a reinstall.
+are.
 
-New [merge drivers] may be added by adding a shell script named `<ext>.merge`
+New Git [merge drivers] may be added by adding a shell script named `<ext>.merge`
 and updating `.gitattributes` in the root of the repository to include the line
-`*.<ext> merge=<ext>`. The install script will set up the merge driver to point
-to the `.merge` file directly, and editing it will not require a reinstall.
+`*.<ext> merge=<ext>`.
 
-`tools/hooks/python.sh` may be used as a trampoline to ensure that the correct
-version of Python is found.
+Adding or removing hooks or merge drivers requires running the install script
+again, but modifying them does not. See existing `.hook` and `.merge` files for examples.
 
-[Git hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks
+[hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks
 [merge drivers]: https://git-scm.com/docs/gitattributes#_performing_a_three_way_merge
-[Git for Windows]: https://gitforwindows.org/
 [mapmerge2]: ../mapmerge2/README.md
-[fix icon conflicts]: ../mapmerge2/merge_driver_dmi.py
+[fix icon conflicts]: https://tgstation13.org/wiki/Resolving_icon_conflicts
+[fix map conflicts]: https://tgstation13.org/wiki/Map_Merger

tools/hooks/Uninstall.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python" -m hooks.install --uninstall %*
+@pause

tools/hooks/dmi.merge

@@ -1,2 +1,2 @@
 #!/bin/sh
-exec tools/hooks/python.sh -m merge_driver_dmi "$@"
+exec tools/bootstrap/python -m dmi.merge_driver "$@"

tools/hooks/dmm.merge

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec tools/bootstrap/python -m mapmerge2.merge_driver "$@"

tools/hooks/install.bat

@@ -1,16 +0,0 @@
-@echo off
-cd %~dp0
-for %%f in (*.hook) do (
-	echo Installing hook: %%~nf
-	copy %%f ..\..\.git\hooks\%%~nf >nul
-)
-for %%f in (*.merge) do (
-	echo Installing merge driver: %%~nf
-	echo [merge "%%~nf"]^
-
-	driver = tools/hooks/%%f %%P %%O %%A %%B %%L >> ..\..\.git\config
-)
-echo Installing Python dependencies
-python -m pip install -r ..\mapmerge2\requirements.txt
-echo Done
-pause

tools/hooks/install.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# hooks/install.py
+#
+# This script is configured by adding `*.hook` and `*.merge` files in the same
+# directory. Such files should be `#!/bin/sh` scripts, usually invoking Python.
+# This installer will have to be re-run any time a hook or merge file is added
+# or removed, but not when they are changed.
+#
+# Merge drivers will also need a corresponding entry in the `.gitattributes`
+# file.
+
+import os
+import stat
+import glob
+import re
+import pygit2
+import shlex
+
+
+def write_hook(fname, command):
+    with open(fname, 'w', encoding='utf-8', newline='\n') as f:
+        print("#!/bin/sh", file=f)
+        print("exec", command, file=f)
+
+    # chmod +x
+    st = os.stat(fname)
+    if not hasattr(st, 'st_file_attributes'):
+        os.chmod(fname, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def _find_stuff(target=None):
+    repo_dir = pygit2.discover_repository(target or os.getcwd())
+    repo = pygit2.Repository(repo_dir)
+    # Strips any active worktree to find the hooks directory.
+    root_repo_dir = re.sub(r'/.git/worktrees/[^/]+/', '/.git/', repo_dir)
+    hooks_dir = os.path.join(root_repo_dir, 'hooks')
+    return repo, hooks_dir
+
+
+def uninstall(target=None, keep=()):
+    repo, hooks_dir = _find_stuff(target)
+
+    # Remove hooks
+    for fname in glob.glob(os.path.join(hooks_dir, '*')):
+        _, shortname = os.path.split(fname)
+        if not fname.endswith('.sample') and f"{shortname}.hook" not in keep:
+            print('Removing hook:', shortname)
+            os.unlink(fname)
+
+    # Remove merge driver configuration
+    for entry in repo.config:
+        match = re.match(r'^merge\.([^.]+)\.driver$', entry.name)
+        if match and f"{match.group(1)}.merge" not in keep:
+            print('Removing merge driver:', match.group(1))
+            del repo.config[entry.name]
+
+
+def install(target=None):
+    repo, hooks_dir = _find_stuff(target)
+    tools_hooks = os.path.split(__file__)[0]
+
+    keep = set()
+    for full_path in glob.glob(os.path.join(tools_hooks, '*.hook')):
+        _, fname = os.path.split(full_path)
+        name, _ = os.path.splitext(fname)
+        print('Installing hook:', name)
+        keep.add(fname)
+        relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/'))
+        write_hook(os.path.join(hooks_dir, name), f'{relative_path} "$@"')
+
+    # Use libgit2 config manipulation to set the merge driver config.
+    for full_path in glob.glob(os.path.join(tools_hooks, '*.merge')):
+        # Merge drivers are documented here: https://git-scm.com/docs/gitattributes
+        _, fname = os.path.split(full_path)
+        name, _ = os.path.splitext(fname)
+        print('Installing merge driver:', name)
+        keep.add(fname)
+        # %P: "real" path of the file, should not usually be read or modified
+        # %O: ancestor's version
+        # %A: current version, and also the output path
+        # %B: other branches' version
+        # %L: conflict marker size
+        relative_path = shlex.quote(os.path.relpath(full_path, repo.workdir).replace('\\', '/'))
+        repo.config[f"merge.{name}.driver"] = f'{relative_path} %P %O %A %B %L'
+
+    uninstall(target, keep=keep)
+
+
+def main(argv):
+    if len(argv) <= 1:
+        return install()
+    elif argv[1] == '--uninstall':
+        return uninstall()
+    else:
+        print("Usage: python -m hooks.install [--uninstall]")
+        return 1
+
+
+if __name__ == '__main__':
+    import sys
+    exit(main(sys.argv))

tools/hooks/install.sh

@@ -1,20 +1,2 @@
-#!/bin/bash
-set -e
-shopt -s nullglob
-cd "$(dirname "$0")"
-for f in *.hook; do
-	echo Installing hook: ${f%.hook}
-	cp $f ../../.git/hooks/${f%.hook}
-done
-for f in *.merge; do
-	echo Installing merge driver: ${f%.merge}
-	git config --replace-all merge.${f%.merge}.driver "tools/hooks/$f %P %O %A %B %L"
-done
-
-echo "Installing tgui hooks"
-../../tgui/bin/tgui --install-git-hooks
-
-echo "Installing Python dependencies"
-./python.sh -m pip install -r ../mapmerge2/requirements.txt
-
-echo "Done"
+#!/bin/sh
+exec "$(dirname "$0")/../bootstrap/python" -m hooks.install "$@"

tools/hooks/merge_frontend.py

@@ -0,0 +1,169 @@
+# merge_frontend.py
+import sys
+import io
+import os
+import pygit2
+import collections
+import typing
+
+
+ENCODING = 'utf-8'
+
+
+class MergeReturn(typing.NamedTuple):
+    success: bool
+    merge_result: typing.Optional[object]
+
+
+class MergeDriver:
+    driver_id: typing.Optional[str] = None
+
+    def pre_announce(self, path: str):
+        """
+        Called before merge() is called, with a human-friendly path for output.
+        """
+        print(f"Merging {self.driver_id}: {path}")
+
+    def merge(self, base: typing.BinaryIO, left: typing.BinaryIO, right: typing.BinaryIO) -> MergeReturn:
+        """
+        Read from three BinaryIOs: base (common ancestor), left (ours), and
+        right (theirs). Perform the actual three-way merge operation. Leave
+        conflict markers if necessary.
+
+        Return (False, None) to indicate the merge driver totally failed.
+        Return (False, merge_result) if the result contains conflict markers.
+        Return (True, merge_result) if everything went smoothly.
+        """
+        raise NotImplementedError
+
+    def to_file(self, output: typing.BinaryIO, merge_result: object):
+        """
+        Save the merge() result to the given output stream.
+        Override this if the merge() result is not bytes or str.
+        """
+        if isinstance(merge_result, bytes):
+            output.write(merge_result)
+        elif isinstance(merge_result, str):
+            with io.TextIOWrapper(output, ENCODING) as f:
+                f.write(merge_result)
+        else:
+            raise NotImplementedError
+
+    def post_announce(self, success: bool, merge_result: object):
+        """
+        Called after merge() is called, to warn the user if action is needed.
+        """
+        if not success:
+            print("!!! Manual merge required")
+            if merge_result:
+                print("    A best-effort merge was performed. You must finish the job yourself.")
+            else:
+                print("    No merge was possible. You must resolve the conflict yourself.")
+
+    def main(self, args: typing.List[str] = None):
+        return _main(self, args or sys.argv[1:])
+
+
+def _main(driver: MergeDriver, args: typing.List[str]):
+    if len(args) > 0 and args[0] == '--posthoc':
+        return _posthoc_main(driver, args[1:])
+    else:
+        return _driver_main(driver, args)
+
+
+def _driver_main(driver: MergeDriver, args: typing.List[str]):
+    """
+    Act like a normal Git merge driver, called by Git during a merge.
+    """
+    if len(args) != 5:
+        print("merge driver called with wrong number of arguments")
+        print("    usage: %P %O %A %B %L")
+        return 1
+
+    path, path_base, path_left, path_right, _ = args
+    driver.pre_announce(path)
+
+    with open(path_base, 'rb') as io_base:
+        with open(path_left, 'rb') as io_left:
+            with open(path_right, 'rb') as io_right:
+                success, merge_result = driver.merge(io_base, io_left, io_right)
+
+    if merge_result:
+        # If we got anything, write it to the working directory.
+        with open(path_left, 'wb') as io_output:
+            driver.to_file(io_output, merge_result)
+
+    driver.post_announce(success, merge_result)
+    if not success:
+        # If we were not successful, do not mark the conflict as resolved.
+        return 1
+
+
+def _posthoc_main(driver: MergeDriver, args: typing.List[str]):
+    """
+    Apply merge driver logic to a repository which is already in a conflicted
+    state, running the driver on any conflicted files.
+    """
+    repo_dir = pygit2.discover_repository(os.getcwd())
+    repo = pygit2.Repository(repo_dir)
+    conflicts = repo.index.conflicts
+    if not conflicts:
+        print("There are no unresolved conflicts.")
+        return 0
+
+    all_success = True
+    index_changed = False
+    any_attempted = False
+    for base, left, right in list(conflicts):
+        if not base or not left or not right:
+            # (not left) or (not right): deleted in one branch, modified in the other.
+            # (not base): added differently in both branches.
+            # In either case, there's nothing we can do for now.
+            continue
+
+        path = left.path
+        if not _applies_to(repo, driver, path):
+            # Skip the file if it's not the right extension.
+            continue
+
+        any_attempted = True
+        driver.pre_announce(path)
+        io_base = io.BytesIO(repo[base.id].data)
+        io_left = io.BytesIO(repo[left.id].data)
+        io_right = io.BytesIO(repo[right.id].data)
+        success, merge_result = driver.merge(io_base, io_left, io_right)
+        if merge_result:
+            # If we got anything, write it to the working directory.
+            with open(os.path.join(repo.workdir, path), 'wb') as io_output:
+                driver.to_file(io_output, merge_result)
+
+            if success:
+                # If we were successful, mark the conflict as resolved.
+                with open(os.path.join(repo.workdir, path), 'rb') as io_readback:
+                    contents = io_readback.read()
+                merged_id = repo.create_blob(contents)
+                repo.index.add(pygit2.IndexEntry(path, merged_id, left.mode))
+                del conflicts[path]
+                index_changed = True
+        if not success:
+            all_success = False
+        driver.post_announce(success, merge_result)
+
+    if index_changed:
+        repo.index.write()
+
+    if not any_attempted:
+        print("There are no unresolved", driver.driver_id, "conflicts.")
+
+    if not all_success:
+        # Not usually observed, but indicate the failure just in case.
+        return 1
+
+
+def _applies_to(repo: pygit2.Repository, driver: MergeDriver, path: str):
+    """
+    Check if the current merge driver is a candidate to handle a given path.
+    """
+    if not driver.driver_id:
+        raise ValueError('Driver must have ID to perform post-hoc merge')
+    return repo.get_attr(path, 'merge') == driver.driver_id

tools/hooks/pre-commit.hook

@@ -1,3 +1,2 @@
 #!/bin/sh
-# `sh` must be used here instead of `bash` to support GitHub Desktop.
-exec tools/hooks/python.sh -m precommit
+exec tools/bootstrap/python -m mapmerge2.precommit

tools/hooks/python.sh

@@ -1,32 +1,17 @@
 #!/bin/sh
-# `sh` must be used here instead of `bash` to support GitHub Desktop.
 set -e
-
-# Strip the "App Execution Aliases" from $PATH. Even if the user installed
-# Python using the Windows Store on purpose, these aliases always generate
-# "Permission denied" errors when sh.exe tries to invoke them.
-PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":")
-
-# Try to find a Python executable.
-if command -v python3 >/dev/null 2>&1; then
-	PY=python3
-elif command -v python >/dev/null 2>&1; then
-	PY=python
-elif command -v py >/dev/null 2>&1; then
-	PY="py -3"
+if [ "$*" = "-m precommit" ]; then
+	echo "Hooks are being updated..."
+	echo "Details: https://github.com/tgstation/tgstation/pull/55658"
+	if [ "$(uname -o)" = "Msys" ]; then
+		tools/hooks/Install.bat
+	else
+		tools/hooks/install.sh
+	fi
+	echo "---------------"
+	exec tools/hooks/pre-commit.hook
 else
-	echo "Please install Python from https://www.python.org/downloads/"
+	echo "tools/hooks/python.sh is replaced by tools/bootstrap/python"
+	echo "Details: https://github.com/tgstation/tgstation/pull/55658"
 	exit 1
 fi
-
-# Deduce the path separator and add the mapmerge package to the search path.
-PATHSEP=$($PY - <<'EOF'
-import sys, os
-if sys.version_info.major != 3 or sys.version_info.minor < 6:
-	sys.stderr.write("Python 3.6 or later is required, but you have:\n" + sys.version + "\n")
-	exit(1)
-print(os.pathsep)
-EOF
-)
-export PYTHONPATH=tools/mapmerge2/${PATHSEP}${PYTHONPATH}
-exec $PY "$@"

tools/makeChangelog.bat

@@ -1,4 +1,4 @@
 @echo off
 rem Cheridan asked for this. - N3X
-call python ss13_genchangelog.py ../html/changelog.html ../html/changelogs
-pause
+call "%~dp0\bootstrap\python" ss13_genchangelog.py ../html/changelog.html ../html/changelogs
+pause

tools/mapmerge2/I Forgot To Map Merge.bat

@@ -0,0 +1,7 @@
+@echo off
+echo Installing hooks for next time...
+call "%~dp0\..\bootstrap\python.bat" -m hooks.install
+echo.
+echo Fixing things up...
+call "%~dp0\..\bootstrap\python.bat" -m mapmerge2.fixup
+pause

tools/mapmerge2/Prepare Maps.bat

@@ -1,12 +0,0 @@
-@echo off
-cd ../../_maps/
-
-for /R %%f in (*.dmm) do copy "%%f" "%%f.backup"
-
-cls
-echo All dmm files in _maps directories have been backed up
-echo Now you can make your changes...
-echo ---
-echo Remember to run mapmerge.bat just before you commit your changes!
-echo ---
-pause

tools/mapmerge2/README.md

@@ -1,8 +1,27 @@
-# Map Merge 2
+# Map Merger
 
-**Map Merge 2** is an improvement over previous map merging scripts, with
-better merge-conflict prevention, multi-Z support, and automatic handling of
-key overflow. For up-to-date tips and tricks, also visit the [Map Merger] wiki article.
+The **Map Merger** is a collection of scripts that keep this repository's maps
+in a format which is easier to track in Git and less likely to cause merge
+conflicts. When merge conflicts do occur, it can sometimes resolve them.
+
+For detailed troubleshooting instructions and other tips, visit the
+[Map Merger] wiki article.
+
+## Installation
+
+To install the [Git hooks], open the `tools/hooks/` folder and double-click
+`Install.bat`. Linux users run `tools/hooks/install.sh`.
+
+## Manual Use
+
+If using a Git GUI which is not compatible with the hooks:
+
+* Before committing, double-click `Run Before Committing.bat`
+* When a merge has map conflicts, double-click `Resolve Map Conflicts.bat`
+
+The console will show whether the operation succeeded.
+
+For more details, see the [Map Merger] wiki article.
 
 ## What Map Merging Is
 
@@ -13,16 +32,8 @@ version of the map while maintaining all the actual changes. It requires an old
 version of the map to use as a reference and a new version of the map which
 contains the desired changes.
 
-## Installation
-
-To install Python dependencies, run `requirements-install.bat`, or run
-`python -m pip install -r requirements.txt` directly. See the [Git hooks]
-documentation to install the Git pre-commit hook which runs the map merger
-automatically, or use `tools/mapmerge/Prepare Maps.bat` to save backups before
-running `mapmerge.bat`.
-
-For up-to-date installation and detailed troubleshooting instructions, visit
-the [Map Merger] wiki article.
+Map Merge 2 adds multi-Z support, automatic handling of key overflow, better
+merge conflict prevention, and a real merge conflict resolver.
 
 ## Code Structure
 

tools/mapmerge2/Resolve Map Conflicts.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python.bat" -m mapmerge2.merge_driver --posthoc %*
+@pause

tools/mapmerge2/Run Before Committing.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python" -m mapmerge2.precommit --use-workdir %*
+@pause

tools/mapmerge2/convert.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-import frontend
-import dmm
+from . import frontend, dmm
 
 if __name__ == '__main__':
     settings = frontend.read_settings()

tools/mapmerge2/dmm.py

@@ -22,20 +22,19 @@ class DMM:
 
     @staticmethod
     def from_file(fname):
-        # stream the file rather than forcing all its contents to memory
         with open(fname, 'r', encoding=ENCODING) as f:
-            return _parse(iter(lambda: f.read(1), ''))
+            return _parse(f.read())
 
     @staticmethod
     def from_bytes(bytes):
         return _parse(bytes.decode(ENCODING))
 
-    def to_file(self, fname, tgm = True):
+    def to_file(self, fname, *, tgm = True):
         self._presave_checks()
         with open(fname, 'w', newline='\n', encoding=ENCODING) as f:
             (save_tgm if tgm else save_dmm)(self, f)
 
-    def to_bytes(self, tgm = True):
+    def to_bytes(self, *, tgm = True):
         self._presave_checks()
         bio = io.BytesIO()
         with io.TextIOWrapper(bio, newline='\n', encoding=ENCODING) as f:
@@ -43,6 +42,21 @@ class DMM:
             f.flush()
             return bio.getvalue()
 
+    def get_or_generate_key(self, tile):
+        try:
+            return self.dictionary.inv[tile]
+        except KeyError:
+            key = self.generate_new_key()
+            self.dictionary[key] = tile
+            return key
+
+    def get_tile(self, coord):
+        return self.dictionary[self.grid[coord]]
+
+    def set_tile(self, coord, tile):
+        tile = tuple(tile)
+        self.grid[coord] = self.get_or_generate_key(tile)
+
     def generate_new_key(self):
         free_keys = self._ensure_free_keys(1)
         max_key = max_key_for(self.key_length)
@@ -119,6 +133,9 @@ class DMM:
             for x in range(1, self.size.x + 1):
                 yield (y, x)
 
+    def __repr__(self):
+        return f"DMM(size={self.size}, key_length={self.key_length}, dictionary_size={len(self.dictionary)})"
+
 # ----------
 # key handling
 
@@ -221,10 +238,8 @@ def is_bad_atom_ordering(key, atoms):
         print(f"Warning: key '{key}' is missing either a turf or area")
     return can_fix
 
-def fix_atom_ordering(atoms):
-    movables = []
-    turfs = []
-    areas = []
+def split_atom_groups(atoms):
+    movables, turfs, areas = [], [], []
     for each in atoms:
         if each.startswith('/turf'):
             turfs.append(each)
@@ -232,6 +247,10 @@ def fix_atom_ordering(atoms):
             areas.append(each)
         else:
             movables.append(each)
+    return movables, turfs, areas
+
+def fix_atom_ordering(atoms):
+    movables, turfs, areas = split_atom_groups(atoms)
     movables.extend(turfs)
     movables.extend(areas)
     return movables
@@ -281,7 +300,7 @@ def save_tgm(dmm, output):
         output.write("\n")
         for x in range(1, max_x + 1):
             output.write(f"({x},{1},{z}) = {{\"\n")
-            for y in range(1, max_y + 1):
+            for y in range(max_y, 0, -1):
                 output.write(f"{num_to_key(dmm.grid[x, y, z], dmm.key_length)}\n")
             output.write("\"}\n")
 
@@ -303,7 +322,7 @@ def save_dmm(dmm, output):
     for z in range(1, max_z + 1):
         output.write(f"(1,1,{z}) = {{\"\n")
 
-        for y in range(1, max_y + 1):
+        for y in range(max_y, 0, -1):
             for x in range(1, max_x + 1):
                 try:
                     output.write(num_to_key(dmm.grid[x, y, z], dmm.key_length))
@@ -536,7 +555,17 @@ def _parse(map_raw_text):
     if curr_y > maxy:
         maxy = curr_y
 
+    if not grid:
+        # Usually caused by unbalanced quotes.
+        max_key = num_to_key(max(dictionary.keys()), key_length, True)
+        raise ValueError(f"dmm failed to parse, check for a syntax error near or after key {max_key!r}")
+
+    # Convert from raw .dmm coordinates to DM/BYOND coordinates by flipping Y
+    grid2 = dict()
+    for (x, y, z), tile in grid.items():
+        grid2[x, maxy + 1 - y, z] = tile
+
     data = DMM(key_length, Coordinate(maxx, maxy, maxz))
     data.dictionary = dictionary
-    data.grid = grid
+    data.grid = grid2
     return data

tools/mapmerge2/dmm2tgm.bat

@@ -1,5 +0,0 @@
-@echo off
-set MAPROOT=../../_maps/
-set TGM=1
-python convert.py
-pause

tools/mapmerge2/dmm_test.py

@@ -0,0 +1,39 @@
+import os
+import sys
+from .dmm import *
+
+
+def _self_test():
+    # test: can we load every DMM in the tree
+    count = 0
+    for dirpath, dirnames, filenames in os.walk('.'):
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+        for filename in filenames:
+            if filename.endswith('.dmm'):
+                fullpath = os.path.join(dirpath, filename)
+                try:
+                    DMM.from_file(fullpath)
+                except Exception:
+                    print('Failed on:', fullpath)
+                    raise
+                count += 1
+
+    print(f"{os.path.relpath(__file__)}: successfully parsed {count} .dmm files")
+
+
+def _usage():
+    print(f"Usage:")
+    print(f"    tools{os.sep}bootstrap{os.sep}python -m {__spec__.name}")
+    exit(1)
+
+
+def _main():
+    if len(sys.argv) == 1:
+        return _self_test()
+
+    return _usage()
+
+
+if __name__ == '__main__':
+    _main()

tools/mapmerge2/fixup.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+import os
+import pygit2
+from . import dmm
+from .mapmerge import merge_map
+
+
+STATUS_INDEX = (pygit2.GIT_STATUS_INDEX_NEW
+    | pygit2.GIT_STATUS_INDEX_MODIFIED
+    | pygit2.GIT_STATUS_INDEX_DELETED
+    | pygit2.GIT_STATUS_INDEX_RENAMED
+    | pygit2.GIT_STATUS_INDEX_TYPECHANGE
+)
+STATUS_WT = (pygit2.GIT_STATUS_WT_NEW
+    | pygit2.GIT_STATUS_WT_MODIFIED
+    | pygit2.GIT_STATUS_WT_DELETED
+    | pygit2.GIT_STATUS_WT_RENAMED
+    | pygit2.GIT_STATUS_WT_TYPECHANGE
+)
+ABBREV_LEN = 12
+TGM_HEADER = dmm.TGM_HEADER.encode(dmm.ENCODING)
+
+
+def walk_tree(tree, *, _prefix=''):
+    for child in tree:
+        if isinstance(child, pygit2.Tree):
+            yield from walk_tree(child, _prefix=f'{_prefix}{child.name}/')
+        else:
+            yield f'{_prefix}{child.name}', child
+
+
+def insert_into_tree(repo, tree_builder, path, blob_oid):
+    try:
+        first, rest = path.split('/', 1)
+    except ValueError:
+        tree_builder.insert(path, blob_oid, pygit2.GIT_FILEMODE_BLOB)
+    else:
+        inner = repo.TreeBuilder(tree_builder.get(first))
+        insert_into_tree(repo, inner, rest, blob_oid)
+        tree_builder.insert(first, inner.write(), pygit2.GIT_FILEMODE_TREE)
+
+
+def main(repo):
+    if repo.index.conflicts:
+        print("You need to resolve merge conflicts first.")
+        return 1
+
+    # Ensure the index is clean.
+    for path, status in repo.status().items():
+        if status & pygit2.GIT_STATUS_IGNORED:
+            continue
+        if status & STATUS_INDEX:
+            print("You have changes staged for commit. Commit them or unstage them first.")
+            print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.")
+            return 1
+        if path.endswith(".dmm") and (status & STATUS_WT):
+            print("You have modified maps. Commit them first.")
+            print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.")
+            return 1
+
+    # Read the HEAD commit.
+    head_commit = repo[repo.head.target]
+    head_files = {}
+    for path, blob in walk_tree(head_commit.tree):
+        if path.endswith(".dmm"):
+            data = blob.read_raw()
+            if not data.startswith(TGM_HEADER):
+                head_files[path] = dmm.DMM.from_bytes(data)
+
+    if not head_files:
+        print("All committed maps appear to be in the correct format.")
+        print("If you are about to commit maps for the first time, run `Run Before Committing.bat`.")
+        return 1
+
+    # Work backwards to find a base for each map, converting as found.
+    converted = {}
+    if len(head_commit.parents) != 1:
+        print("Unable to automatically fix anything because HEAD is a merge commit.")
+        return 1
+    commit_message_lines = []
+    working_commit = head_commit.parents[0]
+    while len(converted) < len(head_files):
+        for path in head_files.keys() - converted.keys():
+            try:
+                blob = working_commit.tree[path]
+            except KeyError:
+                commit_message_lines.append(f"{'new':{ABBREV_LEN}}: {path}")
+                print(f"Converting new map: {path}")
+                converted[path] = head_files[path]
+            else:
+                data = blob.read_raw()
+                if data.startswith(TGM_HEADER):
+                    str_id = str(working_commit.id)[:ABBREV_LEN]
+                    commit_message_lines.append(f"{str_id}: {path}")
+                    print(f"Converting map: {path}")
+                    converted[path] = merge_map(head_files[path], dmm.DMM.from_bytes(data))
+        if len(working_commit.parents) != 1:
+            print("A merge commit was encountered before good versions of these maps were found:")
+            print("\n".join(f"    {x}" for x in head_files.keys() - base_files.keys()))
+            return 1
+        working_commit = working_commit.parents[0]
+
+    # Okay, do the actual work.
+    tree_builder = repo.TreeBuilder(head_commit.tree)
+    for path, merged_map in converted.items():
+        blob_oid = repo.create_blob(merged_map.to_bytes())
+        insert_into_tree(repo, tree_builder, path, blob_oid)
+        repo.index.add(pygit2.IndexEntry(path, blob_oid, repo.index[path].mode))
+        merged_map.to_file(os.path.join(repo.workdir, path))
+
+    # Save the index.
+    repo.index.write()
+
+    # Commit the index to the current branch.
+    signature = pygit2.Signature(repo.config['user.name'], repo.config['user.email'])
+    joined = "\n".join(commit_message_lines)
+    repo.create_commit(
+        repo.head.name,
+        signature,  # author
+        signature,  # committer
+        f'Convert maps to TGM\n\n{joined}\n\nAutomatically commited by: {os.path.relpath(__file__, repo.workdir)}',
+        tree_builder.write(),
+        [head_commit.id],
+    )
+
+    # Success.
+    print("Successfully committed a fixup. Push as needed.")
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main(pygit2.Repository(pygit2.discover_repository(os.getcwd()))))

tools/mapmerge2/mapmerge.bat

@@ -1,5 +0,0 @@
-@echo off
-set MAPROOT=../../_maps/
-set TGM=1
-python mapmerge.py
-pause

tools/mapmerge2/mapmerge.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-import frontend
 import shutil
-from dmm import *
 from collections import defaultdict
+from . import frontend
+from .dmm import *
 
 def merge_map(new_map, old_map, delete_unused=False):
     if new_map.key_length != old_map.key_length:
@@ -66,7 +66,7 @@ def merge_map(new_map, old_map, delete_unused=False):
 
     # step two: delete unused keys
     if unused_keys:
-        print(f"Notice: Trimming {len(unused_keys)} unused dictionary keys.")
+        #print(f"Notice: Trimming {len(unused_keys)} unused dictionary keys.")
         for key in unused_keys:
             del merged.dictionary[key]
 

tools/mapmerge2/merge_driver.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+import sys
+import collections
+from . import dmm, mapmerge
+from hooks.merge_frontend import MergeDriver
+
+
+debug_stats = collections.defaultdict(int)
+
+
+def select(base, left, right, *, debug=None):
+    if left == right:
+        # whether or not it's in the base, both sides agree
+        if debug:
+            debug_stats[f"select {debug} both"] += 1
+        return left
+    elif base == left:
+        # base == left, but right is different: accept right
+        if debug:
+            debug_stats[f"select {debug} right"] += 1
+        return right
+    elif base == right:
+        # base == right, but left is different: accept left
+        if debug:
+            debug_stats[f"select {debug} left"] += 1
+        return left
+    else:
+        # all three versions are different
+        if debug:
+            debug_stats[f"select {debug} fail"] += 1
+        return None
+
+
+def three_way_merge(base, left, right):
+    if base.size != left.size or base.size != right.size:
+        print("Dimensions have changed:")
+        print(f"    Base: {base.size}")
+        print(f"    Ours: {left.size}")
+        print(f"    Theirs: {right.size}")
+        return True, None
+
+    trouble = False
+    merged = dmm.DMM(base.key_length, base.size)
+    merged.dictionary = base.dictionary.copy()
+
+    for (z, y, x) in base.coords_zyx:
+        coord = x, y, z
+        base_tile = base.get_tile(coord)
+        left_tile = left.get_tile(coord)
+        right_tile = right.get_tile(coord)
+
+        # try to merge the whole tiles
+        whole_tile_merge = select(base_tile, left_tile, right_tile, debug='tile')
+        if whole_tile_merge is not None:
+            merged.set_tile(coord, whole_tile_merge)
+            continue
+
+        # try to merge each group independently (movables, turfs, areas)
+        base_movables, base_turfs, base_areas = dmm.split_atom_groups(base_tile)
+        left_movables, left_turfs, left_areas = dmm.split_atom_groups(left_tile)
+        right_movables, right_turfs, right_areas = dmm.split_atom_groups(right_tile)
+
+        merged_movables = select(base_movables, left_movables, right_movables, debug='movable')
+        merged_turfs = select(base_turfs, left_turfs, right_turfs, debug='turf')
+        merged_areas = select(base_areas, left_areas, right_areas, debug='area')
+
+        if merged_movables is not None and merged_turfs is not None and merged_areas is not None:
+            merged.set_tile(coord, merged_movables + merged_turfs + merged_areas)
+            continue
+
+        # TODO: more advanced strategies?
+
+        # fall back to requiring manual conflict resolution
+        trouble = True
+        print(f" C: Both sides touch the tile at {coord}")
+
+        if merged_movables is None:
+            obj_name = "---Merge conflict marker---"
+            merged_movables = left_movables + [f'/obj{{name = "{obj_name}"}}'] + right_movables
+            print(f"    Left and right movable groups are split by an `/obj` named \"{obj_name}\"")
+        if merged_turfs is None:
+            merged_turfs = left_turfs
+            print(f"    Saving turf: {', '.join(left_turfs)}")
+            print(f"    Alternative: {', '.join(right_turfs)}")
+            print(f"    Original:    {', '.join(base_turfs)}")
+        if merged_areas is None:
+            merged_areas = left_areas
+            print(f"    Saving area: {', '.join(left_areas)}")
+            print(f"    Alternative: {', '.join(right_areas)}")
+            print(f"    Original:    {', '.join(base_areas)}")
+
+        merged.set_tile(coord, merged_movables + merged_turfs + merged_areas)
+
+    merged = mapmerge.merge_map(merged, base)
+    return trouble, merged
+
+
+class DmmDriver(MergeDriver):
+    driver_id = 'dmm'
+
+    def merge(self, base, left, right):
+        map_base = dmm.DMM.from_bytes(base.read())
+        map_left = dmm.DMM.from_bytes(left.read())
+        map_right = dmm.DMM.from_bytes(right.read())
+        trouble, merge_result = three_way_merge(map_base, map_left, map_right)
+        return not trouble, merge_result
+
+    def to_file(self, outfile, merge_result):
+        outfile.write(merge_result.to_bytes())
+
+    def post_announce(self, success, merge_result):
+        if not success:
+            print("!!! Manual merge required!")
+            if merge_result:
+                print("    A best-effort merge was performed. You must edit the map and confirm")
+                print("    that all coordinates mentioned above are as desired.")
+            else:
+                print("    The map was totally unable to be merged; you must start with one version")
+                print("    or the other and manually resolve the conflict. Information about the")
+                print("    conflicting tiles is listed above.")
+
+
+if __name__ == '__main__':
+    exit(DmmDriver().main())

tools/mapmerge2/precommit.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import os
+import sys
 import pygit2
-import dmm
-from mapmerge import merge_map
+from . import dmm
+from .mapmerge import merge_map
 
-def main(repo):
+
+def main(repo, *, use_workdir=False):
     if repo.index.conflicts:
         print("You need to resolve merge conflicts first.")
         return 1
@@ -16,12 +18,21 @@ def main(repo):
     except KeyError:
         pass
 
+    target_statuses = pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_INDEX_NEW
+    skip_to_file_statuses = pygit2.GIT_STATUS_WT_DELETED | pygit2.GIT_STATUS_WT_MODIFIED
+    if use_workdir:
+        target_statuses |= pygit2.GIT_STATUS_WT_MODIFIED | pygit2.GIT_STATUS_WT_NEW
+        skip_to_file_statuses &= ~pygit2.GIT_STATUS_WT_MODIFIED
+
     changed = 0
     for path, status in repo.status().items():
-        if path.endswith(".dmm") and (status & (pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_INDEX_NEW)):
+        if path.endswith(".dmm") and (status & target_statuses):
             # read the index
             index_entry = repo.index[path]
-            index_map = dmm.DMM.from_bytes(repo[index_entry.id].read_raw())
+            if use_workdir:
+                index_map = dmm.DMM.from_file(os.path.join(repo.workdir, path))
+            else:
+                index_map = dmm.DMM.from_bytes(repo[index_entry.id].read_raw())
 
             try:
                 head_blob = repo[repo[repo.head.target].tree[path].id]
@@ -32,7 +43,7 @@ def main(repo):
                 merged_map = index_map
             else:
                 # Entry in HEAD, merge the index over it
-                print(f"Merging map: {path}", flush=True)
+                print(f"Converting map: {path}", flush=True)
                 assert not (status & pygit2.GIT_STATUS_INDEX_NEW)
                 head_map = dmm.DMM.from_bytes(head_blob.read_raw())
                 merged_map = merge_map(index_map, head_map)
@@ -43,15 +54,16 @@ def main(repo):
             changed += 1
 
             # write to the working directory if that's clean
-            if status & (pygit2.GIT_STATUS_WT_DELETED | pygit2.GIT_STATUS_WT_MODIFIED):
+            if status & skip_to_file_statuses:
                 print(f"Warning: {path} has unindexed changes, not overwriting them")
             else:
                 merged_map.to_file(os.path.join(repo.workdir, path))
 
     if changed:
         repo.index.write()
-        print(f"Merged {changed} maps.")
     return 0
 
+
 if __name__ == '__main__':
-    exit(main(pygit2.Repository(pygit2.discover_repository(os.getcwd()))))
+    repo = pygit2.Repository(pygit2.discover_repository(os.getcwd()))
+    exit(main(repo, use_workdir='--use-workdir' in sys.argv))

tools/mapmerge2/requirements-install.bat

@@ -1,3 +0,0 @@
-@echo off
-python -m pip install -r requirements.txt
-pause

tools/mapmerge2/requirements.txt

@@ -1,3 +0,0 @@
-pygit2==1.0.1
-bidict==0.13.1
-Pillow==7.2.0

tools/mapmerge2/tgm2dmm.bat

@@ -1,5 +0,0 @@
-@echo off
-set MAPROOT=../../_maps/
-set TGM=0
-python convert.py
-pause

tools/requirements.txt

@@ -0,0 +1,7 @@
+pygit2==1.0.1
+bidict==0.13.1
+Pillow==7.2.0
+
+# changelogs
+PyYaml==5.3.1
+beautifulsoup4==4.9.3