A set of tools that might be useful later (#7180)

.gitignore

@@ -1,5 +1,4 @@
 #ignore misc BYOND files
-Thumbs.db
 vchat.db
 vchat.db*
 *.log
@@ -11,10 +10,61 @@ vchat.db*
 *.before
 *.pyc
 *.pid
-data
-data/
 cfg/
 
+#Ignore everything in datafolder and subdirectories
+/data/**/*
+/tmp/**/*
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-*.txt
+
+# Unit test / coverage reports
+.cache
+
+# pyenv
+.python-version
+
+# dotenv
+.env
+
+### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Global/Windows.gitignore
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows shortcuts
+*.lnk
+
+### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Global/OSX.gitignore
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
 #Visual studio stuff
 *.vscode/*
 !/.vscode/launch.json

_build_dependencies.sh

@@ -1,11 +1,17 @@
+#!/bin/sh
+
 # This file has all the information on what versions of libraries are thrown into the code
-# For dreamchecker
+
-export SPACEMAN_DMM_VERSION=suite-1.7
+# byond version
-# For NanoUI + TGUI
-export NODE_VERSION=16
-# Byond Major
 export BYOND_MAJOR=514
-# Byond Minor
 export BYOND_MINOR=1589
-# Macro Count
 export MACRO_COUNT=4
+
+# node version
+export NODE_VERSION=16
+
+# SpacemanDMM git tag
+export SPACEMAN_DMM_VERSION=suite-1.7
+
+# Python version for mapmerge and other tools
+export PYTHON_VERSION=3.9.0

tools/bootstrap/node

@@ -0,0 +1,73 @@
+#!/bin/sh
+# bootstrap/node
+#
+# Node-finding script for all `sh` environments, including Linux, MSYS2,
+# Git for Windows, and GitHub Desktop. Invokable from CLI or automation.
+#
+# If a node.exe installed by `node_.ps1` is present, it will be used.
+# Otherwise, this script requires a system `node` to be provided.
+set -e
+
+# Convenience variables
+Bootstrap="$(dirname "$0")"
+Cache="$Bootstrap/.cache"
+if [ "$TG_BOOTSTRAP_CACHE" ]; then
+	Cache="$TG_BOOTSTRAP_CACHE"
+fi
+OldPWD="$PWD"
+cd "$Bootstrap/../.."
+. ./_build_dependencies.sh  # sets NODE_VERSION_PRECISE
+cd "$OldPWD"
+NodeVersion="$NODE_VERSION_PRECISE"
+NodeFullVersion="node-v$NodeVersion-win-x64"
+NodeDir="$Cache/$NodeFullVersion"
+NodeExe="$NodeDir/node.exe"
+is_vendored="1"
+
+# If a bootstrapped Node is not present, search on $PATH.
+if [ "$(uname)" = "Linux" ] || [ ! -f "$NodeExe" ]; then
+	if [ "$TG_BOOTSTRAP_NODE_LINUX" ]; then
+		NodeFullVersion="node-v$NodeVersion-linux-x64"
+		NodeDir="$Cache/$NodeFullVersion/bin"
+		NodeExe="$NodeDir/node"
+
+		if [ ! -f "$NodeExe" ]; then
+			mkdir -p "$Cache"
+			Archive="$(realpath "$Cache/node-v$NodeVersion.tar.gz")"
+			curl "https://nodejs.org/download/release/v$NodeVersion/$NodeFullVersion.tar.gz" -o "$Archive"
+			(cd "$Cache" && tar xf "$Archive")
+		fi
+	elif command -v node >/dev/null 2>&1; then
+		NodeExe="node"
+		is_vendored="0"
+	else
+		echo
+		if command -v apt-get >/dev/null 2>&1; then
+			# Ubuntu advice
+			echo "Please install Node using your system's package manager:"
+			echo "    sudo apt-get install nodejs"
+		elif uname | grep -q MSYS; then
+			# MSYS2 (not packaged) or Git for Windows advice
+			echo "Please run bootstrap/node.bat instead of bootstrap/node once"
+			echo "to install Node automatically, or install it from https://nodejs.org/"
+		elif command -v pacman >/dev/null 2>&1; then
+			# Arch advice
+			echo "Please install Node using your system's package manager:"
+			echo "    sudo pacman -S nodejs"
+		else
+			# Generic advice
+			echo "Please install Node from https://nodejs.org/ or using your system's package manager."
+		fi
+		echo
+		exit 1
+	fi
+fi
+
+# Invoke Node with all command-line arguments
+if [ "$is_vendored" = "1" ]; then
+	PATH="$(readlink -f "$NodeDir"):$PATH"
+	echo "Using vendored Node $("$NodeExe" --version)"
+else
+	echo "Using system-wide Node $("$NodeExe" --version)"
+fi
+exec "$NodeExe" "$@"

tools/bootstrap/node.bat

@@ -0,0 +1,20 @@
+@echo off
+set NODE_SKIP_PLATFORM_CHECK=1
+call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Download-Node
+for /f "tokens=* USEBACKQ" %%s in (`
+	call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Get-Path
+`) do (
+	set "PATH=%%s;%PATH%"
+)
+where node.exe >nul 2>nul
+if %errorlevel% == 0 (
+	echo | set /p printed_str="Using vendored Node "
+	call node.exe --version
+	call node.exe %*
+	goto exit_with_last_error_level
+)
+echo "node.bat: Failed to bootstrap Node!"
+%COMSPEC% /c exit 1
+
+:exit_with_last_error_level
+if not %errorlevel% == 0 %COMSPEC% /c exit %errorlevel% >nul

tools/bootstrap/node_.ps1

@@ -0,0 +1,59 @@
+## bootstrap/node_.ps1
+## Downloads a Node version to a cache directory and invokes it.
+
+$ErrorActionPreference = "Stop"
+
+function Extract-Variable {
+	param([string] $Path, [string] $Key)
+	foreach ($Line in Get-Content $Path) {
+		if ($Line.StartsWith("export $Key=")) {
+			return $Line.Substring("export $Key=".Length)
+		}
+	}
+	throw "Couldn't find value for $Key in $Path"
+}
+
+function Download-Node {
+	if (Test-Path $NodeTarget -PathType Leaf) {
+		return
+	}
+	Write-Output "Downloading Node v$NodeVersion (may take a while)"
+	New-Item $NodeTargetDir -ItemType Directory -ErrorAction silentlyContinue | Out-Null
+	$WebClient = New-Object Net.WebClient
+	$WebClient.DownloadFile($NodeSource, "$NodeTarget.downloading")
+	Rename-Item "$NodeTarget.downloading" $NodeTarget
+}
+
+## Convenience variables
+$BaseDir = Split-Path $script:MyInvocation.MyCommand.Path
+$Cache = "$BaseDir\.cache"
+if ($Env:TG_BOOTSTRAP_CACHE) {
+	$Cache = $Env:TG_BOOTSTRAP_CACHE
+}
+$NodeVersion = Extract-Variable -Path "$BaseDir\..\..\_build_dependencies.sh" -Key "NODE_VERSION_PRECISE"
+$NodeSource = "https://nodejs.org/download/release/v$NodeVersion/win-x64/node.exe"
+$NodeTargetDir = "$Cache\node-v$NodeVersion-x64"
+$NodeTarget = "$NodeTargetDir\node.exe"
+
+## Just print the path and exit
+if ($Args.length -eq 1 -and $Args[0] -eq "Get-Path") {
+	Write-Output "$NodeTargetDir"
+	exit 0
+}
+
+## Just download node and exit
+if ($Args.length -eq 1 -and $Args[0] -eq "Download-Node") {
+	Download-Node
+	exit 0
+}
+
+## Download node
+Download-Node
+
+## Set PATH so that recursive calls find it
+$Env:PATH = "$NodeTargetDir;$ENV:Path"
+
+## Invoke Node with all command-line arguments
+$ErrorActionPreference = "Continue"
+& "$NodeTarget" @Args
+exit $LastExitCode

tools/bootstrap/python

@@ -0,0 +1,118 @@
+#!/bin/sh
+# bootstrap/python
+#
+# Python-finding script for all `sh` environments, including Linux, MSYS2,
+# Git for Windows, and GitHub Desktop. Invokable from CLI or automation.
+#
+# If a python.exe installed by `python_.ps1` is present, it will be used.
+# Otherwise, this script requires a system `python3` and `pip` to be provided,
+# and will create a standard virtualenv in which to install `requirements.txt`.
+set -e
+
+# Convenience variables
+Bootstrap="$(dirname "$0")"
+Sdk="$(dirname "$Bootstrap")"
+Cache="$Bootstrap/.cache"
+if [ "$TG_BOOTSTRAP_CACHE" ]; then
+	Cache="$TG_BOOTSTRAP_CACHE"
+fi
+OldPWD="$PWD"
+cd "$Bootstrap/../.."
+. ./_build_dependencies.sh  # sets PYTHON_VERSION
+cd "$OldPWD"
+PythonVersion="$PYTHON_VERSION"
+PythonDir="$Cache/python-$PythonVersion"
+PythonExe="$PythonDir/python.exe"
+Log="$Cache/last-command.log"
+
+# If a portable Python for Windows is not present, search on $PATH.
+if [ "$(uname)" = "Linux" ] || [ ! -f "$PythonExe" ]; then
+	# Strip the "App Execution Aliases" from $PATH. Even if the user installed
+	# Python using the Windows Store on purpose, these aliases always generate
+	# "Permission denied" errors when sh.exe tries to invoke them.
+	PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":")
+
+	# Try to find a Python executable.
+	if command -v python3 >/dev/null 2>&1; then
+		PythonExe=python3
+	elif command -v python >/dev/null 2>&1; then
+		PythonExe=python
+	elif command -v py >/dev/null 2>&1; then
+		PythonExe="py -3"
+	else
+		echo
+		if command -v apt-get >/dev/null 2>&1; then
+			echo "Please install Python using your system's package manager:"
+			echo "    sudo apt-get install python3 python3-pip"
+		elif [ "$(uname -o)" = "Msys" ]; then
+			echo "Please run tools/bootstrap/python.bat instead of tools/bootstrap/python once to"
+			echo "install Python automatically, or install it from https://www.python.org/downloads/"
+			# TODO: give MSYS pacman advice?
+		elif command -v pacman >/dev/null 2>&1; then
+			echo "Please install Python using your system's package manager:"
+			echo "    sudo pacman -S python python-pip"
+		else
+			echo "Please install Python from https://www.python.org/downloads/ or using your system's package manager."
+		fi
+		echo
+		exit 1
+	fi
+
+	# Create a venv and activate it
+	PythonDir="$Cache/venv"
+	if [ ! -d "$PythonDir" ]; then
+		echo "Creating virtualenv..."
+		"$PythonExe" -m venv "$PythonDir"
+	fi
+	if [ -f "$PythonDir/bin/python" ]; then
+		PythonExe="$PythonDir/bin/python"
+	elif [ -f "$PythonDir/scripts/python3.exe" ]; then
+		PythonExe="$PythonDir/scripts/python3.exe";
+	else
+		echo "bootstrap/python failed to find the python executable inside its virtualenv"
+		exit 1
+	fi
+fi
+
+# Use pip to install our requirements
+if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then
+	echo "Updating dependencies..."
+	"$PythonExe" -m pip install -U wheel
+	"$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt"
+	cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt"
+	echo "---"
+fi
+
+# Verify version and deduce the path separator
+PythonMajor=${PythonVersion%%.*}
+PythonMinor=${PythonVersion#*.}
+PythonMinor=${PythonMinor%.*}
+PATHSEP=$("$PythonExe" - "$PythonMajor" "$PythonMinor" <<'EOF'
+import sys, os
+if sys.version_info.major != int(sys.argv[1]) or sys.version_info.minor < int(sys.argv[2]):
+	print("Error: Python ", sys.argv[1], ".", sys.argv[2], " or later is required, but you have:\n", sys.version, sep="", file=sys.stderr)
+	exit(1)
+print(os.pathsep)
+EOF
+)
+
+# Cheap shell function if tee.exe is not available
+if ! command -v tee >/dev/null 2>&1; then
+	tee() {
+		# Fudge: assume $1 is always "-a"
+		while read -r line; do
+			echo "$line" >> "$2"
+			echo "$line"
+		done
+	}
+fi
+
+# Invoke python with all command-line arguments
+export PYTHONPATH="$Sdk$PATHSEP${PYTHONPATH:-}"
+mkdir -p "$Cache"
+printf '%s\n' "$PythonExe" "$@" > "$Log"
+printf -- '---\n' >> "$Log"
+exec 4>&1
+exitstatus=$({ { set +e; "$PythonExe" -u "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1)
+exec 4>&-
+exit "$exitstatus"

tools/bootstrap/python.bat

@@ -0,0 +1,2 @@
+@echo off
+call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\python_.ps1" %*

tools/bootstrap/python_.ps1

@@ -0,0 +1,108 @@
+# bootstrap/python_.ps1
+#
+# Python bootstrapping script for Windows.
+#
+# Automatically downloads a portable edition of a pinned Python version to
+# a cache directory, installs Pip, installs `requirements.txt`, and then invokes
+# Python.
+#
+# The underscore in the name is so that typing `bootstrap/python` into
+# PowerShell finds the `.bat` file first, which ensures this script executes
+# regardless of ExecutionPolicy.
+$host.ui.RawUI.WindowTitle = "starting :: python $args"
+$ErrorActionPreference = "Stop"
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+function ExtractVersion {
+	param([string] $Path, [string] $Key)
+	foreach ($Line in Get-Content $Path) {
+		if ($Line.StartsWith("export $Key=")) {
+			return $Line.Substring("export $Key=".Length)
+		}
+	}
+	throw "Couldn't find value for $Key in $Path"
+}
+
+# Convenience variables
+$Bootstrap = Split-Path $script:MyInvocation.MyCommand.Path
+$Tools = Split-Path $Bootstrap
+$Cache = "$Bootstrap/.cache"
+if ($Env:TG_BOOTSTRAP_CACHE) {
+	$Cache = $Env:TG_BOOTSTRAP_CACHE
+}
+$PythonVersion = ExtractVersion -Path "$Bootstrap/../../_build_dependencies.sh" -Key "PYTHON_VERSION"
+$PythonDir = "$Cache/python-$PythonVersion"
+$PythonExe = "$PythonDir/python.exe"
+$Log = "$Cache/last-command.log"
+
+# Download and unzip a portable version of Python
+if (!(Test-Path $PythonExe -PathType Leaf)) {
+	$host.ui.RawUI.WindowTitle = "Downloading Python $PythonVersion..."
+	New-Item $Cache -ItemType Directory -ErrorAction silentlyContinue | Out-Null
+
+	$Archive = "$Cache/python-$PythonVersion-embed.zip"
+	Invoke-WebRequest `
+		"https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" `
+		-OutFile $Archive `
+		-ErrorAction Stop
+
+	[System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir)
+
+	$PythonVersionArray = $PythonVersion.Split(".")
+	$PythonVersionString = "python$($PythonVersionArray[0])$($PythonVersionArray[1])"
+	Write-Output "Generating PATH descriptor."
+	New-Item "$Cache/$PythonVersionString._pth" | Out-Null
+	Set-Content "$Cache/$PythonVersionString._pth" "$PythonVersionString.zip`n.`n..\..\..`nimport site`n"
+	# Copy a ._pth file without "import site" commented, so pip will work
+	Copy-Item "$Cache/$PythonVersionString._pth" $PythonDir `
+		-ErrorAction Stop
+
+	Remove-Item $Archive
+}
+
+# Install pip
+if (!(Test-Path "$PythonDir/Scripts/pip.exe")) {
+	$host.ui.RawUI.WindowTitle = "Downloading Pip..."
+
+	Invoke-WebRequest "https://bootstrap.pypa.io/get-pip.py" `
+		-OutFile "$Cache/get-pip.py" `
+		-ErrorAction Stop
+
+	& $PythonExe "$Cache/get-pip.py" --no-warn-script-location
+	if ($LASTEXITCODE -ne 0) {
+		exit $LASTEXITCODE
+	}
+
+	Remove-Item "$Cache/get-pip.py" `
+		-ErrorAction Stop
+}
+
+# Use pip to install our requirements
+if (!(Test-Path "$PythonDir/requirements.txt") -or ((Get-FileHash "$Tools/requirements.txt").hash -ne (Get-FileHash "$PythonDir/requirements.txt").hash)) {
+	$host.ui.RawUI.WindowTitle = "Updating dependencies..."
+
+	& $PythonExe -m pip install -U pip -r "$Tools/requirements.txt"
+	if ($LASTEXITCODE -ne 0) {
+		exit $LASTEXITCODE
+	}
+
+	Copy-Item "$Tools/requirements.txt" "$PythonDir/requirements.txt"
+	Write-Output "`n---`n"
+}
+
+# Invoke python with all command-line arguments
+Write-Output $PythonExe | Out-File -Encoding utf8 $Log
+[System.String]::Join([System.Environment]::NewLine, $args) | Out-File -Encoding utf8 -Append $Log
+Write-Output "---" | Out-File -Encoding utf8 -Append $Log
+$host.ui.RawUI.WindowTitle = "python $args"
+$ErrorActionPreference = "Continue"
+& $PythonExe -u $args 2>&1 | ForEach-Object {
+	$str = "$_"
+	if ($_.GetType() -eq [System.Management.Automation.ErrorRecord]) {
+		$str = $str.TrimEnd("`r`n")
+	}
+	$str | Out-File -Encoding utf8 -Append $Log
+	$str | Out-Host
+}
+exit $LastExitCode

tools/requirements.txt

@@ -0,0 +1,10 @@
+pygit2==1.7.2
+bidict==0.22.0
+Pillow==9.3.0
+
+# changelogs
+PyYaml==6.0.1
+beautifulsoup4==4.9.3
+
+# ezdb
+mysql-connector-python==8.0.33