diff --git a/.gitignore b/.gitignore index 518220dd6e..b521523030 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ #ignore misc BYOND files -Thumbs.db vchat.db vchat.db* *.log @@ -11,10 +10,61 @@ vchat.db* *.before *.pyc *.pid -data -data/ cfg/ +#Ignore everything in datafolder and subdirectories +/data/**/* +/tmp/**/* + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-*.txt + +# Unit test / coverage reports +.cache + +# pyenv +.python-version + +# dotenv +.env + +### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Global/Windows.gitignore + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows shortcuts +*.lnk + +### https://raw.github.com/github/gitignore/cc542de017c606138a87ee4880e5f06b3a306def/Global/OSX.gitignore + +.DS_Store +.AppleDouble +.LSOverride + #Visual studio stuff *.vscode/* !/.vscode/launch.json diff --git a/_build_dependencies.sh b/_build_dependencies.sh index b251bfbe32..7d019862f9 100644 --- a/_build_dependencies.sh +++ b/_build_dependencies.sh @@ -1,11 +1,17 @@ +#!/bin/sh + # This file has all the information on what versions of libraries are thrown into the code -# For dreamchecker -export SPACEMAN_DMM_VERSION=suite-1.7 -# For NanoUI + TGUI -export NODE_VERSION=16 -# Byond Major + +# byond version export BYOND_MAJOR=514 -# Byond Minor export BYOND_MINOR=1589 -# Macro Count export MACRO_COUNT=4 + +# node version +export NODE_VERSION=16 + +# SpacemanDMM git tag +export SPACEMAN_DMM_VERSION=suite-1.7 + +# Python version for mapmerge and other tools +export PYTHON_VERSION=3.9.0 diff --git a/tools/bootstrap/node b/tools/bootstrap/node new file mode 100644 index 0000000000..733142d77c --- /dev/null +++ b/tools/bootstrap/node @@ -0,0 +1,73 @@ +#!/bin/sh +# bootstrap/node +# +# Node-finding script for all `sh` environments, including Linux, MSYS2, +# Git for Windows, and GitHub Desktop. Invokable from CLI or automation. +# +# If a node.exe installed by `node_.ps1` is present, it will be used. +# Otherwise, this script requires a system `node` to be provided. +set -e + +# Convenience variables +Bootstrap="$(dirname "$0")" +Cache="$Bootstrap/.cache" +if [ "$TG_BOOTSTRAP_CACHE" ]; then + Cache="$TG_BOOTSTRAP_CACHE" +fi +OldPWD="$PWD" +cd "$Bootstrap/../.." +. ./_build_dependencies.sh # sets NODE_VERSION_PRECISE +cd "$OldPWD" +NodeVersion="$NODE_VERSION_PRECISE" +NodeFullVersion="node-v$NodeVersion-win-x64" +NodeDir="$Cache/$NodeFullVersion" +NodeExe="$NodeDir/node.exe" +is_vendored="1" + +# If a bootstrapped Node is not present, search on $PATH. +if [ "$(uname)" = "Linux" ] || [ ! -f "$NodeExe" ]; then + if [ "$TG_BOOTSTRAP_NODE_LINUX" ]; then + NodeFullVersion="node-v$NodeVersion-linux-x64" + NodeDir="$Cache/$NodeFullVersion/bin" + NodeExe="$NodeDir/node" + + if [ ! -f "$NodeExe" ]; then + mkdir -p "$Cache" + Archive="$(realpath "$Cache/node-v$NodeVersion.tar.gz")" + curl "https://nodejs.org/download/release/v$NodeVersion/$NodeFullVersion.tar.gz" -o "$Archive" + (cd "$Cache" && tar xf "$Archive") + fi + elif command -v node >/dev/null 2>&1; then + NodeExe="node" + is_vendored="0" + else + echo + if command -v apt-get >/dev/null 2>&1; then + # Ubuntu advice + echo "Please install Node using your system's package manager:" + echo " sudo apt-get install nodejs" + elif uname | grep -q MSYS; then + # MSYS2 (not packaged) or Git for Windows advice + echo "Please run bootstrap/node.bat instead of bootstrap/node once" + echo "to install Node automatically, or install it from https://nodejs.org/" + elif command -v pacman >/dev/null 2>&1; then + # Arch advice + echo "Please install Node using your system's package manager:" + echo " sudo pacman -S nodejs" + else + # Generic advice + echo "Please install Node from https://nodejs.org/ or using your system's package manager." + fi + echo + exit 1 + fi +fi + +# Invoke Node with all command-line arguments +if [ "$is_vendored" = "1" ]; then + PATH="$(readlink -f "$NodeDir"):$PATH" + echo "Using vendored Node $("$NodeExe" --version)" +else + echo "Using system-wide Node $("$NodeExe" --version)" +fi +exec "$NodeExe" "$@" diff --git a/tools/bootstrap/node.bat b/tools/bootstrap/node.bat new file mode 100644 index 0000000000..a3fca88674 --- /dev/null +++ b/tools/bootstrap/node.bat @@ -0,0 +1,20 @@ +@echo off +set NODE_SKIP_PLATFORM_CHECK=1 +call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Download-Node +for /f "tokens=* USEBACKQ" %%s in (` + call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Get-Path +`) do ( + set "PATH=%%s;%PATH%" +) +where node.exe >nul 2>nul +if %errorlevel% == 0 ( + echo | set /p printed_str="Using vendored Node " + call node.exe --version + call node.exe %* + goto exit_with_last_error_level +) +echo "node.bat: Failed to bootstrap Node!" +%COMSPEC% /c exit 1 + +:exit_with_last_error_level +if not %errorlevel% == 0 %COMSPEC% /c exit %errorlevel% >nul diff --git a/tools/bootstrap/node_.ps1 b/tools/bootstrap/node_.ps1 new file mode 100644 index 0000000000..ae48aa7de5 --- /dev/null +++ b/tools/bootstrap/node_.ps1 @@ -0,0 +1,59 @@ +## bootstrap/node_.ps1 +## Downloads a Node version to a cache directory and invokes it. + +$ErrorActionPreference = "Stop" + +function Extract-Variable { + param([string] $Path, [string] $Key) + foreach ($Line in Get-Content $Path) { + if ($Line.StartsWith("export $Key=")) { + return $Line.Substring("export $Key=".Length) + } + } + throw "Couldn't find value for $Key in $Path" +} + +function Download-Node { + if (Test-Path $NodeTarget -PathType Leaf) { + return + } + Write-Output "Downloading Node v$NodeVersion (may take a while)" + New-Item $NodeTargetDir -ItemType Directory -ErrorAction silentlyContinue | Out-Null + $WebClient = New-Object Net.WebClient + $WebClient.DownloadFile($NodeSource, "$NodeTarget.downloading") + Rename-Item "$NodeTarget.downloading" $NodeTarget +} + +## Convenience variables +$BaseDir = Split-Path $script:MyInvocation.MyCommand.Path +$Cache = "$BaseDir\.cache" +if ($Env:TG_BOOTSTRAP_CACHE) { + $Cache = $Env:TG_BOOTSTRAP_CACHE +} +$NodeVersion = Extract-Variable -Path "$BaseDir\..\..\_build_dependencies.sh" -Key "NODE_VERSION_PRECISE" +$NodeSource = "https://nodejs.org/download/release/v$NodeVersion/win-x64/node.exe" +$NodeTargetDir = "$Cache\node-v$NodeVersion-x64" +$NodeTarget = "$NodeTargetDir\node.exe" + +## Just print the path and exit +if ($Args.length -eq 1 -and $Args[0] -eq "Get-Path") { + Write-Output "$NodeTargetDir" + exit 0 +} + +## Just download node and exit +if ($Args.length -eq 1 -and $Args[0] -eq "Download-Node") { + Download-Node + exit 0 +} + +## Download node +Download-Node + +## Set PATH so that recursive calls find it +$Env:PATH = "$NodeTargetDir;$ENV:Path" + +## Invoke Node with all command-line arguments +$ErrorActionPreference = "Continue" +& "$NodeTarget" @Args +exit $LastExitCode diff --git a/tools/bootstrap/python b/tools/bootstrap/python new file mode 100644 index 0000000000..d5e031b8aa --- /dev/null +++ b/tools/bootstrap/python @@ -0,0 +1,118 @@ +#!/bin/sh +# bootstrap/python +# +# Python-finding script for all `sh` environments, including Linux, MSYS2, +# Git for Windows, and GitHub Desktop. Invokable from CLI or automation. +# +# If a python.exe installed by `python_.ps1` is present, it will be used. +# Otherwise, this script requires a system `python3` and `pip` to be provided, +# and will create a standard virtualenv in which to install `requirements.txt`. +set -e + +# Convenience variables +Bootstrap="$(dirname "$0")" +Sdk="$(dirname "$Bootstrap")" +Cache="$Bootstrap/.cache" +if [ "$TG_BOOTSTRAP_CACHE" ]; then + Cache="$TG_BOOTSTRAP_CACHE" +fi +OldPWD="$PWD" +cd "$Bootstrap/../.." +. ./_build_dependencies.sh # sets PYTHON_VERSION +cd "$OldPWD" +PythonVersion="$PYTHON_VERSION" +PythonDir="$Cache/python-$PythonVersion" +PythonExe="$PythonDir/python.exe" +Log="$Cache/last-command.log" + +# If a portable Python for Windows is not present, search on $PATH. +if [ "$(uname)" = "Linux" ] || [ ! -f "$PythonExe" ]; then + # Strip the "App Execution Aliases" from $PATH. Even if the user installed + # Python using the Windows Store on purpose, these aliases always generate + # "Permission denied" errors when sh.exe tries to invoke them. + PATH=$(echo "$PATH" | tr ":" "\n" | grep -v "AppData/Local/Microsoft/WindowsApps" | tr "\n" ":") + + # Try to find a Python executable. + if command -v python3 >/dev/null 2>&1; then + PythonExe=python3 + elif command -v python >/dev/null 2>&1; then + PythonExe=python + elif command -v py >/dev/null 2>&1; then + PythonExe="py -3" + else + echo + if command -v apt-get >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo apt-get install python3 python3-pip" + elif [ "$(uname -o)" = "Msys" ]; then + echo "Please run tools/bootstrap/python.bat instead of tools/bootstrap/python once to" + echo "install Python automatically, or install it from https://www.python.org/downloads/" + # TODO: give MSYS pacman advice? + elif command -v pacman >/dev/null 2>&1; then + echo "Please install Python using your system's package manager:" + echo " sudo pacman -S python python-pip" + else + echo "Please install Python from https://www.python.org/downloads/ or using your system's package manager." + fi + echo + exit 1 + fi + + # Create a venv and activate it + PythonDir="$Cache/venv" + if [ ! -d "$PythonDir" ]; then + echo "Creating virtualenv..." + "$PythonExe" -m venv "$PythonDir" + fi + if [ -f "$PythonDir/bin/python" ]; then + PythonExe="$PythonDir/bin/python" + elif [ -f "$PythonDir/scripts/python3.exe" ]; then + PythonExe="$PythonDir/scripts/python3.exe"; + else + echo "bootstrap/python failed to find the python executable inside its virtualenv" + exit 1 + fi +fi + +# Use pip to install our requirements +if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then + echo "Updating dependencies..." + "$PythonExe" -m pip install -U wheel + "$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt" + cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt" + echo "---" +fi + +# Verify version and deduce the path separator +PythonMajor=${PythonVersion%%.*} +PythonMinor=${PythonVersion#*.} +PythonMinor=${PythonMinor%.*} +PATHSEP=$("$PythonExe" - "$PythonMajor" "$PythonMinor" <<'EOF' +import sys, os +if sys.version_info.major != int(sys.argv[1]) or sys.version_info.minor < int(sys.argv[2]): + print("Error: Python ", sys.argv[1], ".", sys.argv[2], " or later is required, but you have:\n", sys.version, sep="", file=sys.stderr) + exit(1) +print(os.pathsep) +EOF +) + +# Cheap shell function if tee.exe is not available +if ! command -v tee >/dev/null 2>&1; then + tee() { + # Fudge: assume $1 is always "-a" + while read -r line; do + echo "$line" >> "$2" + echo "$line" + done + } +fi + +# Invoke python with all command-line arguments +export PYTHONPATH="$Sdk$PATHSEP${PYTHONPATH:-}" +mkdir -p "$Cache" +printf '%s\n' "$PythonExe" "$@" > "$Log" +printf -- '---\n' >> "$Log" +exec 4>&1 +exitstatus=$({ { set +e; "$PythonExe" -u "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1) +exec 4>&- +exit "$exitstatus" diff --git a/tools/bootstrap/python.bat b/tools/bootstrap/python.bat new file mode 100644 index 0000000000..5bec61b84c --- /dev/null +++ b/tools/bootstrap/python.bat @@ -0,0 +1,2 @@ +@echo off +call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\python_.ps1" %* diff --git a/tools/bootstrap/python_.ps1 b/tools/bootstrap/python_.ps1 new file mode 100644 index 0000000000..ed946392d6 --- /dev/null +++ b/tools/bootstrap/python_.ps1 @@ -0,0 +1,108 @@ +# bootstrap/python_.ps1 +# +# Python bootstrapping script for Windows. +# +# Automatically downloads a portable edition of a pinned Python version to +# a cache directory, installs Pip, installs `requirements.txt`, and then invokes +# Python. +# +# The underscore in the name is so that typing `bootstrap/python` into +# PowerShell finds the `.bat` file first, which ensures this script executes +# regardless of ExecutionPolicy. +$host.ui.RawUI.WindowTitle = "starting :: python $args" +$ErrorActionPreference = "Stop" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function ExtractVersion { + param([string] $Path, [string] $Key) + foreach ($Line in Get-Content $Path) { + if ($Line.StartsWith("export $Key=")) { + return $Line.Substring("export $Key=".Length) + } + } + throw "Couldn't find value for $Key in $Path" +} + +# Convenience variables +$Bootstrap = Split-Path $script:MyInvocation.MyCommand.Path +$Tools = Split-Path $Bootstrap +$Cache = "$Bootstrap/.cache" +if ($Env:TG_BOOTSTRAP_CACHE) { + $Cache = $Env:TG_BOOTSTRAP_CACHE +} +$PythonVersion = ExtractVersion -Path "$Bootstrap/../../_build_dependencies.sh" -Key "PYTHON_VERSION" +$PythonDir = "$Cache/python-$PythonVersion" +$PythonExe = "$PythonDir/python.exe" +$Log = "$Cache/last-command.log" + +# Download and unzip a portable version of Python +if (!(Test-Path $PythonExe -PathType Leaf)) { + $host.ui.RawUI.WindowTitle = "Downloading Python $PythonVersion..." + New-Item $Cache -ItemType Directory -ErrorAction silentlyContinue | Out-Null + + $Archive = "$Cache/python-$PythonVersion-embed.zip" + Invoke-WebRequest ` + "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip" ` + -OutFile $Archive ` + -ErrorAction Stop + + [System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir) + + $PythonVersionArray = $PythonVersion.Split(".") + $PythonVersionString = "python$($PythonVersionArray[0])$($PythonVersionArray[1])" + Write-Output "Generating PATH descriptor." + New-Item "$Cache/$PythonVersionString._pth" | Out-Null + Set-Content "$Cache/$PythonVersionString._pth" "$PythonVersionString.zip`n.`n..\..\..`nimport site`n" + # Copy a ._pth file without "import site" commented, so pip will work + Copy-Item "$Cache/$PythonVersionString._pth" $PythonDir ` + -ErrorAction Stop + + Remove-Item $Archive +} + +# Install pip +if (!(Test-Path "$PythonDir/Scripts/pip.exe")) { + $host.ui.RawUI.WindowTitle = "Downloading Pip..." + + Invoke-WebRequest "https://bootstrap.pypa.io/get-pip.py" ` + -OutFile "$Cache/get-pip.py" ` + -ErrorAction Stop + + & $PythonExe "$Cache/get-pip.py" --no-warn-script-location + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Remove-Item "$Cache/get-pip.py" ` + -ErrorAction Stop +} + +# Use pip to install our requirements +if (!(Test-Path "$PythonDir/requirements.txt") -or ((Get-FileHash "$Tools/requirements.txt").hash -ne (Get-FileHash "$PythonDir/requirements.txt").hash)) { + $host.ui.RawUI.WindowTitle = "Updating dependencies..." + + & $PythonExe -m pip install -U pip -r "$Tools/requirements.txt" + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Copy-Item "$Tools/requirements.txt" "$PythonDir/requirements.txt" + Write-Output "`n---`n" +} + +# Invoke python with all command-line arguments +Write-Output $PythonExe | Out-File -Encoding utf8 $Log +[System.String]::Join([System.Environment]::NewLine, $args) | Out-File -Encoding utf8 -Append $Log +Write-Output "---" | Out-File -Encoding utf8 -Append $Log +$host.ui.RawUI.WindowTitle = "python $args" +$ErrorActionPreference = "Continue" +& $PythonExe -u $args 2>&1 | ForEach-Object { + $str = "$_" + if ($_.GetType() -eq [System.Management.Automation.ErrorRecord]) { + $str = $str.TrimEnd("`r`n") + } + $str | Out-File -Encoding utf8 -Append $Log + $str | Out-Host +} +exit $LastExitCode diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000000..229ea29a4d --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,10 @@ +pygit2==1.7.2 +bidict==0.22.0 +Pillow==9.3.0 + +# changelogs +PyYaml==6.0.1 +beautifulsoup4==4.9.3 + +# ezdb +mysql-connector-python==8.0.33