building pipeline

This commit is contained in:
Letter N
2021-03-04 21:51:20 +08:00
parent 23c04cac12
commit dcab40cdb6
134 changed files with 10776 additions and 5991 deletions
+84
View File
@@ -0,0 +1,84 @@
#!/bin/sh
# bootstrap/node
#
# Node-finding script for all `sh` environments, including Linux, MSYS2,
# Git for Windows, and GitHub Desktop. Invokable from CLI or automation.
#
# If a node.exe installed by `node_.ps1` is present, it will be used.
# Otherwise, this script requires a system `node` to be provided.
set -e
# Convenience variables
Bootstrap="$(dirname "$0")"
Cache="$Bootstrap/.cache"
if [ "$TG_BOOTSTRAP_CACHE" ]; then
Cache="$TG_BOOTSTRAP_CACHE"
fi
OldPWD="$PWD"
cd "$Bootstrap/../.."
. ./dependencies.sh # sets NODE_VERSION_PRECISE
cd "$OldPWD"
NodeVersion="$NODE_VERSION_PRECISE"
NodeFullVersion="node-v$NodeVersion-win-x64"
NodeDir="$Cache/$NodeFullVersion"
NodeExe="$NodeDir/node.exe"
Log="$Cache/last-command.log"
# If a bootstrapped Node is not present, search on $PATH.
if [ "$(uname)" = "Linux" ] || [ ! -f "$NodeExe" ]; then
if [ "$TG_BOOTSTRAP_NODE_LINUX" ]; then
NodeFullVersion="node-v$NodeVersion-linux-x64"
NodeDir="$Cache/$NodeFullVersion/bin"
NodeExe="$NodeDir/node"
if [ ! -f "$NodeExe" ]; then
mkdir -p "$Cache"
Archive="$(realpath "$Cache/node-v$NodeVersion.tar.gz")"
curl "https://nodejs.org/download/release/v$NodeVersion/$NodeFullVersion.tar.gz" -o "$Archive"
(cd "$Cache" && tar xf "$Archive")
fi
elif command -v node >/dev/null 2>&1; then
NodeExe="node"
else
echo
if command -v apt-get >/dev/null 2>&1; then
# Ubuntu advice
echo "Please install Node using your system's package manager:"
echo " sudo apt-get install nodejs"
elif uname | grep -q MSYS; then
# MSYS2 (not packaged) or Git for Windows advice
echo "Please run bootstrap/node.bat instead of bootstrap/node once"
echo "to install Node automatically, or install it from https://nodejs.org/"
elif command -v pacman >/dev/null 2>&1; then
# Arch advice
echo "Please install Node using your system's package manager:"
echo " sudo pacman -S nodejs"
else
# Generic advice
echo "Please install Node from https://nodejs.org/ or using your system's package manager."
fi
echo
exit 1
fi
fi
# Cheap shell function if tee.exe is not available
if ! command -v tee >/dev/null 2>&1; then
tee() {
# Fudge: assume $1 is always "-a"
while read -r line; do
echo "$line" >> "$2"
echo "$line"
done
}
fi
# Invoke Node with all command-line arguments
mkdir -p "$Cache"
printf '%s\n' "$NodeExe" "$@" > "$Log"
printf -- '---\n' >> "$Log"
exec 4>&1
PATH="$(readlink -f "$NodeDir"):$PATH" # Set PATH so that recursive calls find it
exitstatus=$({ { set +e; "$NodeExe" "$@" 2>&1 3>&-; printf %s $? >&3; } 4>&- | tee -a "$Log" 1>&4; } 3>&1)
exec 4>&-
exit "$exitstatus"
+9
View File
@@ -0,0 +1,9 @@
@echo off
where node.exe >nul 2>nul
if %errorlevel% == 0 (
echo | set /p printed_str="Using system-wide Node "
call node.exe --version
call node.exe %*
) else (
call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" %*
)
+57
View File
@@ -0,0 +1,57 @@
## bootstrap/node_.ps1
##
## Node bootstrapping script for Windows.
##
## Automatically downloads a Node version to a cache directory and invokes it.
##
## The underscore in the name is so that typing `bootstrap/node` into
## PowerShell finds the `.bat` file first, which ensures this script executes
## regardless of ExecutionPolicy.
#Requires -Version 4.0
$Host.ui.RawUI.WindowTitle = "starting :: node $Args"
$ErrorActionPreference = "Stop"
## This forces UTF-8 encoding across all powershell built-ins
$OutputEncoding = [System.Console]::OutputEncoding = [System.Text.Encoding]::UTF8
$PSDefaultParameterValues['*:Encoding'] = 'utf8'
function ExtractVersion {
param([string] $Path, [string] $Key)
foreach ($Line in Get-Content $Path) {
if ($Line.StartsWith("export $Key=")) {
return $Line.Substring("export $Key=".Length)
}
}
throw "Couldn't find value for $Key in $Path"
}
## Convenience variables
$BaseDir = Split-Path $script:MyInvocation.MyCommand.Path
$Cache = "$BaseDir/.cache"
if ($Env:TG_BOOTSTRAP_CACHE) {
$Cache = $Env:TG_BOOTSTRAP_CACHE
}
$NodeVersion = ExtractVersion -Path "$BaseDir/../../dependencies.sh" -Key "NODE_VERSION_PRECISE"
$NodeDir = "$Cache/node-v$NodeVersion"
$NodeExe = "$NodeDir/node.exe"
## Download and unzip Node
if (!(Test-Path $NodeExe -PathType Leaf)) {
$Host.ui.RawUI.WindowTitle = "Downloading Node $NodeVersion..."
New-Item $NodeDir -ItemType Directory -ErrorAction silentlyContinue | Out-Null
Invoke-WebRequest `
"https://nodejs.org/download/release/v$NodeVersion/win-x86/node.exe" `
-OutFile $NodeExe `
-ErrorAction Stop
}
## Set PATH so that recursive calls find it
$Env:PATH = "$NodeDir;$ENV:Path"
## Invoke Node with all command-line arguments
$Host.ui.RawUI.WindowTitle = "node $Args"
$ErrorActionPreference = "Continue"
& "$NodeExe" @Args
exit $LastExitCode
+1
View File
@@ -77,6 +77,7 @@ fi
# Use pip to install our requirements
if [ ! -f "$PythonDir/requirements.txt" ] || [ "$(b2sum < "$Sdk/requirements.txt")" != "$(b2sum < "$PythonDir/requirements.txt")" ]; then
echo "Updating dependencies..."
"$PythonExe" -m pip install -U wheel
"$PythonExe" -m pip install -U pip -r "$Sdk/requirements.txt"
cp "$Sdk/requirements.txt" "$PythonDir/requirements.txt"
echo "---"
Regular → Executable
View File
+13
View File
@@ -0,0 +1,13 @@
# http://editorconfig.org
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.md]
max_line_length = 80
+21
View File
@@ -0,0 +1,21 @@
# /tg/station build script
This build script is the recommended way to compile the game, including not only the DM code but also the JavaScript and any other dependencies.
- VSCode: use `Ctrl+Shift+B` to build or `F5` to build and run.
- Windows: double-click `Build.bat` in the repository root to build.
- Linux: run `tools/build/build` from the repository root.
The script will skip build steps whose inputs have not changed since the last run.
## Dependencies
- On Windows, `Build.bat` will automatically install a private copy of Node.
- On Linux, install Node using your package manager or from <https://nodejs.org/en/download/>.
## Why?
We used to include compiled versions of the tgui JavaScript code in the Git repository so that the project could be compiled using BYOND only. These pre-compiled files tended to have merge conflicts for no good reason. Using a build script lets us avoid this problem, while keeping builds convenient for people who are not modifying tgui.
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -e
cd "$(dirname "$0")"
exec ../bootstrap/node build.js "$@"
+1
View File
@@ -0,0 +1 @@
@"%~dp0\..\bootstrap\node" "%~dp0\build.js"
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env node
/**
* @file
* @copyright 2020 Aleksej Komarov
* @license MIT
*/
const { resolve: resolvePath } = require('path');
const { resolveGlob } = require('./cbt/fs');
const { exec } = require('./cbt/process');
const { Task, runTasks } = require('./cbt/task');
const { regQuery } = require('./cbt/winreg');
// Change working directory to project root
process.chdir(resolvePath(__dirname, '../../'));
const taskTgui = new Task('tgui')
.depends('tgui/.yarn/releases/*')
.depends('tgui/yarn.lock')
.depends('tgui/webpack.config.js')
.depends('tgui/**/package.json')
.depends('tgui/packages/**/*.js')
.depends('tgui/packages/**/*.jsx')
.provides('tgui/public/tgui.bundle.css')
.provides('tgui/public/tgui.bundle.js')
.provides('tgui/public/tgui-common.chunk.js')
.provides('tgui/public/tgui-panel.bundle.css')
.provides('tgui/public/tgui-panel.bundle.js')
.build(async () => {
// Instead of calling `tgui/bin/tgui`, we reproduce the whole pipeline
// here for maximum compilation speed.
const yarnRelease = resolveGlob('./tgui/.yarn/releases/yarn-*.cjs')[0]
.replace('/tgui/', '/');
const yarn = args => exec('node', [yarnRelease, ...args], {
cwd: './tgui',
});
await yarn(['install']);
await yarn(['run', 'webpack-cli', '--mode=production']);
});
const taskDm = new Task('dm')
.depends('_maps/map_files/generic/**')
.depends('code/**')
.depends('goon/**')
.depends('html/**')
.depends('icons/**')
.depends('interface/**')
.depends('tgui/public/tgui.html')
.depends('tgui/public/*.bundle.*')
.depends('tgui/public/*.chunk.*')
.depends('tgstation.dme')
.provides('tgstation.dmb')
.provides('tgstation.rsc')
.build(async () => {
let compiler = 'dm';
// Let's do some registry queries on Windows, because dm is not in PATH.
if (process.platform === 'win32') {
const installPath = (
await regQuery(
'HKLM\\Software\\Dantom\\BYOND',
'installpath')
|| await regQuery(
'HKLM\\SOFTWARE\\WOW6432Node\\Dantom\\BYOND',
'installpath')
);
if (installPath) {
compiler = resolvePath(installPath, 'bin/dm.exe');
}
} else {
compiler = 'DreamMaker';
}
await exec(compiler, ['tgstation.dme']);
});
// Frontend
const tasksToRun = [
taskTgui,
taskDm,
];
if (process.env['TG_BUILD_TGS_MODE']) {
tasksToRun.pop();
}
runTasks(tasksToRun);
+136
View File
@@ -0,0 +1,136 @@
/**
* @file
* @copyright 2020 Aleksej Komarov
* @license MIT
*/
const fs = require('fs');
const glob = require('./glob');
class File {
constructor(path) {
this.path = path;
}
get stat() {
if (this._stat === undefined) {
this._stat = stat(this.path);
}
return this._stat;
}
exists() {
return this.stat !== null;
}
get mtime() {
return this.stat && this.stat.mtime;
}
touch() {
const time = new Date();
try {
fs.utimesSync(this.path, time, time);
}
catch (err) {
fs.closeSync(fs.openSync(this.path, 'w'));
}
}
}
class Glob {
constructor(path) {
this.path = path;
}
toFiles() {
const paths = glob.sync(this.path, {
strict: false,
silent: true,
});
return paths
.map(path => new File(path))
.filter(file => file.exists());
}
}
/**
* If true, source is newer than target.
* @param {File[]} sources
* @param {File[]} targets
*/
const compareFiles = (sources, targets) => {
let bestSource = null;
let bestTarget = null;
for (const file of sources) {
if (!bestSource || file.mtime > bestSource.mtime) {
bestSource = file;
}
}
for (const file of targets) {
if (!file.exists()) {
return `target '${file.path}' is missing`;
}
if (!bestTarget || file.mtime < bestTarget.mtime) {
bestTarget = file;
}
}
// Doesn't need rebuild if there is no source, but target exists.
if (!bestSource) {
if (bestTarget) {
return false;
}
return 'no known sources or targets';
}
// Always needs a rebuild if no targets were specified (e.g. due to GLOB).
if (!bestTarget) {
return 'no targets were specified';
}
// Needs rebuild if source is newer than target
if (bestSource.mtime > bestTarget.mtime) {
return `source '${bestSource.path}' is newer than target '${bestTarget.path}'`;
}
return false;
};
/**
* Returns file stats for the provided path, or null if file is
* not accessible.
*/
const stat = path => {
try {
return fs.statSync(path);
}
catch {
return null;
}
};
/**
* Resolves a glob pattern and returns files that are safe
* to call `stat` on.
*/
const resolveGlob = globPath => {
const unsafePaths = glob.sync(globPath, {
strict: false,
silent: true,
});
const safePaths = [];
for (let path of unsafePaths) {
try {
fs.statSync(path);
safePaths.push(path);
}
catch {}
}
return safePaths;
};
module.exports = {
File,
Glob,
compareFiles,
stat,
resolveGlob,
compareFiles,
};
File diff suppressed because it is too large Load Diff
+104
View File
@@ -0,0 +1,104 @@
/**
* @file
* @copyright 2020 Aleksej Komarov
* @license MIT
*/
const { spawn } = require('child_process');
const { resolve: resolvePath } = require('path');
const { stat } = require('./fs');
/**
* @type {import('child_process').ChildProcessWithoutNullStreams}
*/
const children = new Set();
const killChildren = () => {
for (const child of children) {
child.kill('SIGTERM');
children.delete(child);
console.log('killed child process');
}
};
const trap = (signals, handler) => {
let readline;
if (process.platform === 'win32') {
readline = require('readline').createInterface({
input: process.stdin,
output: process.stdout,
});
}
for (const signal of signals) {
const handleSignal = () => handler(signal);
if (signal === 'EXIT') {
process.on('exit', handleSignal);
continue;
}
if (readline) {
readline.on('SIG' + signal, handleSignal);
}
process.on('SIG' + signal, handleSignal);
}
};
trap(['EXIT', 'BREAK', 'HUP', 'INT', 'TERM'], signal => {
if (signal !== 'EXIT') {
console.log('Received', signal);
}
killChildren();
if (signal !== 'EXIT') {
process.exit(1);
}
});
const exceptionHandler = err => {
console.log(err);
killChildren();
process.exit(1);
};
process.on('unhandledRejection', exceptionHandler);
process.on('uncaughtException', exceptionHandler);
class ExitError extends Error {}
/**
* @param {string} executable
* @param {string[]} args
* @param {import('child_process').SpawnOptionsWithoutStdio} options
*/
const exec = (executable, args, options) => {
return new Promise((resolve, reject) => {
// If executable exists relative to the current directory,
// use that executable, otherwise spawn should fall back to
// running it from PATH.
if (stat(executable)) {
executable = resolvePath(executable);
}
const child = spawn(executable, args, options);
children.add(child);
child.stdout.on('data', data => {
process.stdout.write(data);
});
child.stderr.on('data', data => {
process.stderr.write(data);
});
child.stdin.end();
child.on('exit', code => {
children.delete(child);
if (code !== 0) {
const error = new ExitError('Process exited with code: ' + code);
error.code = code;
reject(error);
}
else {
resolve(code);
}
});
});
};
module.exports = {
exec,
};
+104
View File
@@ -0,0 +1,104 @@
/**
* @file
* @copyright 2020 Aleksej Komarov
* @license MIT
*/
const { compareFiles, Glob, File } = require('./fs');
class Task {
constructor(name) {
this.name = name;
this.sources = [];
this.targets = [];
this.script = null;
}
depends(path) {
if (path.includes('*')) {
this.sources.push(new Glob(path));
}
else {
this.sources.push(new File(path));
}
return this;
}
provides(path) {
if (path.includes('*')) {
this.targets.push(new Glob(path));
}
else {
this.targets.push(new File(path));
}
return this;
}
build(script) {
this.script = script;
return this;
}
async run() {
/**
* @returns {File[]}
*/
const getFiles = files => files
.flatMap(file => {
if (file instanceof Glob) {
return file.toFiles();
}
if (file instanceof File) {
return file;
}
})
.filter(Boolean);
// Normalize all our dependencies by converting globs to files
const fileSources = getFiles(this.sources);
const fileTargets = getFiles(this.targets);
// Consider dependencies first, and skip the task if it
// doesn't need a rebuild.
let needsRebuild = 'no targets';
if (fileTargets.length > 0) {
needsRebuild = compareFiles(fileSources, fileTargets);
if (!needsRebuild) {
console.warn(` => Skipping '${this.name}' (up to date)`);
return;
}
}
if (!this.script) {
return;
}
console.warn(` => Starting '${this.name}'`);
const startedAt = Date.now();
// Run the script
await this.script();
// Touch all targets so that they don't rebuild again
if (fileTargets.length > 0) {
for (const file of fileTargets) {
file.touch();
}
}
const time = ((Date.now() - startedAt) / 1000) + 's';
console.warn(` => Finished '${this.name}' in ${time}`);
}
}
const runTasks = async tasks => {
const startedAt = Date.now();
// Run all if none of the tasks were specified in command line
const runAll = !tasks.some(task => process.argv.includes(task.name));
for (const task of tasks) {
if (runAll || process.argv.includes(task.name)) {
await task.run();
}
}
const time = ((Date.now() - startedAt) / 1000) + 's';
console.log(` => Done in ${time}`);
process.exit();
};
module.exports = {
Task,
runTasks,
};
+46
View File
@@ -0,0 +1,46 @@
/**
* Tools for dealing with Windows Registry bullshit.
*
* Adapted from `tgui/packages/tgui-dev-server/winreg.js`.
*
* @file
* @copyright 2020 Aleksej Komarov
* @license MIT
*/
const { exec } = require('child_process');
const { promisify } = require('util');
const regQuery = async (path, key) => {
if (process.platform !== 'win32') {
return null;
}
try {
const command = `reg query "${path}" /v ${key}`;
const { stdout } = await promisify(exec)(command);
const keyPattern = ` ${key} `;
const indexOfKey = stdout.indexOf(keyPattern);
if (indexOfKey === -1) {
return null;
}
const indexOfEol = stdout.indexOf('\r\n', indexOfKey);
if (indexOfEol === -1) {
return null;
}
const indexOfValue = stdout.indexOf(
' ',
indexOfKey + keyPattern.length);
if (indexOfValue === -1) {
return null;
}
const value = stdout.substring(indexOfValue + 4, indexOfEol);
return value;
}
catch (err) {
return null;
}
};
module.exports = {
regQuery,
};
+15
View File
@@ -0,0 +1,15 @@
@echo off
cd /D "%~dp0"
set TG_BOOTSTRAP_CACHE=%cd%
IF NOT "%1" == "" (
rem TGS4: we are passed the game directory on the command line
cd %1
) ELSE IF EXIST "..\Game\B\tgstation.dmb" (
rem TGS3: Game/B/tgstation.dmb exists, so build in Game/A
cd ..\Game\A
) ELSE (
rem TGS3: Otherwise build in Game/B
cd ..\Game\B
)
set TG_BUILD_TGS_MODE=1
tools\build\build
+8 -2
View File
@@ -61,6 +61,12 @@ fi
echo "Deploying rust-g..."
git checkout "$RUST_G_VERSION"
~/.cargo/bin/cargo build --release --target=i686-unknown-linux-gnu
mv target/i686-unknown-linux-gnu/release/librust_g.so "$1/rust_g"
env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --release --target=i686-unknown-linux-gnu
mv target/i686-unknown-linux-gnu/release/librust_g.so "$1/librust_g.so"
cd ..
# compile tgui
echo "Compiling tgui..."
cd "$1"
chmod +x tools/bootstrap/node # Workaround for https://github.com/tgstation/tgstation-server/issues/1167
env TG_BOOTSTRAP_CACHE="$original_dir" TG_BOOTSTRAP_NODE_LINUX=1 TG_BUILD_TGS_MODE=1 tools/bootstrap/node tools/build/build.js