okay

tools/bootstrap/node.bat

@@ -4,6 +4,23 @@ if %errorlevel% == 0 (
 	echo | set /p printed_str="Using system-wide Node "
 	call node.exe --version
 	call node.exe %*
-) else (
-	call powershell.exe -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" %*
+	goto exit_with_last_error_level
 )
+call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Download-Node
+for /f "tokens=* USEBACKQ" %%s in (`
+	call powershell -NoLogo -ExecutionPolicy Bypass -File "%~dp0\node_.ps1" Get-Path
+`) do (
+	set "PATH=%%s;%PATH%"
+)
+where node.exe >nul 2>nul
+if %errorlevel% == 0 (
+	echo | set /p printed_str="Using vendored Node "
+	call node.exe --version
+	call node.exe %*
+	goto exit_with_last_error_level
+)
+echo "build.bat: Failed to bootstrap Node!"
+%COMSPEC% /c exit 1
+
+:exit_with_last_error_level
+if not %errorlevel% == 0 %COMSPEC% /c exit %errorlevel% >nul

tools/bootstrap/node_.ps1

@@ -1,23 +1,9 @@
 ## bootstrap/node_.ps1
-##
-## Node bootstrapping script for Windows.
-##
-## Automatically downloads a Node version to a cache directory and invokes it.
-##
-## The underscore in the name is so that typing `bootstrap/node` into
-## PowerShell finds the `.bat` file first, which ensures this script executes
-## regardless of ExecutionPolicy.
+## Downloads a Node version to a cache directory and invokes it.
 
-#Requires -Version 4.0
-
-$Host.ui.RawUI.WindowTitle = "starting :: node $Args"
 $ErrorActionPreference = "Stop"
 
-## This forces UTF-8 encoding across all powershell built-ins
-$OutputEncoding = [System.Console]::OutputEncoding = [System.Text.Encoding]::UTF8
-$PSDefaultParameterValues['*:Encoding'] = 'utf8'
-
-function ExtractVersion {
+function Extract-Variable {
 	param([string] $Path, [string] $Key)
 	foreach ($Line in Get-Content $Path) {
 		if ($Line.StartsWith("export $Key=")) {
@@ -27,31 +13,46 @@ function ExtractVersion {
 	throw "Couldn't find value for $Key in $Path"
 }
 
+function Download-Node {
+	if (Test-Path $NodeTarget -PathType Leaf) {
+		return
+	}
+	Write-Output "Downloading Node v$NodeVersion (may take a while)"
+	New-Item $NodeTargetDir -ItemType Directory -ErrorAction silentlyContinue | Out-Null
+	$WebClient = New-Object Net.WebClient
+	$WebClient.DownloadFile($NodeSource, $NodeTarget)
+}
+
 ## Convenience variables
 $BaseDir = Split-Path $script:MyInvocation.MyCommand.Path
-$Cache = "$BaseDir/.cache"
+$Cache = "$BaseDir\.cache"
 if ($Env:TG_BOOTSTRAP_CACHE) {
 	$Cache = $Env:TG_BOOTSTRAP_CACHE
 }
-$NodeVersion = ExtractVersion -Path "$BaseDir/../../dependencies.sh" -Key "NODE_VERSION_PRECISE"
-$NodeDir = "$Cache/node-v$NodeVersion"
-$NodeExe = "$NodeDir/node.exe"
+$NodeVersion = Extract-Variable -Path "$BaseDir\..\..\dependencies.sh" -Key "NODE_VERSION_PRECISE"
+$NodeSource = "https://nodejs.org/download/release/v$NodeVersion/win-x86/node.exe"
+$NodeTargetDir = "$Cache\node-v$NodeVersion"
+$NodeTarget = "$NodeTargetDir\node.exe"
 
-## Download and unzip Node
-if (!(Test-Path $NodeExe -PathType Leaf)) {
-	$Host.ui.RawUI.WindowTitle = "Downloading Node $NodeVersion..."
-	New-Item $NodeDir -ItemType Directory -ErrorAction silentlyContinue | Out-Null
-	Invoke-WebRequest `
-		"https://nodejs.org/download/release/v$NodeVersion/win-x86/node.exe" `
-		-OutFile $NodeExe `
-		-ErrorAction Stop
+## Just print the path and exit
+if ($Args.length -eq 1 -and $Args[0] -eq "Get-Path") {
+	Write-Output "$NodeTargetDir"
+	exit 0
 }
 
+## Just download node and exit
+if ($Args.length -eq 1 -and $Args[0] -eq "Download-Node") {
+	Download-Node
+	exit 0
+}
+
+## Download node
+Download-Node
+
 ## Set PATH so that recursive calls find it
-$Env:PATH = "$NodeDir;$ENV:Path"
+$Env:PATH = "$NodeTargetDir;$ENV:Path"
 
 ## Invoke Node with all command-line arguments
-$Host.ui.RawUI.WindowTitle = "node $Args"
 $ErrorActionPreference = "Continue"
-& "$NodeExe" @Args
+& "$NodeTarget" @Args
 exit $LastExitCode

tools/build/README.md

@@ -2,19 +2,22 @@
 
 This build script is the recommended way to compile the game, including not only the DM code but also the JavaScript and any other dependencies.
 
-- VSCode: use `Ctrl+Shift+B` to build or `F5` to build and run.
-
-- Windows: double-click `Build.bat` in the repository root to build.
-
-- Linux: run `tools/build/build` from the repository root.
+- VSCode:
+  a) Press `Ctrl+Shift+B` to build.
+  b) Press `F5` to build and run with debugger attached.
+- Windows:
+  a) Double-click `BUILD.bat` in the repository root to build (will wait for a key press before it closes).
+  b) Double-click `tools/build/build.bat` to build (will exit as soon as it finishes building).
+- Linux:
+  a) Run `tools/build/build` from the repository root.
 
 The script will skip build steps whose inputs have not changed since the last run.
 
 ## Dependencies
 
-- On Windows, `Build.bat` will automatically install a private copy of Node.
-
+- On Windows, `BUILD.bat` will automatically install a private (vendored) copy of Node.
 - On Linux, install Node using your package manager or from <https://nodejs.org/en/download/>.
+- On Linux , unless using tgs4 or later you will need to compile rust-g on the server and obtain a .so file, for instructions see https://github.com/tgstation/rust-g
 
 ## Why?
 

tools/build/build.js

@@ -5,82 +5,221 @@
  * @license MIT
  */
 
-const { resolve: resolvePath } = require('path');
-const { resolveGlob } = require('./cbt/fs');
+// Change working directory to project root
+process.chdir(require('path').resolve(__dirname, '../../'));
+
+// Validate NodeJS version
+const NODE_VERSION = parseInt(process.versions.node.match(/(\d+)/)[1]);
+const NODE_VERSION_TARGET = parseInt(require('fs')
+  .readFileSync('dependencies.sh', 'utf-8')
+  .match(/NODE_VERSION=(\d+)/)[1]);
+if (NODE_VERSION < NODE_VERSION_TARGET) {
+  console.error('Your current Node.js version is out of date.');
+  console.error('You have two options:');
+  console.error('  a) Go to https://nodejs.org/ and install the latest LTS release of Node.js');
+  console.error('  b) Uninstall Node.js (our build system automatically downloads one)');
+  process.exit(1);
+}
+
+const STANDARD_BUILD = "Standard Build"
+const TGS_BUILD = "TGS Build"
+const ALL_MAPS_BUILD = "CI All Maps Build"
+const TEST_RUN_BUILD = "CI Integration Tests Build"
+const NO_DM_BUILD = "Except DM Build"
+
+let BUILD_MODE = STANDARD_BUILD;
+if (process.env.CBT_BUILD_MODE) {
+  switch (process.env.CBT_BUILD_MODE) {
+    case "ALL_MAPS":
+      BUILD_MODE = ALL_MAPS_BUILD
+      break;
+    case "TEST_RUN":
+      BUILD_MODE = TEST_RUN_BUILD
+      break;
+    case "TGS":
+      BUILD_MODE = TGS_BUILD
+      break;
+    case "NO_DM":
+      BUILD_MODE = NO_DM_BUILD
+      break;
+    default:
+      BUILD_MODE = process.env.CBT_BUILD_MODE
+      break;
+  }
+}
+console.log(`Starting CBT in ${BUILD_MODE} mode.`)
+
+const DME_NAME = 'tgstation'
+
+// Main
+// --------------------------------------------------------
+
+const { resolveGlob, stat } = require('./cbt/fs');
 const { exec } = require('./cbt/process');
 const { Task, runTasks } = require('./cbt/task');
 const { regQuery } = require('./cbt/winreg');
+const fs = require('fs');
 
-// Change working directory to project root
-process.chdir(resolvePath(__dirname, '../../'));
+const yarn = args => {
+  const yarnPath = resolveGlob('./tgui/.yarn/releases/yarn-*.cjs')[0]
+    .replace('/tgui/', '/');
+  return exec('node', [yarnPath, ...args], {
+    cwd: './tgui',
+  });
+};
 
-const taskTgui = new Task('tgui')
-  .depends('tgui/.yarn/releases/*')
+/** Installs all tgui dependencies */
+const taskYarn = new Task('yarn')
+  // The following dependencies skip what could be considered an important
+  // step in Yarn: it verifies the integrity of cache. With this setup, if
+  // cache ever becomes corrupted, your only option is to clean build.
+  .depends('tgui/.yarn/+(cache|releases|plugins|sdks)/**/*')
+  .depends('tgui/**/package.json')
   .depends('tgui/yarn.lock')
+  // Phony target (automatically created at the end of the task)
+  .provides('tgui/.yarn/install-target')
+  .build(() => yarn(['install']));
+
+/** Builds svg fonts */
+const taskTgfont = new Task('tgfont')
+  .depends('tgui/.yarn/install-target')
+  .depends('tgui/packages/tgfont/**/*.+(js|cjs|svg)')
+  .depends('tgui/packages/tgfont/package.json')
+  .provides('tgui/packages/tgfont/dist/tgfont.css')
+  .provides('tgui/packages/tgfont/dist/tgfont.eot')
+  .provides('tgui/packages/tgfont/dist/tgfont.woff2')
+  .build(() => yarn(['workspace', 'tgfont', 'build']));
+
+/** Builds tgui */
+const taskTgui = new Task('tgui')
+  .depends('tgui/.yarn/install-target')
   .depends('tgui/webpack.config.js')
   .depends('tgui/**/package.json')
-  .depends('tgui/packages/**/*.js')
-  .depends('tgui/packages/**/*.jsx')
+  .depends('tgui/packages/**/*.+(js|cjs|ts|tsx|scss)')
   .provides('tgui/public/tgui.bundle.css')
   .provides('tgui/public/tgui.bundle.js')
-  .provides('tgui/public/tgui-common.chunk.js')
+  .provides('tgui/public/tgui-common.bundle.js')
   .provides('tgui/public/tgui-panel.bundle.css')
   .provides('tgui/public/tgui-panel.bundle.js')
   .build(async () => {
-    // Instead of calling `tgui/bin/tgui`, we reproduce the whole pipeline
-    // here for maximum compilation speed.
-    const yarnRelease = resolveGlob('./tgui/.yarn/releases/yarn-*.cjs')[0]
-      .replace('/tgui/', '/');
-    const yarn = args => exec('node', [yarnRelease, ...args], {
-      cwd: './tgui',
-    });
-    await yarn(['install']);
     await yarn(['run', 'webpack-cli', '--mode=production']);
   });
 
-const taskDm = new Task('dm')
+/**
+ * Prepends the defines to the .dme.
+ * Does not clean them up, as this is intended for TGS which
+ * clones new copies anyway.
+ */
+const taskPrependDefines = (...defines) => new Task('prepend-defines')
+  .build(async () => {
+    const dmeContents = fs.readFileSync(`${DME_NAME}.dme`);
+    const textToWrite = defines.map(define => `#define ${define}\n`);
+    fs.writeFileSync(`${DME_NAME}.dme`, `${textToWrite}\n${dmeContents}`);
+  });
+
+const taskDm = (...injectedDefines) => new Task('dm')
   .depends('_maps/map_files/generic/**')
   .depends('code/**')
   .depends('goon/**')
   .depends('html/**')
   .depends('icons/**')
   .depends('interface/**')
+  .depends(process.platform === 'win32' ? 'byond-extools.*' : 'libbyond-extools.*')
   .depends('tgui/public/tgui.html')
   .depends('tgui/public/*.bundle.*')
-  .depends('tgui/public/*.chunk.*')
-  .depends(process.platform === 'win32' ? 'byond-extools.*' : 'libbyond-extools.*')
-  .depends('tgstation.dme')
-  .provides('tgstation.dmb')
-  .provides('tgstation.rsc')
+  .depends(`${DME_NAME}.dme`)
+  .provides(`${DME_NAME}.dmb`)
+  .provides(`${DME_NAME}.rsc`)
   .build(async () => {
-    let compiler = 'dm';
-    // Let's do some registry queries on Windows, because dm is not in PATH.
-    if (process.platform === 'win32') {
-      const installPath = (
-        await regQuery(
-          'HKLM\\Software\\Dantom\\BYOND',
-          'installpath')
-        || await regQuery(
-          'HKLM\\SOFTWARE\\WOW6432Node\\Dantom\\BYOND',
-          'installpath')
-      );
-      if (installPath) {
-        compiler = resolvePath(installPath, 'bin/dm.exe');
+    const dmPath = await (async () => {
+      // Search in array of paths
+      const paths = [
+        ...((process.env.DM_EXE && process.env.DM_EXE.split(',')) || []),
+        'C:\\Program Files\\BYOND\\bin\\dm.exe',
+        'C:\\Program Files (x86)\\BYOND\\bin\\dm.exe',
+        ['reg', 'HKLM\\Software\\Dantom\\BYOND', 'installpath'],
+        ['reg', 'HKLM\\SOFTWARE\\WOW6432Node\\Dantom\\BYOND', 'installpath'],
+      ];
+      const isFile = path => {
+        try {
+          const fstat = stat(path);
+          return fstat && fstat.isFile();
+        }
+        catch (err) {}
+        return false;
+      };
+      for (let path of paths) {
+        // Resolve a registry key
+        if (Array.isArray(path)) {
+          const [type, ...args] = path;
+          path = await regQuery(...args);
+        }
+        if (!path) {
+          continue;
+        }
+        // Check if path exists
+        if (isFile(path)) {
+          return path;
+        }
+        if (isFile(path + '/dm.exe')) {
+          return path + '/dm.exe';
+        }
+        if (isFile(path + '/bin/dm.exe')) {
+          return path + '/bin/dm.exe';
+        }
       }
-    } else {
-      compiler = 'DreamMaker';
+      // Default paths
+      return (
+        process.platform === 'win32' && 'dm.exe'
+        || 'DreamMaker'
+      );
+    })();
+    if (injectedDefines.length) {
+        const injectedContent = injectedDefines
+          .map(x => `#define ${x}\n`)
+          .join('')
+        // Create mdme file
+        fs.writeFileSync(`${DME_NAME}.mdme`, injectedContent)
+        // Add the actual dme content
+        const dme_content = fs.readFileSync(`${DME_NAME}.dme`)
+        fs.appendFileSync(`${DME_NAME}.mdme`, dme_content)
+        await exec(dmPath, [`${DME_NAME}.mdme`]);
+        // Rename dmb
+        fs.renameSync(`${DME_NAME}.mdme.dmb`, `${DME_NAME}.dmb`)
+        // Rename rsc
+        fs.renameSync(`${DME_NAME}.mdme.rsc`, `${DME_NAME}.rsc`)
+        // Remove mdme
+        fs.unlinkSync(`${DME_NAME}.mdme`)
+    }
+    else {
+      await exec(dmPath, [`${DME_NAME}.dme`]);
     }
-    await exec(compiler, ['tgstation.dme']);
   });
 
 // Frontend
-const tasksToRun = [
+let tasksToRun = [
+  taskYarn,
+  taskTgfont,
   taskTgui,
-  taskDm,
 ];
-
-if (process.env['TG_BUILD_TGS_MODE']) {
-  tasksToRun.pop();
+switch (BUILD_MODE) {
+  case STANDARD_BUILD:
+    tasksToRun.push(taskDm('CBT'));
+    break;
+  case TGS_BUILD:
+    tasksToRun.push(taskPrependDefines('TGS'));
+    break;
+  case ALL_MAPS_BUILD:
+    tasksToRun.push(taskDm('CBT','CIBUILDING','CITESTING','ALL_MAPS'));
+    break;
+  case TEST_RUN_BUILD:
+    tasksToRun.push(taskDm('CBT','CIBUILDING'));
+    break;
+  case NO_DM_BUILD:
+    break;
+  default:
+    console.error(`Unknown build mode : ${BUILD_MODE}`)
+    break;
 }
 
 runTasks(tasksToRun);

tools/build/cbt/fs.js

@@ -132,5 +132,4 @@ module.exports = {
   compareFiles,
   stat,
   resolveGlob,
-  compareFiles,
 };

tools/build/cbt/process.js

@@ -78,13 +78,10 @@ const exec = (executable, args, options) => {
     }
     const child = spawn(executable, args, options);
     children.add(child);
-    child.stdout.on('data', data => {
-      process.stdout.write(data);
-    });
-    child.stderr.on('data', data => {
-      process.stderr.write(data);
-    });
+    child.stdout.pipe(process.stdout, { end: false });
+    child.stderr.pipe(process.stderr, { end: false });
     child.stdin.end();
+    child.on('error', err => reject(err));
     child.on('exit', code => {
       children.delete(child);
       if (code !== 0) {
@@ -101,4 +98,5 @@ const exec = (executable, args, options) => {
 
 module.exports = {
   exec,
+  ExitError,
 };

tools/build/cbt/task.js

@@ -69,6 +69,9 @@ class Task {
     if (!this.script) {
       return;
     }
+    if (process.env.DEBUG && needsRebuild) {
+      console.debug(`    Reason: ${needsRebuild}`);
+    }
     console.warn(` => Starting '${this.name}'`);
     const startedAt = Date.now();
     // Run the script