Adds a fuckload of tool updates

also updates tgui

tools/appveyor/build.ps1

@@ -0,0 +1,10 @@
+if(!(Test-Path -Path "C:/byond")){
+    bash tools/appveyor/download_byond.sh
+    [System.IO.Compression.ZipFile]::ExtractToDirectory("C:/byond.zip", "C:/")
+    Remove-Item C:/byond.zip
+}
+
+Set-Location $env:APPVEYOR_BUILD_FOLDER
+
+&"C:/byond/bin/dm.exe" -max_errors 0 tgstation.dme
+exit $LASTEXITCODE

tools/appveyor/download_byond.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+source dependencies.sh
+echo "Downloading BYOND version $BYOND_MAJOR.$BYOND_MINOR"
+curl "http://www.byond.com/download/build/$BYOND_MAJOR/$BYOND_MAJOR.${BYOND_MINOR}_byond.zip" -o C:/byond.zip

tools/deploy.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#Run this in the repo root after compiling
+#First arg is path to where you want to deploy
+#creates a work tree free of everything except what's necessary to run the game
+
+#second arg is working directory if necessary
+if [[ $# -eq 2 ]] ; then
+  cd $2
+fi
+
+mkdir -p \
+    $1/_maps \
+    $1/icons/minimaps \
+    $1/sound/chatter \
+    $1/sound/voice/complionator \
+    $1/sound/instruments \
+    $1/strings
+
+if [ -d ".git" ]; then
+  mkdir -p $1/.git/logs
+  cp -r .git/logs/* $1/.git/logs/
+fi
+
+cp tgstation.dmb tgstation.rsc $1/
+cp -r _maps/* $1/_maps/
+cp icons/default_title.dmi $1/icons/
+cp -r icons/minimaps/* $1/icons/minimaps/
+cp -r sound/chatter/* $1/sound/chatter/
+cp -r sound/voice/complionator/* $1/sound/voice/complionator/
+cp -r sound/instruments/* $1/sound/instruments/
+cp -r strings/* $1/strings/
+
+#remove .dm files from _maps
+
+#this regrettably doesn't work with windows find
+#find $1/_maps -name "*.dm" -type f -delete
+
+#dlls on windows
+cp rust_g* $1/ || true
+cp *BSQL.* $1/ || true

tools/mapmerge2/dmm.py

@@ -58,6 +58,23 @@ class DMM:
 
         raise RuntimeError("ran out of keys, this shouldn't happen")
 
+    def overwrite_key(self, key, fixed, bad_keys):
+        try:
+            self.dictionary[key] = fixed
+            return None
+        except bidict.DuplicationError:
+            old_key = self.dictionary.inv[fixed]
+            bad_keys[key] = old_key
+            print(f"Merging '{num_to_key(key, self.key_length)}' into '{num_to_key(old_key, self.key_length)}'")
+            return old_key
+
+    def reassign_bad_keys(self, bad_keys):
+        if not bad_keys:
+            return
+        for k, v in self.grid.items():
+            # reassign the grid entries which used the old key
+            self.grid[k] = bad_keys.get(v, v)
+
     def _presave_checks(self):
         # last-second handling of bogus keys to help prevent and fix broken maps
         self._ensure_free_keys(0)
@@ -70,9 +87,16 @@ class DMM:
                 new_key = bad_keys[k] = self.generate_new_key()
                 self.dictionary.forceput(new_key, self.dictionary[k])
                 print(f"    {num_to_key(k, self.key_length, True)} -> {num_to_key(new_key, self.key_length)}")
-            for k, v in self.grid.items():
-                # reassign the grid entries which used the old key
-                self.grid[k] = bad_keys.get(v, v)
+
+        # handle entries in the dictionary which have atoms in the wrong order
+        keys = list(self.dictionary.keys())
+        for key in keys:
+            value = self.dictionary[key]
+            if is_bad_atom_ordering(num_to_key(key, self.key_length, True), value):
+                fixed = tuple(fix_atom_ordering(value))
+                self.overwrite_key(key, fixed, bad_keys)
+
+        self.reassign_bad_keys(bad_keys)
 
     def _ensure_free_keys(self, desired):
         # ensure that free keys exist by increasing the key length if necessary
@@ -179,6 +203,45 @@ def parse_map_atom(atom):
 
     return path, vars
 
+def is_bad_atom_ordering(key, atoms):
+    seen_turfs = 0
+    seen_areas = 0
+    can_fix = False
+    for each in atoms:
+        if each.startswith('/turf'):
+            if seen_turfs == 1:
+                print(f"Warning: key '{key}' has multiple turfs!")
+            if seen_areas:
+                print(f"Warning: key '{key}' has area before turf (autofixing...)")
+                can_fix = True
+            seen_turfs += 1
+        elif each.startswith('/area'):
+            if seen_areas == 1:
+                print(f"Warning: key '{key}' has multiple areas!!!")
+            seen_areas += 1
+        else:
+            if (seen_turfs or seen_areas) and not can_fix:
+                print(f"Warning: key '{key}' has movable after turf or area (autofixing...)")
+                can_fix = True
+    if not seen_areas or not seen_turfs:
+        print(f"Warning: key '{key}' is missing either a turf or area")
+    return can_fix
+
+def fix_atom_ordering(atoms):
+    movables = []
+    turfs = []
+    areas = []
+    for each in atoms:
+        if each.startswith('/turf'):
+            turfs.append(each)
+        elif each.startswith('/area'):
+            areas.append(each)
+        else:
+            movables.append(each)
+    movables.extend(turfs)
+    movables.extend(areas)
+    return movables
+
 # ----------
 # TGM writer
 

tools/mapmerge2/precommit.py

@@ -9,6 +9,13 @@ def main(repo):
         print("You need to resolve merge conflicts first.")
         return 1
 
+    try:
+        repo.lookup_reference('MERGE_HEAD')
+        print("Not running mapmerge for merge commit.")
+        return 0
+    except KeyError:
+        pass
+
     changed = 0
     for path, status in repo.status().items():
         if path.endswith(".dmm") and (status & (pygit2.GIT_STATUS_INDEX_MODIFIED | pygit2.GIT_STATUS_INDEX_NEW)):
@@ -20,12 +27,12 @@ def main(repo):
                 head_blob = repo[repo[repo.head.target].tree[path].id]
             except KeyError:
                 # New map, no entry in HEAD
-                print(f"Converting new map: {path}")
+                print(f"Converting new map: {path}", flush=True)
                 assert (status & pygit2.GIT_STATUS_INDEX_NEW)
                 merged_map = index_map
             else:
                 # Entry in HEAD, merge the index over it
-                print(f"Merging map: {path}")
+                print(f"Merging map: {path}", flush=True)
                 assert not (status & pygit2.GIT_STATUS_INDEX_NEW)
                 head_map = dmm.DMM.from_bytes(head_blob.read_raw())
                 merged_map = merge_map(index_map, head_map)

tools/mapmerge2/update_paths.py

@@ -0,0 +1,163 @@
+# A script and syntax for applying path updates to maps.
+import re
+import os
+import argparse
+import frontend
+from dmm import *
+
+desc = """
+Update dmm files given update file/string.
+Replacement syntax example:
+    /turf/open/floor/plasteel/warningline : /obj/effect/turf_decal {dir = @OLD ;tag = @SKIP;icon_state = @SKIP}
+    /turf/open/floor/plasteel/warningline : /obj/effect/turf_decal {@OLD} , /obj/thing {icon_state = @OLD:name; name = "meme"}
+    /turf/open/floor/plasteel/warningline{dir=2} : /obj/thing
+New paths properties:
+    @OLD - if used as property name copies all modified properties from original path to this one
+    property = @SKIP - will not copy this property through when global @OLD is used.
+    property = @OLD - will copy this modified property from original object even if global @OLD is not used
+    property = @OLD:name - will copy [name] property from original object even if global @OLD is not used
+    Anything else is copied as written.
+Old paths properties:
+    Will be used as a filter.
+    property = @UNSET - will apply the rule only if the property is not mapedited
+"""
+
+default_map_directory = "../../_maps"
+replacement_re = re.compile('\s*([^{]*)\s*(\{(.*)\})?')
+
+#urgent todo: replace with actual parser, this is slow as janitor in crit
+split_re = re.compile('((?:[A-Za-z0-9_\-$]+)\s*=\s*(?:"(?:.+?)"|[^";]*)|@OLD)')
+
+
+def props_to_string(props):
+    return "{{{0}}}".format(";".join([k+" = "+props[k] for k in props]))
+
+
+def string_to_props(propstring, verbose = False):
+    props = dict()
+    for raw_prop in re.split(split_re, propstring):
+        if not raw_prop or raw_prop.strip() == ';':
+            continue
+        prop = raw_prop.split('=', maxsplit=1)
+        props[prop[0].strip()] = prop[1].strip() if len(prop) > 1 else None
+    if verbose:
+        print("{0} to {1}".format(propstring, props))
+    return props
+
+
+def parse_rep_string(replacement_string, verbose = False):
+    # translates /blah/blah {meme = "test",} into path,prop dictionary tuple
+    match = re.match(replacement_re, replacement_string)
+    path = match.group(1)
+    props = match.group(3)
+    if props:
+        prop_dict = string_to_props(props, verbose)
+    else:
+        prop_dict = dict()
+    return path.strip(), prop_dict
+
+
+def update_path(dmm_data, replacement_string, verbose=False):
+    old_path_part, new_path_part = replacement_string.split(':', maxsplit=1)
+    old_path, old_path_props = parse_rep_string(old_path_part, verbose)
+    new_paths = list()
+    for replacement_def in new_path_part.split(','):
+        new_path, new_path_props = parse_rep_string(replacement_def, verbose)
+        new_paths.append((new_path, new_path_props))
+
+    def replace_def(match):
+        if match.group(2):
+            old_props = string_to_props(match.group(2), verbose)
+        else:
+            old_props = dict()
+        for filter_prop in old_path_props:
+            if filter_prop not in old_props:
+                if old_path_props[filter_prop] == "@UNSET":
+                    continue
+                else:
+                    return [match.group(0)]
+            else:
+                if old_props[filter_prop] != old_path_props[filter_prop] or old_path_props[filter_prop] == "@UNSET":
+                    return [match.group(0)] #does not match current filter, skip the change.
+        if verbose:
+            print("Found match : {0}".format(match.group(0)))
+        out_paths = []
+        for new_path, new_props in new_paths:
+            out = new_path
+            out_props = dict()
+            for prop_name, prop_value in new_props.items():
+                if prop_name == "@OLD":
+                    out_props = dict(old_props)
+                    continue
+                if prop_value == "@SKIP":
+                    out_props.pop(prop_name, None)
+                    continue
+                if prop_value.startswith("@OLD"):
+                    params = prop_value.split(":")
+                    if prop_name in old_props:
+                        out_props[prop_name] = old_props[params[1]] if len(params) > 1 else old_props[prop_name]
+                    continue
+                out_props[prop_name] = prop_value
+            if out_props:
+                out += props_to_string(out_props)
+            out_paths.append(out)
+        if verbose:
+            print("Replacing with: {0}".format(out_paths))
+        return out_paths
+
+    def get_result(element):
+        p = re.compile("{0}\s*({{(.*)}})?$".format(re.escape(old_path)))
+        match = p.match(element)
+        if match:
+            return replace_def(match) # = re.sub(p,replace_def,element)
+        else:
+            return [element]
+
+    bad_keys = {}
+    keys = list(dmm_data.dictionary.keys())
+    for definition_key in keys:
+        def_value = dmm_data.dictionary[definition_key]
+        new_value = tuple(y for x in def_value for y in get_result(x))
+        if new_value != def_value:
+            dmm_data.overwrite_key(definition_key, new_value, bad_keys)
+    dmm_data.reassign_bad_keys(bad_keys)
+
+
+def update_map(map_filepath, updates, verbose=False):
+    print("Updating: {0}".format(map_filepath))
+    dmm_data = DMM.from_file(map_filepath)
+    for update_string in updates:
+        update_path(dmm_data, update_string, verbose)
+    dmm_data.to_file(map_filepath, True)
+
+
+def update_all_maps(map_directory, updates, verbose=False):
+    for root, _, files in os.walk(map_directory):
+        for filepath in files:
+            if filepath.endswith(".dmm"):
+                path = os.path.join(root, filepath)
+                update_map(path, updates, verbose)
+
+
+def main(args):
+    if args.inline:
+        updates = [args.update_source]
+    else:
+        with open(args.update_source) as f:
+            updates = [line for line in f if line and not line.startswith("#") and not line.isspace()]
+
+    if args.map:
+        update_map(args.map, updates, verbose=args.verbose)
+    else:
+        map_directory = args.directory or frontend.read_settings().map_folder
+        update_all_maps(map_directory, updates, verbose=args.verbose)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("update_source", help="update file path / line of update notation")
+    parser.add_argument("--map", "-m", help="path to update, defaults to all maps in maps directory")
+    parser.add_argument("--directory", "-d", help="path to maps directory, defaults to _maps/")
+    parser.add_argument("--inline", "-i", help="treat update source as update string instead of path", action="store_true")
+    parser.add_argument("--verbose", "-v", help="toggle detailed update information", action="store_true")
+    main(parser.parse_args())

tools/tgs4_scripts/PostCompile.bat

@@ -0,0 +1,3 @@
+@echo off
+
+powershell -NoProfile -ExecutionPolicy Bypass -File PostCompile.ps1 -game_path %1

tools/tgs4_scripts/PostCompile.ps1

@@ -0,0 +1,18 @@
+param(
+    $game_path
+)
+
+Write-Host "Deploying tgstation compilation..."
+
+cd $game_path
+
+mkdir build
+
+#.github is a little special cause of the prefix
+mv .github build/.github
+
+mv * build   #thank god it's that easy 
+
+&"build/tools/deploy.sh" $game_path $game_path/build
+
+Remove-Item build -Recurse

tools/tgs4_scripts/PostCompile.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+#Basically run deploy.sh, but first
+
+echo 'Deploying tgstation compilation...'
+
+cd $1
+
+mkdir build
+
+shopt -s extglob dotglob
+mv !(build) build
+shopt -u dotglob
+
+build/tools/deploy.sh $1 $1/build
+
+rm -rf build

tools/travis/before_build_byond.sh

@@ -4,27 +4,28 @@ set -e
 #If this is the build tools step, we do not bother to install/build byond
 if [ "$BUILD_TOOLS" = true ]; then
   exit 0
-fi;
+fi
+
 echo "Combining maps for building"
 if [ $BUILD_TESTING = true ]; then
     python tools/travis/template_dm_generator.py
-fi;
-
-if [ -d "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}/byond/bin" ];
-then
-  echo "Using cached directory."
-  exit 0
-else
-  echo "Setting up BYOND."
-  mkdir -p "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}"
-  cd "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}"
-  curl "http://www.byond.com/download/build/${BYOND_MAJOR}/${BYOND_MAJOR}.${BYOND_MINOR}_byond_linux.zip" -o byond.zip
-  unzip byond.zip
-  cd byond
-  make here
-  cd ~/
-  exit 0
 fi
 
-#some variable not set correctly, panic
-exit 1
+source dependencies.sh
+
+if [ -d "$HOME/BYOND/byond/bin" ] && grep -Fxq "${BYOND_MAJOR}.${BYOND_MINOR}" $HOME/BYOND/version.txt;
+then
+  echo "Using cached directory."
+else
+  echo "Setting up BYOND."
+  rm -rf "$HOME/BYOND"
+  mkdir -p "$HOME/BYOND"
+  cd "$HOME/BYOND"
+  curl "http://www.byond.com/download/build/${BYOND_MAJOR}/${BYOND_MAJOR}.${BYOND_MINOR}_byond_linux.zip" -o byond.zip
+  unzip byond.zip
+  rm byond.zip
+  cd byond
+  make here
+  echo "$BYOND_MAJOR.$BYOND_MINOR" > "$HOME/BYOND/version.txt"
+  cd ~/
+fi

tools/travis/build_byond.sh

@@ -41,33 +41,36 @@ if [ "$BUILD_TOOLS" = false ]; then
 		exit 1
 	fi;
 
-    source $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}/byond/bin/byondsetup
+    source $HOME/BYOND/byond/bin/byondsetup
 	if [ "$BUILD_TESTING" = true ]; then
 		tools/travis/dm.sh -DTRAVISBUILDING -DTRAVISTESTING -DALL_MAPS tgstation.dme
 	else
 		tools/travis/dm.sh -DTRAVISBUILDING tgstation.dme
 		
-		#config folder should not be mandatory
-		rm -rf config/*
+		tools/deploy.sh travis_test
+		mkdir travis_test/config
 
 		#test config
-		cp tools/travis/travis_config.txt config/config.txt
+		cp tools/travis/travis_config.txt travis_test/config/config.txt
 
 		# get libmariadb, cache it so limmex doesn't get angery
 		if [ -f $HOME/libmariadb ]; then
 			#travis likes to interpret the cache command as it being a file for some reason
 			rm $HOME/libmariadb
-			mkdir $HOME/libmariadb
 		fi
+		mkdir -p $HOME/libmariadb
 		if [ ! -f $HOME/libmariadb/libmariadb.so ]; then
 			wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz
 			tar -xvf mariadb_client-2.0.0-linux.tgz
 			mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so
 			rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux
 		fi
-    	ln -s $HOME/libmariadb/libmariadb.so libmariadb.so
 	
+		cd travis_test
+    	ln -s $HOME/libmariadb/libmariadb.so libmariadb.so
 		DreamDaemon tgstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis"
-		cat data/logs/travis/clean_run.lk
+		cd ..
+		cat travis_test/data/logs/travis/clean_run.lk
+
 	fi;
 fi;

tools/travis/build_dependencies.sh

@@ -2,6 +2,15 @@
 
 set -e
 
+source dependencies.sh
+
+#ensure the Dockerfile version matches the dependencies.sh version
+line=$(head -n 1 Dockerfile)
+if [[ $line != *"$BYOND_MAJOR.$BYOND_MINOR"* ]]; then
+  echo "Dockerfile BYOND version in FROM command does not match dependencies.sh (Or it's not on line 1)!"
+  exit 1
+fi
+
 if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then
     curl https://sh.rustup.rs -sSf | sh -s -- -y --default-host i686-unknown-linux-gnu
     source ~/.profile
@@ -13,7 +22,43 @@ if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then
     git fetch --depth 1 origin $RUST_G_VERSION
     git checkout FETCH_HEAD
     cargo build --release
+    cmp target/rust_g.dm ../code/__DEFINES/rust_g.dm
 
     mkdir -p ~/.byond/bin
     ln -s $PWD/target/release/librust_g.so ~/.byond/bin/rust_g
+
+    mkdir -p ../BSQL/artifacts
+    cd ../BSQL
+    git init
+    git remote add origin https://github.com/tgstation/BSQL
+    git fetch --depth 1 origin $BSQL_VERSION
+    git checkout FETCH_HEAD
+
+    if [ -f "$HOME/MariaDB/libmariadb.so.2" ] && [ -f "$HOME/MariaDB/libmariadb.so" ] && [ -d "$HOME/MariaDB/include" ];
+    then
+        echo "Using cached MariaDB library."
+    else
+        echo "Setting up MariaDB."
+        rm -rf "$HOME/MariaDB"
+        mkdir -p "$HOME/MariaDB"
+        wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-client-lgpl/libmariadb2_2.0.0-1_i386.deb
+        dpkg -x libmariadb2_2.0.0-1_i386.deb /tmp/extract
+        rm libmariadb2_2.0.0-1_i386.deb
+        mv /tmp/extract/usr/lib/i386-linux-gnu/libmariadb.so.2 $HOME/MariaDB/
+        ln -s $HOME/MariaDB/libmariadb.so.2 $HOME/MariaDB/libmariadb.so
+        rm -rf /tmp/extract
+
+        wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-connector-c/libmariadb-dev_2.3.3-1_i386.deb
+        dpkg -x libmariadb-dev_2.3.3-1_i386.deb /tmp/extract
+        rm libmariadb-dev_2.3.3-1_i386.deb
+        mv /tmp/extract/usr/include $HOME/MariaDB/
+        #fuck what is this even?
+        mv $HOME/MariaDB/include/mariadb $HOME/MariaDB/include/mysql
+    fi
+
+    cd artifacts
+    export CXX=g++-7
+    cmake .. -DMARIA_INCLUDE_DIR=$HOME/MariaDB/include
+    make
+    mv src/BSQL/libBSQL.so ../../
 fi

tools/travis/install_build_tools.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+source dependencies.sh
+
 if [ "$BUILD_TOOLS" = true ]; then
       rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install $NODE_VERSION
       npm install -g gulp-cli