Linter Improvements (#37149)

* linter improvements

* update rights

* requirements

* perhaps

* better

* test

* oops

* un-TG it

* Update od_lints.dm

.github/workflows/ci.yml

@@ -38,15 +38,19 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
+      # - uses: actions/setup-python@v4
+      #   with:
+      #     python-version: '3.x'
+
+      - name: Setup Python
+        run: |
+          tools/bootstrap/python -c ''
 
         #TODO: Cache Dreamchecker install
       - name: Run dreamchecker
         run: |
           SPACEMAN_DMM_GIT_TAG="suite-1.8" tools/travis/install_spaceman_dmm.sh dreamchecker
-          ~/dreamchecker
+          ~/dreamchecker 2>&1 | bash tools/ci/annotate_dm.sh
 
       - name: Run map checker
         run: python tools/travis/check_map_files.py maps/
@@ -59,6 +63,17 @@ jobs:
       - name: Make sure test maps or vaults aren't ticked
         run: find -name '*.dme' -exec cat {} \; | awk '/maps\\test.*|\.dmm/ { exit 1 }'
 
+      # OpenDream linting
+      - name: Install OpenDream
+        uses: robinraju/release-downloader@v1.9
+        with:
+          repository: "OpenDreamProject/OpenDream"
+          tag: "latest"
+          fileName: "DMCompiler_linux-x64.tar.gz"
+          extract: true
+      - name: Run OpenDream
+        run: ./DMCompiler_linux-x64/DMCompiler vgstation13.dme --suppress-unimplemented --define=CIBUILDING | bash tools/ci/annotate_od.sh
+
   build:
     name: ${{matrix.job-name}}
     runs-on: ubuntu-latest

.gitignore

@@ -77,3 +77,6 @@ Thumbs.db
 
 # Bootstrapped node installation for the tgui build system
 .cache
+
+# Running OpenDream locally
+vgstation13.json

___odlint.dm

@@ -0,0 +1,10 @@
+// This file is included right at the start of the DME.
+// Its purpose is to enable multiple lints (pragmas) that are supported by OpenDream to better validate the codebase
+// These are essentially nitpicks the DM compiler should pick up on but doesnt
+
+#if !defined(SPACEMAN_DMM) && defined(OPENDREAM)
+// This is in a separate file as a hack to avoid SpacemanDMM
+// evaluating the #pragma lines, even if its outside a block it cares about
+// (Also so people can code-own it. Shoutout to AA)
+#include "tools/ci/od_lints.dm"
+#endif

dependencies.sh

@@ -17,7 +17,7 @@ export NODE_VERSION_PRECISE=16.13.1
 #export SPACEMAN_DMM_VERSION=suite-1.7.1
 
 # Python version for mapmerge and other tools
-#export PYTHON_VERSION=3.6.8
+export PYTHON_VERSION=3.7.9
 
 #rust_g git tag
 export RUST_G_VERSION=3.0.0

tools/bootstrap/python36._pth

@@ -1,6 +0,0 @@
-python36.zip
-.
-..\..\..
-
-# Uncomment to run site.main() automatically
-import site

tools/bootstrap/python_.ps1

@@ -49,8 +49,13 @@ if (!(Test-Path $PythonExe -PathType Leaf)) {
 
 	[System.IO.Compression.ZipFile]::ExtractToDirectory($Archive, $PythonDir)
 
+	$PythonVersionArray = $PythonVersion.Split(".")
+	$PythonVersionString = "python$($PythonVersionArray[0])$($PythonVersionArray[1])"
+	Write-Output "Generating PATH descriptor."
+	New-Item "$Cache/$PythonVersionString._pth" | Out-Null
+	Set-Content "$Cache/$PythonVersionString._pth" "$PythonVersionString.zip`n.`n..\..\..`nimport site`n"
 	# Copy a ._pth file without "import site" commented, so pip will work
-	Copy-Item "$Bootstrap/python36._pth" $PythonDir `
+	Copy-Item "$Cache/$PythonVersionString" $PythonDir `
 		-ErrorAction Stop
 
 	Remove-Item $Archive

tools/ci/annotate_dm.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -euo pipefail
+tools/bootstrap/python -m dm_annotator "$@"

tools/ci/annotate_od.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -euo pipefail
+tools/bootstrap/python -m od_annotator "$@"

tools/ci/od_lints.dm

@@ -0,0 +1,33 @@
+//1000-1999
+#pragma FileAlreadyIncluded error
+#pragma MissingIncludedFile error
+#pragma MisplacedDirective error
+#pragma UndefineMissingDirective error
+#pragma DefinedMissingParen error
+#pragma ErrorDirective error
+#pragma WarningDirective warning
+#pragma MiscapitalizedDirective error
+
+//2000-2999
+#pragma SoftReservedKeyword error
+#pragma DuplicateVariable error
+#pragma DuplicateProcDefinition error
+#pragma PointlessParentCall error
+#pragma PointlessBuiltinCall error
+#pragma SuspiciousMatrixCall error
+#pragma FallbackBuiltinArgument error
+#pragma MalformedRange error
+#pragma InvalidRange error
+#pragma InvalidSetStatement error
+#pragma InvalidOverride error
+#pragma DanglingVarType error
+#pragma MissingInterpolatedExpression error
+#pragma AmbiguousResourcePath error
+
+//3000-3999
+#pragma EmptyBlock disabled
+#pragma EmptyProc disabled
+#pragma UnsafeClientAccess disabled
+#pragma SuspiciousSwitchCase error
+#pragma AssignmentInConditional error
+#pragma AmbiguousInOrder error

tools/dm_annotator/__main__.py

@@ -0,0 +1,51 @@
+import sys
+import re
+import os.path as path
+
+# Usage: python3 annotate_dm.py [filename]
+# If filename is not provided, stdin is checked instead
+
+def red(text):
+    return "\033[31m" + str(text) + "\033[0m"
+
+def green(text):
+    return "\033[32m" + str(text) + "\033[0m"
+
+def yellow(text):
+    return "\033[33m" + str(text) + "\033[0m"
+
+def annotate(raw_output):
+    # Remove ANSI escape codes
+    raw_output = re.sub(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]', '', raw_output)
+
+    print("::group::DreamChecker Output")
+    print(raw_output)
+    print("::endgroup::")
+
+    annotation_regex = r'(?P<filename>.*?), line (?P<line>\d+), column (?P<column>\d+):\s{1,2}(?P<type>error|warning): (?P<message>.*)'
+    has_issues = False
+
+    print("DM Code Annotations:")
+    for annotation in re.finditer(annotation_regex, raw_output):
+        print(f"::{annotation['type']} file={annotation['filename']},line={annotation['line']},col={annotation['column']}::{annotation['message']}")
+        has_issues = True
+
+    if not has_issues:
+        print(green("No DM issues found"))
+
+def main():
+    if len(sys.argv) > 1:
+        if not path.exists(sys.argv[1]):
+            print(red(f"Error: Annotations file '{sys.argv[1]}' does not exist"))
+            sys.exit(1)
+        with open(sys.argv[1], 'r') as f:
+            annotate(f.read())
+    elif not sys.stdin.isatty():
+        annotate(sys.stdin.read())
+    else:
+        print(red("Error: No input provided"))
+        print("Usage: tools/ci/annotate_dm.sh [filename]")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

tools/od_annotator/__main__.py

@@ -0,0 +1,50 @@
+import sys
+import re
+
+def green(text):
+    return "\033[32m" + str(text) + "\033[0m"
+
+def red(text):
+    return "\033[31m" + str(text) + "\033[0m"
+
+def annotate(raw_output):
+    # Remove ANSI escape codes
+    raw_output = re.sub(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]', '', raw_output)
+
+    print("::group::OpenDream Output")
+    print(raw_output)
+    print("::endgroup::")
+
+    annotation_regex = r'((?P<type>Error|Warning) (?P<errorcode>OD(?P<errornumber>\d{4})) at (?P<location>(?P<filename>.+):(?P<line>\d+):(?P<column>\d+)|<internal>): (?P<message>.+))'
+    failures_detected = False
+    expected_failure_case_detected = False # this is just here so this script breaks if we forget to set it to True when we expect a failure. remove this when we have handled the expected failure
+
+    print("OpenDream Code Annotations:")
+    for annotation in re.finditer(annotation_regex, raw_output):
+        message = annotation['message']
+        if message == "Unimplemented proc & var warnings are currently suppressed": # this happens every single run, it's important to know about it but we don't need to throw an error
+            message += " (This is expected and can be ignored)" # also there's no location for it to annotate to since it's an <internal> failure.
+            expected_failure_case_detected = True
+
+        if annotation['type'] == "Error":
+            failures_detected = True
+
+        error_string = f"{annotation['errorcode']}: {message}"
+
+        if annotation['location'] == "<internal>":
+            print(f"::{annotation['type']} file=,line=,col=::{error_string}")
+        else:
+            print(f"::{annotation['type']} file={annotation['filename']},line={annotation['line']},col={annotation['column']}::{error_string}")
+
+    if failures_detected:
+        sys.exit(1)
+        return
+
+    if not expected_failure_case_detected:
+        print(red("Failed to detect the expected failure case! If you have recently changed how we work with OpenDream Pragmas, please fix the od_annotator script!"))
+        sys.exit(1)
+        return
+
+    print(green("No OpenDream issues found!"))
+
+annotate(sys.stdin.read())

tools/requirements.txt

@@ -0,0 +1,13 @@
+pygit2==1.7.2
+bidict==0.22.0
+Pillow==9.3.0
+
+# changelogs
+PyYaml==6.0.1
+beautifulsoup4==4.9.3
+
+# ezdb
+mysql-connector-python==8.0.33
+
+# icon cutter
+numpy==1.26.0

vgstation13.dme

@@ -10,6 +10,7 @@
 #define DEBUG
 // END_PREFERENCES
 // BEGIN_INCLUDE
+#include "___odlint.dm"
 #include "__DEFINES\__compile_options.dm"
 #include "__DEFINES\__opendream.dm"
 #include "__DEFINES\__spaceman_dmm.dm"