cyberpresidentvanellope/vgstation13
1
0
Fork 0
mirror of https://github.com/vgstation-coders/vgstation13.git synced 2025-12-10 02:16:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrote runtime condenser (#16495)

Browse Source 
* Rewrote runtime condenser

* Addressed 2/3 reviews

* fuck

* REMOVE .clone()

* Reduce unecessary allocations and hash map lookups.

Previously, when parsing the details of a runtime, every single details line needed the runtime to be looked up again in the hash map, and every runtime parsed needed at least one allocation to allow for the former. Both of this is has been optimized out by re-ordering the code completely.

* All the features

* Some untested work

* More work on supporting crap stack traces, still WiP

* Infinite loop and stack overflow traces work mostly.

* Add to travis

* Fix compiler warning

* Run cargo fmt

* Removed silly

* Removed more silly

* Verbose option

* rustfmt

* Sorted verbose output

* Stdout by default, stderr for errors

* rustfmt

* Don't panic on invalid UTF-8, static dispatch

* PARALLELISM

* Removed condenser from travis
This commit is contained in:
DamianX
2017-11-26 19:38:43 +01:00
committed by Pieter-Jan Briers
parent 9880b09f03
commit ada12cfc5c
10 changed files with 888 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
code/modules/error_handler/error_handler.dm
View File

@@ -16,7 +16,7 @@
global.total_runtimes++ global.total_runtimes++
var/erroruid = "[e.file][e.line]" var/erroruid = "[e.file][e.line]:[e]"
var/last_seen = global.error_last_seen[erroruid] var/last_seen = global.error_last_seen[erroruid]
var/cooldown = global.error_cooldown[erroruid] || 0 var/cooldown = global.error_cooldown[erroruid] || 0
if (last_seen == null) // A new error! if (last_seen == null) // A new error!
@@ -59,7 +59,7 @@
var/skipcount = abs(global.error_cooldown[erroruid]) - 1 var/skipcount = abs(global.error_cooldown[erroruid]) - 1
global.error_cooldown[erroruid] = 0 global.error_cooldown[erroruid] = 0
if (skipcount > 0) if (skipcount > 0)
world.log << "\[[time_stamp()]] Skipped [skipcount] runtimes in [e.file],[e.line]." world.log << "\[[time_stamp()]] Skipped [skipcount] runtimes in [e.file],[e.line]: [e]"
error_cache.log_error(e, skip_count = skipcount) error_cache.log_error(e, skip_count = skipcount)
global.error_last_seen[erroruid] = world.time global.error_last_seen[erroruid] = world.time

322
tools/Runtime Condenser/Main.cpp
View File

@@ -1,322 +0,0 @@
/* Runtime Condenser by Nodrak
* This will sum up identical runtimes into one, giving a total of how many times it occured. The first occurance
* of the runtime will log the proc, source, usr and src, the rest will just add to the total. Infinite loops will
* also be caught and displayed (if any) above the list of runtimes.
*
* How to use:
* 1) Copy and paste your list of runtimes from Dream Daemon into input.exe
* 2) Run RuntimeCondenser.exe
* 3) Open output.txt for a condensed report of the runtimes
*/
#include <iostream>
#include <fstream>
#include <string>
using namespace std;
//Make all of these global. It's bad yes, but it's a small program so it really doesn't affect anything.
//Because hardcoded numbers are bad :(
const unsigned short maxStorage = 99; //100 - 1
//What we use to read input
string currentLine = "Blank";
string nextLine = "Blank";
string tempLine = "Blank";
//Stores lines we want to keep to print out
string storedRuntime[maxStorage+1];
string storedProc[maxStorage+1];
string storedSource[maxStorage+1];
string storedUsr[maxStorage+1];
string storedSrc[maxStorage+1];
//Stat tracking stuff for output
unsigned int totalRuntimes = 0;
unsigned int totalUniqueRuntimes = 0;
unsigned int totalInfiniteLoops = 0;
unsigned int totalUniqueInfiniteLoops = 0;
//Misc
unsigned int numRuntime[maxStorage+1]; //Number of times a specific runtime has occured
bool checkNextLines = false; //Used in case byond has condensed a large number of similar runtimes
int storedIterator = 0; //Used to remember where we stored the runtime
bool readFromFile()
{
//Open file to read
ifstream inputFile("input.txt");
if(inputFile.is_open())
{
while(!inputFile.eof()) //Until end of file
{
//If we've run out of storage
if(storedRuntime[maxStorage] != "Blank") break;
//Update our lines
currentLine = nextLine;
getline(inputFile, nextLine);
//After finding a new runtime, check to see if there are extra values to store
if(checkNextLines)
{
//Skip ahead
currentLine = nextLine;
getline(inputFile, nextLine);
//If we find this, we have new stuff to store
if(nextLine.find("proc name:") != std::string::npos)
{
//Store more info
storedSource[storedIterator] = currentLine;
storedUsr[storedIterator] = nextLine;
//Skip ahead again
currentLine = nextLine;
getline(inputFile, nextLine);
//Store the last of the info
storedSrc[storedIterator] = nextLine;
}
checkNextLines = false;
}
//Found an infinite loop!
if(currentLine.find("Infinite loop suspected") != std::string::npos || currentLine.find("Maximum recursion level reached") != std::string::npos)
{
totalInfiniteLoops++;
for(int i=0; i <= maxStorage; i++)
{
//We've already encountered this
if(currentLine == storedRuntime[i])
{
numRuntime[i]++;
break;
}
//We've never encoutnered this
if(storedRuntime[i] == "Blank")
{
storedRuntime[i] = currentLine;
currentLine = nextLine;
getline(inputFile, nextLine); //Skip the "if this is not an infinite loop" line
storedProc[i] = nextLine;
numRuntime[i] = 1;
checkNextLines = true;
storedIterator = i;
totalUniqueInfiniteLoops++;
break;
}
}
}
//Found a runtime!
else if(currentLine.find("Runtime in") != std::string::npos)
{
totalRuntimes++;
for(int i=0; i <= maxStorage; i++)
{
// example runtime string:
// [02:00:37] Runtime in global cache.dm,33: Cannot read null.cache
// We only care about the characters after the timestamp.
tempLine = currentLine;
tempLine.erase(0, 11);
//We've already encountered this
if(tempLine == storedRuntime[i])
{
numRuntime[i]++;
break;
}
//We've never encoutnered this
if(storedRuntime[i] == "Blank")
{
storedRuntime[i] = tempLine;
cout << tempLine << '\n';
storedProc[i] = nextLine;
numRuntime[i] = 1;
checkNextLines = true;
storedIterator = i;
totalUniqueRuntimes++;
break;
}
}
}
}
}
else
{
return false;
}
return true;
}
bool writeToFile()
{
//Open and clear the file
ofstream outputFile("Output.txt", ios::trunc);
if(outputFile.is_open())
{
outputFile << "Note: The proc name, source file, src and usr are all from the FIRST of the identical runtimes. Everything else is cropped.\n\n";
if(totalUniqueInfiniteLoops > 0)
{
outputFile << "Total unique infinite loops: " << totalUniqueInfiniteLoops << endl;
}
if(totalInfiniteLoops > 0)
{
outputFile << "Total infinite loops: " << totalInfiniteLoops << endl;
}
outputFile << "Total unique runtimes: " << totalUniqueRuntimes << endl;
outputFile << "Total runtimes: " << totalRuntimes << endl << endl;
//Display a warning if we've hit the maximum space we've allocated for storage
if(totalUniqueRuntimes + totalUniqueInfiniteLoops >= maxStorage)
{
outputFile << "Warning: The maximum number of unique runtimes has been hit. If there were more, they have been cropped out.\n\n";
}
//If we have infinite loops, display them first.
if(totalInfiniteLoops > 0)
{
outputFile << "** Infinite loops **";
for(int i=0; i <= maxStorage; i++)
{
if(storedRuntime[i].find("Infinite loop suspected") != std::string::npos || storedRuntime[i].find("Maximum recursion level reached") != std::string::npos)
{
if(numRuntime[i] != 0) outputFile << endl << endl << "The following infinite loop has occured " << numRuntime[i] << " time(s).\n";
if(storedRuntime[i] != "Blank") outputFile << storedRuntime[i] << endl;
if(storedProc[i] != "Blank") outputFile << storedProc[i] << endl;
if(storedSource[i] != "Blank") outputFile << storedSource[i] << endl;
if(storedUsr[i] != "Blank") outputFile << storedUsr[i] << endl;
if(storedSrc[i] != "Blank") outputFile << storedSrc[i] << endl;
}
}
outputFile << endl << endl; //For spacing
}
//Do runtimes next
outputFile << "** Runtimes **";
for(int i=0; i <= maxStorage; i++)
{
if(storedRuntime[i].find("Infinite loop suspected") != std::string::npos || storedRuntime[i].find("Maximum recursion level reached") != std::string::npos) continue;
if(numRuntime[i] != 0) outputFile << endl << endl << "The following runtime has occured " << numRuntime[i] << " time(s).\n";
if(storedRuntime[i] != "Blank") outputFile << storedRuntime[i] << endl;
if(storedProc[i] != "Blank") outputFile << storedProc[i] << endl;
if(storedSource[i] != "Blank") outputFile << storedSource[i] << endl;
if(storedUsr[i] != "Blank") outputFile << storedUsr[i] << endl;
if(storedSrc[i] != "Blank") outputFile << storedSrc[i] << endl;
}
outputFile.close();
}
else
{
return false;
}
return true;
}
void sortRuntimes()
{
string tempRuntime[maxStorage+1];
string tempProc[maxStorage+1];
string tempSource[maxStorage+1];
string tempUsr[maxStorage+1];
string tempSrc[maxStorage+1];
unsigned int tempNumRuntime[maxStorage+1];
unsigned int highestCount = 0; //Used for descending order
// int keepLooping = 0;
//Move all of our data into temporary arrays. Also clear the stored data (not necessary but.. just incase)
for(int i=0; i <= maxStorage; i++)
{
//Get the largest occurance of a single runtime
if(highestCount < numRuntime[i])
{
highestCount = numRuntime[i];
}
tempRuntime[i] = storedRuntime[i]; storedRuntime[i] = "Blank";
tempProc[i] = storedProc[i]; storedProc[i] = "Blank";
tempSource[i] = storedSource[i]; storedSource[i] = "Blank";
tempUsr[i] = storedUsr[i]; storedUsr[i] = "Blank";
tempSrc[i] = storedSrc[i]; storedSrc[i] = "Blank";
tempNumRuntime[i] = numRuntime[i]; numRuntime[i] = 0;
}
while(highestCount > 0)
{
for(int i=0; i <= maxStorage; i++) //For every runtime
{
if(tempNumRuntime[i] == highestCount) //If the number of occurances of that runtime is equal to our current highest
{
for(int j=0; j <= maxStorage; j++) //Find the next available slot and store the info
{
if(storedRuntime[j] == "Blank") //Found an empty spot
{
storedRuntime[j] = tempRuntime[i];
storedProc[j] = tempProc[i];
storedSource[j] = tempSource[i];
storedUsr[j] = tempUsr[i];
storedSrc[j] = tempSrc[i];
numRuntime[j] = tempNumRuntime[i];
break;
}
}
}
}
highestCount--; //Lower our 'highest' by one and continue
}
}
int main() {
char exit; //Used to stop the program from immediatly exiting
//Start everything fresh. "Blank" should never occur in the runtime logs on its own.
for(int i=0; i <= maxStorage; i++)
{
storedRuntime[i] = "Blank";
storedProc[i] = "Blank";
storedSource[i] = "Blank";
storedUsr[i] = "Blank";
storedSrc[i] = "Blank";
numRuntime[i] = 0;
}
if(readFromFile())
{
cout << "Input read successfully!\n";
}
else
{
cout << "Input failed to open, shutting down.\n";
cout << "\nEnter any letter to quit.\n";
cin >> exit;
return 1;
}
sortRuntimes();
if(writeToFile())
{
cout << "Output was successful!\n";
cout << "\nEnter any letter to quit.\n";
cin >> exit;
return 0;
}
else
{
cout << "The output file could not be opened, shutting down.\n";
cout << "\nEnter any letter to quit.\n";
cin >> exit;
return 0;
}
return 0;
}

BIN
tools/Runtime Condenser/RuntimeCondenser.exe
View File

Binary file not shown.

2
tools/runtime-condenser/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
target/**
*.rs.bk

395
tools/runtime-condenser/Cargo.lock generated Normal file
View File

@@ -0,0 +1,395 @@
[[package]]
name = "aho-corasick"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ansi_term"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "atty"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
"termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bitflags"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "clap"
version = "2.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "coco"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "dtoa"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "either"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fuchsia-zircon"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fuchsia-zircon-sys"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itoa"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "num_cpus"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "quote"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rand"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rayon"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rayon-core"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rayon-hash"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "redox_syscall"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "redox_termios"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex-syntax"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "runtime-condenser"
version = "0.1.1"
dependencies = [
"clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon-hash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "scopeguard"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde_derive"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive_internals 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_derive_internals"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
"synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_json"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "strsim"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "syn"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "synom"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "termion"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "textwrap"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "thread_local"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-width"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unicode-xid"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unreachable"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "utf8-ranges"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "vec_map"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699"
"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
"checksum atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21e50800ec991574876040fff8ee46b136a53e985286fbe6a3bdfe6421b78860"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
"checksum clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8c532887f1a292d17de05ae858a8fe50a301e196f9ef0ddb7ccd0d1d00f180"
"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3"
"checksum fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6c0581a4e363262e52b87f59ee2afe3415361c6ec35e665924eb08afe8ff159"
"checksum fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "43f3795b4bae048dc6123a6b972cadde2e676f9ded08aef6bb77f5f157684a82"
"checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum lazy_static 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "236eb37a62591d4a41a89b7763d7de3e06ca02d5ab2815446a8bae5d2f8c2d57"
"checksum libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "5ba3df4dcb460b9dfbd070d41c94c19209620c191b0340b929ce748a2bcd42d2"
"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a"
"checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0"
"checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
"checksum rand 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6475140dfd8655aeb72e1fd4b7a1cc1c202be65d71669476e392fe62532b9edd"
"checksum rayon 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed02d09394c94ffbdfdc755ad62a132e94c3224a8354e78a1200ced34df12edf"
"checksum rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e64b609139d83da75902f88fd6c01820046840a18471e4dfcd5ac7c0f46bea53"
"checksum rayon-hash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "09861171bbbe8c2c7e170e8f5833cc461bdb04b386d56918a4aa790cfe16091c"
"checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b"
"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
"checksum serde 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6eda663e865517ee783b0891a3f6eb3a253e0b0dabb46418969ee9635beadd9e"
"checksum serde_derive 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "652bc323d694dc925829725ec6c890156d8e70ae5202919869cb00fe2eff3788"
"checksum serde_derive_internals 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32f1926285523b2db55df263d2aa4eb69ddcfa7a7eade6430323637866b513ab"
"checksum serde_json 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e4586746d1974a030c48919731ecffd0ed28d0c40749d0d18d43b3a7d6c9b20e"
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693"
"checksum thread_local 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1697c4b57aeeb7a536b647165a2825faddffb1d3bad386d507709bd51a90bb14"
"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f"
"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"

18
tools/runtime-condenser/Cargo.toml Normal file
View File

@@ -0,0 +1,18 @@
[package]
authors = [
"Damian <damian@autistici.org>",
"PJB3005 <pieterjan.briers@gmail.com>"
]
name = "runtime-condenser"
version = "0.1.1"
publish = false
[dependencies]
clap = "2.27.1"
lazy_static = "0.2.10"
serde = "1.0.21"
serde_derive = "1.0.21"
serde_json = "1.0.6"
regex = "0.2.2"
rayon = "0.9.0"
rayon-hash = "0.1.0"

0
tools/Runtime Condenser/Input.txt → tools/runtime-condenser/input.txt
View File

0
tools/Runtime Condenser/Output.txt → tools/runtime-condenser/output.txt
View File

BIN
tools/runtime-condenser/runtime-condenser.exe Normal file
View File

Binary file not shown.

471
tools/runtime-condenser/src/main.rs Normal file
View File

@@ -0,0 +1,471 @@
extern crate clap;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate serde_derive;
extern crate serde;
extern crate serde_json;
extern crate regex;
extern crate rayon_hash;
extern crate rayon;
use std::cmp::Ordering;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use clap::{App, Arg};
use regex::Regex;
use rayon::prelude::*;
const DEFAULT_INPUT_FILE: &str = "input.txt";
lazy_static! {
static ref STACK_FORMATTING_REGEX: Regex = {
Regex::new(r"^(?:\.\.\.|(?:.+? \(.+?\): )?.+\(.*?\))$").unwrap()
};
}
type Runtimes = rayon_hash::HashMap<String, RuntimeData>;
struct SerializableRuntimes<'a> {
map: &'a Runtimes,
}
impl<'a> serde::Serialize for SerializableRuntimes<'a> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.collect_map(self.map)
}
}
#[derive(Serialize, Eq, PartialEq, Debug)]
struct RuntimeData {
pub details: String,
pub counter: usize,
pub kind: RuntimeKind,
}
impl std::fmt::Display for RuntimeData {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
write!(f, "{}", self.details)?;
writeln!(f, "Occurred {} times.", self.counter)?;
Ok(())
}
}
#[derive(Hash, Eq, PartialEq, Copy, Clone, Debug, Serialize)]
enum RuntimeKind {
RuntimeError,
InfiniteLoop,
RecursionLimit,
}
impl RuntimeKind {
pub fn has_poor_formatting(&self) -> bool {
*self == RuntimeKind::InfiniteLoop || *self == RuntimeKind::RecursionLimit
}
}
fn line_kind(line: &str) -> LineKind {
if line == "Infinite loop suspected--switching proc to background." {
return LineKind::InfiniteLoop;
}
if line ==
"runtime error: Maximum recursion level reached (perhaps there is an infinite loop)"
{
return LineKind::RecursionLimit;
}
if line.len() > 22 && line[9..].starts_with("] Runtime in ") {
return LineKind::Runtime;
}
if line.len() > 22 && line[9..].starts_with("] Skipped ") {
return LineKind::Skipped;
}
LineKind::Junk
}
enum LineKind {
InfiniteLoop,
RecursionLimit,
Runtime,
Skipped,
Junk,
}
fn ignore_invalid_utf8(result: &std::io::Result<String>) -> bool {
match *result {
Ok(_) => true,
Err(ref error) => {
if let std::io::ErrorKind::InvalidData = error.kind() {
eprintln!("Invalid UTF-8 input, skipping. {}", error);
return false;
}
true
}
}
}
fn parse_from_file<R: Read>(file: R) -> Runtimes {
let mut runtimes = Runtimes::default();
let reader = BufReader::new(file);
let mut lines = reader.lines().filter(ignore_invalid_utf8).map(
std::result::Result::unwrap,
);
// Manually iterate so we maintain control over the iterator
// and can pass it down mid-loop.
// A regular for loop borrows it mutably until the loop is done.
while let Some(mut line) = lines.next() {
while let Some(newline) = parse_line(&mut lines, &mut runtimes, &line) {
// If the parsing ate the next line and gave it back we do that one instead.
line = newline;
}
}
runtimes
}
fn parse_line<L: Iterator<Item = String>>(
lines: &mut L,
runtimes: &mut Runtimes,
currentline: &str,
) -> Option<String> {
match line_kind(currentline) {
LineKind::InfiniteLoop => {
// Skip next 1 line so we arrive at the "proc name:"
lines.next();
if let Some(line) = lines.next() {
parse_runtime(lines, runtimes, &line[11..], RuntimeKind::InfiniteLoop)
} else {
None
}
}
LineKind::RecursionLimit => {
// Skip next 1 line so we arrive at the "proc name:"
lines.next();
if let Some(line) = lines.next() {
parse_runtime(lines, runtimes, &line[11..], RuntimeKind::RecursionLimit)
} else {
None
}
}
LineKind::Runtime => {
parse_runtime(
lines,
runtimes,
&currentline[22..],
RuntimeKind::RuntimeError,
)
}
LineKind::Skipped => {
// Read amount of runtimes skipped
let countstart = &currentline[19..];
let endindex = countstart
.char_indices()
.take_while(|&(_, c)| c.is_digit(10))
.last()
.unwrap()
.0;
let count = countstart[..endindex + 1].parse::<usize>().unwrap();
// Now to get the key.
let key = &countstart[endindex + 14..];
if let Some(runtime) = runtimes.get_mut(key) {
runtime.counter += count;
} else {
eprintln!(
"Found skip, but we have no runtime with said key. If this is an older \
log file: ignore this. {}",
key
);
}
None
}
LineKind::Junk => None,
}
}
fn parse_runtime<L: Iterator<Item = String>>(
lines: &mut L,
runtimes: &mut Runtimes,
key: &str,
kind: RuntimeKind,
) -> Option<String> {
if runtimes.contains_key(key) {
let runtime = runtimes.get_mut(key).unwrap();
runtime.counter += 1;
// Skip lines starting with two spaces since those are the trace and other details.
// Skipping these here is faster than letting the main loop do it
// as the main loop has to do multiple equality checks,
// but here it's just checking two spaces.
for line in lines {
if !line.starts_with(" ") {
return Some(line);
}
}
// We don't have to handle poorly-formatted errors (infinite loops, recursion) here:
// the trace gets treated as junk and ignored.
// Theoretically it would be an improvement but infinite loops and stack overflows
// are too rare to care about for me to bother optimizing it.
None
} else {
let mut details = String::new();
let mut outstring = None;
for line in lines.by_ref() {
if !line.starts_with(" ") {
outstring = Some(line);
break;
}
details.push_str(&line);
details.push('\n');
}
// TODO: Maybe merge this with the above loop?
if kind.has_poor_formatting() && outstring.is_some() &&
STACK_FORMATTING_REGEX.is_match(outstring.as_ref().unwrap())
{
// RIGHT we have to handle the stack trace with special behavior
// because we can't intercept infinite loops/stack overflows in /world/Error,
// meaning they're still too poorly formatted to parse easily like a runtime.
// Thanks, Lummox.
details.push_str(outstring.as_ref().unwrap());
details.push('\n');
// We read the first line and a trace maxes out at 20 lines so
// take a MAX of 19 lines.
for line in lines.by_ref().take(19) {
// The [ is the start of a regular runtime.
// Other infinite loops or recursion limits can't match against the stack regex.
// This should prevent us missing other runtimes due to erroneous stack parsing.
if line.starts_with('[') || !STACK_FORMATTING_REGEX.is_match(&line) {
outstring = Some(line);
break;
}
details.push_str(&line);
details.push('\n');
}
}
let new_entry = RuntimeData {
details: details,
counter: 1,
kind: kind,
};
runtimes.insert(key.to_owned(), new_entry);
outstring
}
}
fn total_runtimes(runtimes: &Runtimes) -> usize {
runtimes.values().map(|data| data.counter).sum()
}
fn total_unique_runtimes(runtimes: &Runtimes) -> usize {
runtimes.len()
}
// Small struct to make sorting easier.
#[derive(Eq, PartialEq)]
struct KeyRuntimePair<'a>(&'a str, &'a RuntimeData);
impl<'a> PartialOrd for KeyRuntimePair<'a> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl<'a> Ord for KeyRuntimePair<'a> {
fn cmp(&self, other: &Self) -> Ordering {
match self.1.counter.cmp(&other.1.counter) {
// Same count, compare keys to alphabetically sort those instead.
Ordering::Equal => self.0.cmp(other.0).reverse(),
x => x,
}
}
}
fn write_runtimes<W: Write>(
runtimes: &Runtimes,
file: &mut W,
verbose: bool,
) -> std::io::Result<()> {
writeln!(
file,
"Total errors: {}. Total unique errors: {}.
--------------------------------------
\
Runtime errors:",
total_runtimes(runtimes),
total_unique_runtimes(runtimes)
)?;
let highest_count = runtimes.values().map(|r| r.counter).max().unwrap_or(0);
let width = format!("{}", highest_count).len();
// Runtimes are the most common so we pre-allocate the vector for them.
let mut all_runtimes = Vec::with_capacity(total_unique_runtimes(runtimes));
let mut all_infinite_loops = Vec::new();
let mut all_recursion_limits = Vec::new();
for pair in runtimes.iter().map(|(a, b)| KeyRuntimePair(a, b)) {
match pair.1.kind {
RuntimeKind::RuntimeError => all_runtimes.push(pair),
RuntimeKind::InfiniteLoop => all_infinite_loops.push(pair),
RuntimeKind::RecursionLimit => all_recursion_limits.push(pair),
}
}
all_runtimes.sort_unstable();
all_infinite_loops.sort_unstable();
all_recursion_limits.sort_unstable();
for &KeyRuntimePair(ident, runtime) in all_runtimes.iter().rev() {
writeln!(
file,
"x{:<width$} {}",
runtime.counter,
ident,
width = width
)?;
}
writeln!(
file,
"--------------------------------------
Infinite loops:"
)?;
for &KeyRuntimePair(ident, runtime) in all_infinite_loops.iter().rev() {
writeln!(
file,
"x{:<width$} {}",
runtime.counter,
ident,
width = width
)?;
}
writeln!(
file,
"--------------------------------------
Recursion limits reached:"
)?;
for &KeyRuntimePair(ident, runtime) in all_recursion_limits.iter().rev() {
writeln!(
file,
"x{:<width$} {}",
runtime.counter,
ident,
width = width
)?;
}
if verbose {
writeln!(
file,
"--------------------------------------
Full log:"
)?;
for &KeyRuntimePair(ident, runtime) in
all_runtimes
.iter()
.rev()
.chain(all_infinite_loops.iter().rev())
.chain(all_recursion_limits.iter().rev())
{
writeln!(file,
"x{} {}\n{}",
runtime.counter,
ident,
runtime.details,)?;
}
}
Ok(())
}
fn do_output<W: Write>(
output_writer: &mut W,
runtimes: &mut Runtimes,
verbose: bool,
json: bool,
) -> std::io::Result<()> {
if json {
output_writer.write_all(
serde_json::to_string(&SerializableRuntimes { map: runtimes })
.expect("Unable to format output as JSON")
.as_bytes(),
)
} else {
write_runtimes(runtimes, output_writer, verbose)
}
}
fn main() {
let matches = App::new("/vg/station 13 Runtime Condenser")
.version("0.1")
.author("/vg/station 13 Developers")
.about(
"Compresses and filters runtime errors output by Dream Daemon, showing a more \
readable summary.",
)
.arg(Arg::with_name("json").long("json").short("j").help(
"Output in JSON.",
))
.arg(Arg::with_name("verbose").long("verbose").short("-v").help(
"Output full details and call traces (JSON mode always does this)",
))
.arg(
Arg::with_name("input")
.long("input")
.short("i")
.help("Specifies the input file to read from.")
.default_value(DEFAULT_INPUT_FILE)
.takes_value(true)
.multiple(true),
)
.arg(
Arg::with_name("output")
.long("output")
.short("o")
.help("Specifies the output file to write to.")
.takes_value(true),
)
.get_matches();
let json = matches.is_present("json");
let verbose = matches.is_present("verbose");
let input: Vec<&str> = matches.values_of("input").unwrap().collect();
let output = matches.value_of("output");
let mut final_runtimes = Runtimes::default();
let partials: Vec<Runtimes> = input
.par_iter()
.map(|filename| {
let input_file = File::open(filename).expect("Error opening input file.");
parse_from_file(input_file)
})
.collect();
for hashmap in partials {
for (ident, runtime) in hashmap {
if final_runtimes.contains_key(&ident) {
let final_runtime = final_runtimes.get_mut(&ident).unwrap();
final_runtime.counter += runtime.counter;
} else {
let _ = final_runtimes.insert(ident, runtime);
}
}
}
let stdout = std::io::stdout();
match output {
Some(file_name) => {
let mut buffered_out =
BufWriter::new(File::create(file_name).expect("Error creating file"));
do_output(&mut buffered_out, &mut final_runtimes, verbose, json)
}
None => do_output(&mut stdout.lock(), &mut final_runtimes, verbose, json),
}.expect("Error outputting to file.");
}