From f3149e9b26cc718289a3b89938e70e2f4a7b4eaf Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 20 Sep 2017 15:17:12 -0400 Subject: [PATCH] Per repository configuration --- TGS3.json | 58 ++++++++++ TGServerService/Compiler.cs | 90 +++++++++------ TGServerService/DreamDaemon.cs | 3 +- TGServerService/InterfaceBase.cs | 1 + TGServerService/Program.cs | 12 +- TGServerService/Repository.cs | 186 +++++++++++++++++++++++++------ TGServerService/ServerService.cs | 1 + TGStationServer3.sln | 1 + 8 files changed, 276 insertions(+), 76 deletions(-) create mode 100644 TGS3.json diff --git a/TGS3.json b/TGS3.json new file mode 100644 index 0000000000..a6e21bf294 --- /dev/null +++ b/TGS3.json @@ -0,0 +1,58 @@ +{ + "documentation": [ + "Place this file in the root of your repo in order to configure TGS3 per repository options", + "This file is not required, it merely configures changelog generation, static directories, and .dlls", + "All keys except the ones below are ignored by the service" + ], + "changelog": { + "script": "path/to/changelog/generator.py", + "arguments": "space delimited \"script arguments\"", + "pip_dependancies": [ + "optional", + "list", + "of", + "pip", + "install", + "packages" + ] + }, + "synchronize_paths": [ + "list", + "of", + "regex", + "paths", + "to", + "stage", + "for", + "git", + "commit" + ], + "static_directories": [ + "list", + "of", + "directories", + "that", + "should", + "not", + "be", + "changed", + "with", + "repo", + "updates", + "et al" + ], + "dlls": [ + "list", + "of", + ".dlls", + "that", + "your", + "game", + "uses", + "these", + "will", + "be", + "handled", + "properly" + ] +} \ No newline at end of file diff --git a/TGServerService/Compiler.cs b/TGServerService/Compiler.cs index 43fb196608..f6fad6f544 100644 --- a/TGServerService/Compiler.cs +++ b/TGServerService/Compiler.cs @@ -22,8 +22,6 @@ namespace TGServerService #endregion const string StaticDirs = "Static"; - const string StaticDataDir = StaticDirs + "/data"; - const string StaticConfigDir = StaticDirs + "/config"; const string StaticBackupDir = "Static_BACKUP"; const string LibMySQLFile = "/libmysql.dll"; @@ -40,9 +38,6 @@ namespace TGServerService const string InterfaceDLLName = "TGServiceInterface.dll"; - List copyExcludeList = new List { ".git", "data", "config", "libmysql.dll" }; //shit we handle - List deleteExcludeList = new List { "data", "config", "libmysql.dll" }; //shit we handle - object CompilerLock = new object(); TGCompilerStatus compilerCurrentStatus; string lastCompilerError; @@ -130,32 +125,37 @@ namespace TGServerService return TGCompilerStatus.Uninitialized; } + void CleanGameFolderList(string GameDir, IList theList) + { + foreach (var I in theList) + { + var the_path = Path.Combine(GameDir, I); + if (Directory.Exists(the_path)) + Directory.Delete(the_path); + } + } + //we need to remove symlinks before we can recursively delete void CleanGameFolder() { if (Directory.Exists(Path.Combine(GameDirA, InterfaceDLLName))) Directory.Delete(Path.Combine(GameDirA, InterfaceDLLName)); - if (Directory.Exists(GameDirA + LibMySQLFile)) - Directory.Delete(GameDirA + LibMySQLFile); - - if (Directory.Exists(GameDirA + "/data")) - Directory.Delete(GameDirA + "/data"); - - if (Directory.Exists(GameDirA + "/config")) - Directory.Delete(GameDirA + "/config"); + var Config = LoadRepoConfig(); + if (Config != null) + { + CleanGameFolderList(GameDirA, Config.StaticDirectoryPaths); + CleanGameFolderList(GameDirA, Config.DLLPaths); + } if (Directory.Exists(Path.Combine(GameDirB, InterfaceDLLName))) Directory.Delete(Path.Combine(GameDirB, InterfaceDLLName)); - if (Directory.Exists(GameDirB + LibMySQLFile)) - Directory.Delete(GameDirB + LibMySQLFile); - - if (Directory.Exists(GameDirB + "/data")) - Directory.Delete(GameDirB + "/data"); - - if (Directory.Exists(GameDirB + "/config")) - Directory.Delete(GameDirB + "/config"); + if (Config != null) + { + CleanGameFolderList(GameDirB, Config.StaticDirectoryPaths); + CleanGameFolderList(GameDirB, Config.DLLPaths); + } if (Directory.Exists(GameDirLive)) Directory.Delete(GameDirLive); @@ -194,14 +194,14 @@ namespace TGServerService Directory.CreateDirectory(GameDirA); Directory.CreateDirectory(GameDirB); - CreateSymlink(GameDirA + "/data", StaticDataDir); - CreateSymlink(GameDirB + "/data", StaticDataDir); + var Config = LoadRepoConfig(); - CreateSymlink(GameDirA + "/config", StaticConfigDir); - CreateSymlink(GameDirB + "/config", StaticConfigDir); - - CreateSymlink(GameDirA + LibMySQLFile, StaticDirs + LibMySQLFile); - CreateSymlink(GameDirB + LibMySQLFile, StaticDirs + LibMySQLFile); + if (Config != null) { + foreach (var I in Config.StaticDirectoryPaths) + CreateSymlink(Path.Combine(GameDirA, I), Path.Combine(StaticDirs, I)); + foreach (var I in Config.DLLPaths) + CreateSymlink(Path.Combine(GameDirA, I), Path.Combine(StaticDirs, I)); + } CreateSymlink(Path.Combine(GameDirA, InterfaceDLLName), Path.Combine(StaticDirs, InterfaceDLLName)); CreateSymlink(Path.Combine(GameDirB, InterfaceDLLName), Path.Combine(StaticDirs, InterfaceDLLName)); @@ -316,16 +316,33 @@ namespace TGServerService var resurrectee = GetStagingDir(); - Program.DeleteDirectory(resurrectee, true, deleteExcludeList); + var Config = LoadRepoConfig(); + var deleteExcludeList = new List(); + if (Config != null) + { + deleteExcludeList.AddRange(Config.StaticDirectoryPaths); + deleteExcludeList.AddRange(Config.DLLPaths); + Program.DeleteDirectory(resurrectee, true, deleteExcludeList); + } Directory.CreateDirectory(resurrectee + "/.git/logs"); - if (!Directory.Exists(resurrectee + "/config")) - CreateSymlink(resurrectee + "/config", StaticConfigDir); - if (!Directory.Exists(resurrectee + "/data")) - CreateSymlink(resurrectee + "/data", StaticDataDir); - if (!File.Exists(resurrectee + LibMySQLFile)) - CreateSymlink(resurrectee + LibMySQLFile, StaticDirs + LibMySQLFile); + if (Config != null) + { + foreach (var I in Config.StaticDirectoryPaths) + { + var the_path = Path.Combine(resurrectee, I); + if (!Directory.Exists(the_path)) + CreateSymlink(Path.Combine(resurrectee, I), Path.Combine(StaticDirs, I)); + } + foreach (var I in Config.DLLPaths) + { + var the_path = Path.Combine(resurrectee, I); + if (!File.Exists(the_path)) + CreateSymlink(the_path, Path.Combine(StaticDirs, I)); + } + } + if (!File.Exists(Path.Combine(resurrectee, InterfaceDLLName))) CreateSymlink(Path.Combine(resurrectee, InterfaceDLLName), Path.Combine(StaticDirs, InterfaceDLLName)); @@ -354,7 +371,8 @@ namespace TGServerService } try { - Program.CopyDirectory(RepoPath, resurrectee, copyExcludeList); + deleteExcludeList.Add(".git"); + Program.CopyDirectory(RepoPath, resurrectee, deleteExcludeList); //just the tip const string GitLogsDir = "/.git/logs"; Program.CopyDirectory(RepoPath + GitLogsDir, resurrectee + GitLogsDir); diff --git a/TGServerService/DreamDaemon.cs b/TGServerService/DreamDaemon.cs index a0c10da0ef..825f8a5c41 100644 --- a/TGServerService/DreamDaemon.cs +++ b/TGServerService/DreamDaemon.cs @@ -38,7 +38,6 @@ namespace TGServerService //Only need 1 proc instance void InitDreamDaemon() { - UpdateInterfaceDll(false); var Reattach = Properties.Settings.Default.ReattachToDD; if (Reattach) try @@ -374,7 +373,7 @@ namespace TGServerService return; //Copy the interface dll to the static dir var InterfacePath = Assembly.GetAssembly(typeof(DDInteropCallHolder)).Location; - File.Copy(InterfacePath, targetPath, true); + Program.CopyFileForceDirectories(InterfacePath, targetPath, true); } //used by Start and Watchdog to start a DD instance diff --git a/TGServerService/InterfaceBase.cs b/TGServerService/InterfaceBase.cs index 83fb9a0642..6728d5c78f 100644 --- a/TGServerService/InterfaceBase.cs +++ b/TGServerService/InterfaceBase.cs @@ -18,6 +18,7 @@ namespace TGServerService { FindTheDroidsWereLookingFor(); InitChat(); + InitRepo(); InitByond(); InitCompiler(); InitDreamDaemon(); diff --git a/TGServerService/Program.cs b/TGServerService/Program.cs index 922ab55e7d..8a1fdff9f1 100644 --- a/TGServerService/Program.cs +++ b/TGServerService/Program.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Security.Cryptography; -using System.Text; namespace TGServerService { @@ -12,6 +11,16 @@ namespace TGServerService //Everything in this file is just generic helpers + public static void CopyFileForceDirectories(string source, string dest, bool overwrite) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(dest)); + } + catch { } //we don't care if the above errors + File.Copy(source, dest, overwrite); //if this throws errors thats all we care about + } + //http://stackoverflow.com/questions/1701457/directory-delete-doesnt-work-access-denied-error-but-under-windows-explorer-it public static void DeleteDirectory(string path, bool ContentsOnly = false, IList excludeRoot = null) { @@ -30,6 +39,7 @@ namespace TGServerService di.Delete(true); } } + static void NormalizeAndDelete(DirectoryInfo dir, IList excludeRoot) { foreach (var subDir in dir.GetDirectories()) diff --git a/TGServerService/Repository.cs b/TGServerService/Repository.cs index 3e6e266bce..2e378ec579 100644 --- a/TGServerService/Repository.cs +++ b/TGServerService/Repository.cs @@ -14,8 +14,7 @@ namespace TGServerService partial class TGStationServer : ITGRepository, IDisposable { const string RepoPath = "Repository"; - const string RepoConfig = RepoPath + "/config"; - const string RepoData = RepoPath + "/data"; + const string RepoTGS3SettingsPath = RepoPath + "/TGS3.json"; const string RepoErrorUpToDate = "Already up to date!"; const string SSHPushRemote = "ssh_push_target"; const string PrivateKeyPath = "RepoKey/private_key.txt"; @@ -30,6 +29,80 @@ namespace TGServerService Repository Repo; int currentProgress = -1; + /// + /// Repo specific information about the installation + /// Requires RepoLock and !RepoBusy to be instantiated + /// + class RepoConfig + { + public RepoConfig() + { + if (!File.Exists(RepoTGS3SettingsPath)) + return; + var rawdata = File.ReadAllText(RepoTGS3SettingsPath); + var Deserializer = new JavaScriptSerializer(); + var json = Deserializer.Deserialize>(rawdata); + try + { + var details = (IDictionary)json["changelog"]; + PathToChangelogPy = (string)details["script"]; + ChangelogPyArguments = (string)details["arguments"]; + ChangelogSupport = true; + try + { + PipDependancies = (IList)details["pip_dependancies"]; + } + catch { } + try + { + ChangelogPathsToStage = (IList)details["synchronize_paths"]; + } + catch { } + } + catch { + ChangelogSupport = false; + } + try + { + StaticDirectoryPaths = (IList)json["static_directories"]; + } + catch { } + try + { + DLLPaths = (IList)json["dlls"]; + } + catch { } + } + public readonly bool ChangelogSupport; + public readonly string PathToChangelogPy; + public readonly string ChangelogPyArguments; + public readonly IList PipDependancies = new List(); + public readonly IList ChangelogPathsToStage = new List(); + public readonly IList StaticDirectoryPaths = new List(); + public readonly IList DLLPaths = new List(); + } + + RepoConfig _CurrentRepoConfig; + + void InitRepo() + { + if(Exists()) + UpdateInterfaceDll(false); + } + + RepoConfig LoadRepoConfig() + { + if (_CurrentRepoConfig == null) + lock (RepoLock) + { + if (RepoBusy) + return null; + if (LoadRepo() == null) + _CurrentRepoConfig = new RepoConfig(); + } + return _CurrentRepoConfig; + } + //public api public bool OperationInProgress() { @@ -155,12 +228,8 @@ namespace TGServerService //create an ssh remote for pushing Repo.Network.Remotes.Add(SSHPushRemote, RepoURL.Replace("git://", "ssh://").Replace("https://", "ssh://")); - lock (configLock) - { - Program.CopyDirectory(RepoConfig, StaticConfigDir); - } - Program.CopyDirectory(RepoData, StaticDataDir, null, true); - File.Copy(RepoPath + LibMySQLFile, StaticDirs + LibMySQLFile, true); + InitialConfigureRepository(); + SendMessage("REPO: Clone complete!", ChatMessageType.DeveloperInfo); TGServerService.WriteInfo("Repository {0}:{1} successfully cloned", TGServerService.EventID.RepoClone); } @@ -185,6 +254,48 @@ namespace TGServerService } } + void InitialConfigureRepository() + { + Directory.CreateDirectory(StaticDirs); + UpdateInterfaceDll(false); + var Config = new RepoConfig(); //RepoBusy is set if we're here + foreach(var I in Config.StaticDirectoryPaths) + { + try + { + var source = Path.Combine(RepoPath, I); + var dest = Path.Combine(StaticDirs, I); + if (Directory.Exists(source)) + Program.CopyDirectory(source, dest); + else + Directory.CreateDirectory(dest); + } + catch + { + TGServerService.WriteWarning("Could not setup static directory: " + I, TGServerService.EventID.RepoConfigurationFail); + } + } + foreach(var I in Config.DLLPaths) + { + try + { + var source = Path.Combine(RepoPath, I); + if (!File.Exists(source)) + { + TGServerService.WriteWarning("Could not find DLL: " + I, TGServerService.EventID.RepoConfigurationFail); + continue; + } + var dest = Path.Combine(StaticDirs, I); + Program.CopyFileForceDirectories(source, dest, false); + } + catch + { + TGServerService.WriteWarning("Could not setup static DLL: " + I, TGServerService.EventID.RepoConfigurationFail); + } + } + _CurrentRepoConfig = Config; + } + //kicks off the cloning thread //public api public string Setup(string RepoURL, string BranchName) @@ -324,6 +435,7 @@ namespace TGServerService Commands.Checkout(Repo, sha, Opts); var res = ResetNoLock(null); UpdateSubmodules(); + _CurrentRepoConfig = new RepoConfig(); SendMessage("REPO: Checkout complete!", ChatMessageType.DeveloperInfo); TGServerService.WriteInfo("Repo checked out " + sha, TGServerService.EventID.RepoCheckout); return res; @@ -391,6 +503,7 @@ namespace TGServerService if (res != null) throw new Exception(res); UpdateSubmodules(); + _CurrentRepoConfig = new RepoConfig(); TGServerService.WriteInfo("Repo merge updated to " + originBranch.Tip.Sha, TGServerService.EventID.RepoMergeUpdate); return null; } @@ -497,6 +610,7 @@ namespace TGServerService lock (RepoLock) { var res = LoadRepo() ?? ResetNoLock(trackedBranch ? (Repo.Head.TrackedBranch ?? Repo.Head) : Repo.Head); + _CurrentRepoConfig = new RepoConfig(); if (res == null) { SendMessage(String.Format("REPO: Hard reset to {0}branch", trackedBranch ? "tracked " : ""), ChatMessageType.DeveloperInfo); @@ -604,6 +718,7 @@ namespace TGServerService if (Result == null) try { + _CurrentRepoConfig = new RepoConfig(); UpdateSubmodules(); } catch (Exception e) @@ -727,9 +842,12 @@ namespace TGServerService public string PushChangelog() { - if (!SSHAuth()) + var Config = LoadRepoConfig(); + if (Config == null) + return "Error reading changelog configuration"; + if(!Config.ChangelogSupport || !SSHAuth()) return null; - return LocalIsRemote() ? Commit() ?? Push() : "Can't push changelog: HEAD does not match tracked remote branch"; + return LocalIsRemote() ? Commit(Config) ?? Push() : "Can't push changelog: HEAD does not match tracked remote branch"; } FetchOptions GenerateFetchOptions() @@ -781,7 +899,7 @@ namespace TGServerService } } - string Commit() + string Commit(RepoConfig Config) { lock (RepoLock) { @@ -791,8 +909,8 @@ namespace TGServerService try { // Stage the file - Commands.Stage(Repo, "html/changelog.html"); - Commands.Stage(Repo, "html/changelogs/*"); + foreach(var I in Config.ChangelogPathsToStage) + Commands.Stage(Repo, I); var status = Repo.RetrieveStatus(); var sum = status.Added.Count() + status.Removed.Count() + status.Modified.Count(); @@ -878,9 +996,19 @@ namespace TGServerService //impl proc just for single level recursion public string GenerateChangelogImpl(out string error, bool recurse = false) { - const string ChangelogPy = RepoPath + "/tools/ss13_genchangelog.py"; - const string ChangelogHtml = RepoPath + "/html/changelog.html"; - const string ChangelogDir = RepoPath + "/html/changelogs"; + var RConfig = LoadRepoConfig(); + if (RConfig == null) + { + error = null; + return "Error loading changelog config!"; + } + if (!RConfig.ChangelogSupport) + { + error = null; + return null; + } + + string ChangelogPy = Path.Combine(RepoPath, RConfig.PathToChangelogPy); if (!Exists()) { error = "Repo does not exist!"; @@ -899,23 +1027,13 @@ namespace TGServerService error = "Missing changelog generation script!"; return null; } - if (!File.Exists(ChangelogHtml)) - { - error = "Missing changelog html!"; - return null; - } - if (!Directory.Exists(ChangelogDir)) - { - error = "Missing auto changelog directory!"; - return null; - } var Config = Properties.Settings.Default; var PythonFile = Config.PythonPath + "/python.exe"; if (!File.Exists(PythonFile)) { - error = "Cannot locate python 2.7!"; + error = "Cannot locate python!"; return null; } try @@ -925,7 +1043,7 @@ namespace TGServerService using (var python = new Process()) { python.StartInfo.FileName = PythonFile; - python.StartInfo.Arguments = String.Format("{0} {1} {2}", ChangelogPy, ChangelogHtml, ChangelogDir); + python.StartInfo.Arguments = String.Format("{0} {1}", ChangelogPy, RConfig.ChangelogPyArguments); python.StartInfo.UseShellExecute = false; python.StartInfo.RedirectStandardOutput = true; python.Start(); @@ -939,7 +1057,7 @@ namespace TGServerService } if (exitCode != 0) { - if (recurse) + if (recurse || RConfig.PipDependancies.Count == 0) { error = "Script failed!"; return result; @@ -947,12 +1065,11 @@ namespace TGServerService //update pip deps and try again string PipFile = Config.PythonPath + "/scripts/pip.exe"; - bool runningBSoup = false; - while (true) + foreach(var I in RConfig.PipDependancies) using (var pip = new Process()) { pip.StartInfo.FileName = PipFile; - pip.StartInfo.Arguments = !runningBSoup ? "install PyYaml" : "install beautifulsoup4"; + pip.StartInfo.Arguments = "install " + I; pip.StartInfo.UseShellExecute = false; pip.StartInfo.RedirectStandardOutput = true; pip.Start(); @@ -966,11 +1083,6 @@ namespace TGServerService error = "Script and pip failed!"; return result; } - - if (runningBSoup) - break; - else - runningBSoup = true; } //and recurse return GenerateChangelogImpl(out error, true); diff --git a/TGServerService/ServerService.cs b/TGServerService/ServerService.cs index 3241c4c295..061b922d9b 100644 --- a/TGServerService/ServerService.cs +++ b/TGServerService/ServerService.cs @@ -80,6 +80,7 @@ namespace TGServerService PreactionFail = 6900, InteropCallException = 7000, APIVersionMismatch = 7100, + RepoConfigurationFail = 7200, } static TGServerService ActiveService; //So everyone else can write to our eventlog diff --git a/TGStationServer3.sln b/TGStationServer3.sln index 5d14e44608..48358be2aa 100644 --- a/TGStationServer3.sln +++ b/TGStationServer3.sln @@ -73,6 +73,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DMAPI", "DMAPI", "{9032B448 DMAPI\server_tools.dm = DMAPI\server_tools.dm DMAPI\st_commands.dm = DMAPI\st_commands.dm DMAPI\st_interface.dm = DMAPI\st_interface.dm + TGS3.json = TGS3.json EndProjectSection EndProject Global