Merge pull request #1855 from tgstation/DoublePersist [APIDeploy][NugetDeploy]

Autismo Feature Saturday
This commit is contained in:
Jordan Dominion
2024-08-11 08:40:29 -04:00
committed by GitHub
35 changed files with 373 additions and 96 deletions
+5 -5
View File
@@ -3,12 +3,12 @@
<!-- Integration tests will ensure they match across the board -->
<Import Project="WebpanelVersion.props" />
<PropertyGroup>
<TgsCoreVersion>6.8.0</TgsCoreVersion>
<TgsConfigVersion>5.1.0</TgsConfigVersion>
<TgsApiVersion>10.6.0</TgsApiVersion>
<TgsCoreVersion>6.9.0</TgsCoreVersion>
<TgsConfigVersion>5.2.0</TgsConfigVersion>
<TgsApiVersion>10.7.0</TgsApiVersion>
<TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
<TgsApiLibraryVersion>13.6.0</TgsApiLibraryVersion>
<TgsClientVersion>15.6.0</TgsClientVersion>
<TgsApiLibraryVersion>13.7.0</TgsApiLibraryVersion>
<TgsClientVersion>16.0.0</TgsClientVersion>
<TgsDmapiVersion>7.2.0</TgsDmapiVersion>
<TgsInteropVersion>5.9.0</TgsInteropVersion>
<TgsHostWatchdogVersion>1.4.1</TgsHostWatchdogVersion>
@@ -108,5 +108,11 @@ namespace Tgstation.Server.Api.Models
/// </summary>
[Description("Reconnect chat bot")]
ReconnectChatBot,
/// <summary>
/// When a repository is recloned.
/// </summary>
[Description("Reclone repository")]
RepositoryReclone,
}
}
@@ -16,5 +16,10 @@ namespace Tgstation.Server.Api.Models.Response
/// The latest available version of the Tgstation.Server.Host assembly from the upstream repository. If <see cref="Version.Major"/> is not equal to 4 the update cannot be applied due to API changes.
/// </summary>
public Version? LatestVersion { get; set; }
/// <summary>
/// This response is cached. This field indicates the <see cref="DateTimeOffset"/> when it was generated.
/// </summary>
public DateTimeOffset? GeneratedAt { get; set; }
}
}
@@ -59,5 +59,11 @@ namespace Tgstation.Server.Api.Models.Response
/// </summary>
[ResponseOptions]
public bool? CurrentAllowWebclient { get; set; }
/// <summary>
/// The amount of RAM in use by the game server in bytes.
/// </summary>
[ResponseOptions]
public long? ImmediateMemoryUsage { get; set; }
}
}
@@ -82,5 +82,10 @@ namespace Tgstation.Server.Api.Rights
/// User may change submodule update settings.
/// </summary>
ChangeSubmoduleUpdate = 1 << 13,
/// <summary>
/// User may trigger repository recloning.
/// </summary>
Reclone = 1 << 14,
}
}
@@ -24,7 +24,7 @@ namespace Tgstation.Server.Client
}
/// <inheritdoc />
public ValueTask<AdministrationResponse> Read(CancellationToken cancellationToken) => ApiClient.Read<AdministrationResponse>(Routes.Administration, cancellationToken);
public ValueTask<AdministrationResponse> Read(bool forceFresh, CancellationToken cancellationToken) => ApiClient.Read<AdministrationResponse>($"{Routes.Administration}?fresh={forceFresh}", cancellationToken);
/// <inheritdoc />
public async ValueTask<ServerUpdateResponse> Update(
@@ -43,7 +43,10 @@ namespace Tgstation.Server.Client.Components
public ValueTask<JobResponse> Restart(CancellationToken cancellationToken) => apiClient.Patch<JobResponse>(Routes.DreamDaemon, instance.Id!.Value, cancellationToken);
/// <inheritdoc />
public ValueTask<DreamDaemonResponse> Read(CancellationToken cancellationToken) => apiClient.Read<DreamDaemonResponse>(Routes.DreamDaemon, instance.Id!.Value, cancellationToken);
public ValueTask<DreamDaemonResponse> Read(CancellationToken cancellationToken) => apiClient.Read<DreamDaemonResponse>(
Routes.DreamDaemon,
instance.Id!.Value,
cancellationToken);
/// <inheritdoc />
public ValueTask<DreamDaemonResponse> Update(DreamDaemonRequest dreamDaemon, CancellationToken cancellationToken) => apiClient.Update<DreamDaemonRequest, DreamDaemonResponse>(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id!.Value, cancellationToken);
@@ -16,7 +16,7 @@ namespace Tgstation.Server.Client.Components
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="DreamDaemonResponse"/> information.</returns>
ValueTask<DreamDaemonResponse> Read(CancellationToken cancellationToken);
ValueTask<DreamDaemonResponse> Read(CancellationToken cancellationToken = default);
/// <summary>
/// Start <see cref="DreamDaemonResponse"/>.
@@ -40,5 +40,12 @@ namespace Tgstation.Server.Client.Components
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="RepositoryResponse"/>.</returns>
ValueTask<RepositoryResponse> Delete(CancellationToken cancellationToken);
/// <summary>
/// Deletes and reclones the repository.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="RepositoryResponse"/>.</returns>
ValueTask<RepositoryResponse> Reclone(CancellationToken cancellationToken);
}
}
@@ -44,5 +44,8 @@ namespace Tgstation.Server.Client.Components
/// <inheritdoc />
public ValueTask<RepositoryResponse> Update(RepositoryUpdateRequest repository, CancellationToken cancellationToken) => apiClient.Update<RepositoryUpdateRequest, RepositoryResponse>(Routes.Repository, repository ?? throw new ArgumentNullException(nameof(repository)), instance.Id!.Value, cancellationToken);
/// <inheritdoc />
public ValueTask<RepositoryResponse> Reclone(CancellationToken cancellationToken) => apiClient.Patch<RepositoryResponse>(Routes.Repository, instance.Id!.Value, cancellationToken);
}
}
@@ -17,9 +17,10 @@ namespace Tgstation.Server.Client
/// <summary>
/// Get the <see cref="AdministrationResponse"/> represented by the <see cref="IAdministrationClient"/>.
/// </summary>
/// <param name="forceFresh">If <see langword="true"/> the response will be forcefully regenerated.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="AdministrationResponse"/> represented by the <see cref="IAdministrationClient"/>.</returns>
ValueTask<AdministrationResponse> Read(CancellationToken cancellationToken);
ValueTask<AdministrationResponse> Read(bool forceFresh = false, CancellationToken cancellationToken = default);
/// <summary>
/// Updates the <see cref="AdministrationResponse"/> setttings.
@@ -50,7 +50,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
public async ValueTask<MessageContent> Invoke(string arguments, ChatUser user, CancellationToken cancellationToken)
{
string result;
if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO"))
if (arguments.Split(' ').Any(x => x.Equals("--repo", StringComparison.OrdinalIgnoreCase)))
{
if (repositoryManager.CloneInProgress || repositoryManager.InUse)
return new MessageContent
@@ -58,15 +58,13 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
Text = "Repository busy! Try again later",
};
using (var repo = await repositoryManager.LoadRepository(cancellationToken))
{
if (repo == null)
return new MessageContent
{
Text = "Repository unavailable!",
};
result = repo.Head;
}
using var repo = await repositoryManager.LoadRepository(cancellationToken);
if (repo == null)
return new MessageContent
{
Text = "Repository unavailable!",
};
result = repo.Head;
}
else
{
@@ -157,6 +157,8 @@ namespace Tgstation.Server.Host.Components.Engine
{
Logger.LogTrace("OD repo seems to already exist, attempting load and fetch...");
repo = await repositoryManager.LoadRepository(cancellationToken);
if (repo == null)
throw new JobException("Can't load OpenDream repository! Please delete cache from disk!");
await repo!.FetchOrigin(
progressSection1,
@@ -176,6 +178,7 @@ namespace Tgstation.Server.Host.Components.Engine
null,
null,
true,
false,
progressSection2,
cancellationToken);
@@ -224,7 +224,7 @@ namespace Tgstation.Server.Host.Components.Engine
/// <inheritdoc />
protected override string GetDreamDaemonName(Version byondVersion, out bool supportsCli)
{
supportsCli = byondVersion >= DDExeVersion;
supportsCli = byondVersion >= DDExeVersion && !sessionConfiguration.ForceUseDreamDaemonExe;
return supportsCli ? "dd.exe" : "dreamdaemon.exe";
}
@@ -12,7 +12,7 @@
RepoResetOrigin,
/// <summary>
/// Parameters: Checkout target.
/// Parameters: Checkout target, hard reset flag (If "True", this is actually a hard reset, not a checkout).
/// </summary>
[EventScript("RepoCheckout")]
RepoCheckout,
@@ -47,6 +47,7 @@ namespace Tgstation.Server.Host.Components.Repository
/// <param name="username">The optional username used for fetching from submodule repositories.</param>
/// <param name="password">The optional password used for fetching from submodule repositories.</param>
/// <param name="updateSubmodules">If a submodule update should be attempted after the merge.</param>
/// <param name="moveCurrentReference">If a hard reset to the target committish should be performed instead of a checkout.</param>
/// <param name="progressReporter">The optional <see cref="JobProgressReporter"/> to report progress of the operation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
@@ -55,6 +56,7 @@ namespace Tgstation.Server.Host.Components.Repository
string? username,
string? password,
bool updateSubmodules,
bool moveCurrentReference,
JobProgressReporter? progressReporter,
CancellationToken cancellationToken);
@@ -295,7 +295,7 @@ namespace Tgstation.Server.Host.Components.Repository
var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha;
logger.LogDebug("Merge conflict, aborting and reverting to {revertTarget}", revertTo);
progressReporter.ReportProgress(0);
RawCheckout(revertTo, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken);
RawCheckout(revertTo, false, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
@@ -376,19 +376,21 @@ namespace Tgstation.Server.Host.Components.Repository
string? username,
string? password,
bool updateSubmodules,
bool moveCurrentReference,
JobProgressReporter? progressReporter,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(committish);
logger.LogDebug("Checkout object: {committish}...", committish);
await eventConsumer.HandleEvent(EventType.RepoCheckout, new List<string> { committish }, false, cancellationToken);
await eventConsumer.HandleEvent(EventType.RepoCheckout, new List<string> { committish, moveCurrentReference.ToString() }, false, cancellationToken);
await Task.Factory.StartNew(
() =>
{
libGitRepo.RemoveUntrackedFiles();
RawCheckout(
committish,
moveCurrentReference,
progressReporter?.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0),
cancellationToken);
},
@@ -879,9 +881,10 @@ namespace Tgstation.Server.Host.Components.Repository
/// Runs a blocking force checkout to <paramref name="committish"/>.
/// </summary>
/// <param name="committish">The committish to checkout.</param>
/// <param name="moveCurrentReference">If a hard reset should actually be performed.</param>
/// <param name="progressReporter">The optional <see cref="JobProgressReporter"/> for the operation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
void RawCheckout(string committish, JobProgressReporter? progressReporter, CancellationToken cancellationToken)
void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter? progressReporter, CancellationToken cancellationToken)
{
logger.LogTrace("Checkout: {committish}", committish);
@@ -900,34 +903,52 @@ namespace Tgstation.Server.Host.Components.Repository
cancellationToken.ThrowIfCancellationRequested();
void RunCheckout() => commands.Checkout(
libGitRepo,
checkoutOptions,
committish);
try
if (moveCurrentReference)
{
RunCheckout();
if (Reference == NoReference)
throw new InvalidOperationException("Cannot move current reference when not on reference!");
var gitObject = libGitRepo.Lookup(committish);
if (gitObject == null)
throw new JobException($"Could not find committish: {committish}");
var commit = gitObject.Peel<Commit>();
cancellationToken.ThrowIfCancellationRequested();
libGitRepo.Reset(ResetMode.Hard, commit, checkoutOptions);
}
catch (NotFoundException)
else
{
// Maybe (likely) a remote?
var remoteName = $"origin/{committish}";
var remoteBranch = libGitRepo.Branches.FirstOrDefault(
branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal));
cancellationToken.ThrowIfCancellationRequested();
void RunCheckout() => commands.Checkout(
libGitRepo,
checkoutOptions,
committish);
if (remoteBranch == default)
throw;
try
{
RunCheckout();
}
catch (NotFoundException)
{
// Maybe (likely) a remote?
var remoteName = $"origin/{committish}";
var remoteBranch = libGitRepo.Branches.FirstOrDefault(
branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal));
cancellationToken.ThrowIfCancellationRequested();
logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName);
var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip);
if (remoteBranch == default)
throw;
libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName);
logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName);
var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip);
cancellationToken.ThrowIfCancellationRequested();
libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName);
RunCheckout();
cancellationToken.ThrowIfCancellationRequested();
RunCheckout();
}
}
cancellationToken.ThrowIfCancellationRequested();
@@ -22,11 +22,6 @@ namespace Tgstation.Server.Host.Components.Repository
/// </summary>
sealed class RepositoryUpdateService
{
/// <summary>
/// The <see cref="RepositoryUpdateRequest"/> for the <see cref="RepositoryUpdateService"/>.
/// </summary>
readonly RepositoryUpdateRequest model;
/// <summary>
/// The current <see cref="RepositorySettings"/> for the <see cref="RepositoryUpdateService"/>.
/// </summary>
@@ -50,19 +45,16 @@ namespace Tgstation.Server.Host.Components.Repository
/// <summary>
/// Initializes a new instance of the <see cref="RepositoryUpdateService"/> class.
/// </summary>
/// <param name="model">The value of <see cref="model"/>.</param>
/// <param name="currentModel">The value of <see cref="currentModel"/>.</param>
/// <param name="initiatingUser">The value of <see cref="initiatingUser"/>.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
/// <param name="instanceId">The value of <see cref="instanceId"/>.</param>
public RepositoryUpdateService(
RepositoryUpdateRequest model,
RepositorySettings currentModel,
User initiatingUser,
ILogger<RepositoryUpdateService> logger,
long instanceId)
{
this.model = model ?? throw new ArgumentNullException(nameof(model));
this.currentModel = currentModel ?? throw new ArgumentNullException(nameof(currentModel));
this.initiatingUser = initiatingUser ?? throw new ArgumentNullException(nameof(initiatingUser));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -139,24 +131,25 @@ namespace Tgstation.Server.Host.Components.Repository
/// <summary>
/// The job entrypoint used by <see cref="Controllers.RepositoryController"/> to update the repository's current HEAD.
/// </summary>
/// <param name="instance">The <see cref="IInstanceCore"/> the job is running on. <see langword="null"/> only when performing an instance move operation.</param>
/// <param name="model">The <see cref="RepositoryUpdateRequest"/>.</param>
/// <param name="instance">The <see cref="IInstanceCore"/> the job is running on.</param>
/// <param name="databaseContextFactory">The <see cref="IDatabaseContextFactory"/> for the operation.</param>
/// <param name="job">The running <see cref="Job"/>, ignored.</param>
/// <param name="progressReporter">The <see cref="JobProgressReporter"/> for the job.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
#pragma warning disable CA1502, CA1506 // TODO: Decomplexify
public async ValueTask RepositoryUpdateJob(
RepositoryUpdateRequest model,
IInstanceCore? instance,
IDatabaseContextFactory databaseContextFactory,
Job job,
JobProgressReporter progressReporter,
CancellationToken cancellationToken)
#pragma warning restore CA1502, CA1506
{
ArgumentNullException.ThrowIfNull(model);
ArgumentNullException.ThrowIfNull(instance);
_ = job; // shuts up an IDE warning
ArgumentNullException.ThrowIfNull(databaseContextFactory);
ArgumentNullException.ThrowIfNull(progressReporter);
var repoManager = instance.RepositoryManager;
using var repo = await repoManager.LoadRepository(cancellationToken) ?? throw new JobException(ErrorCode.RepoMissing);
@@ -315,6 +308,7 @@ namespace Tgstation.Server.Host.Components.Repository
currentModel.AccessUser,
currentModel.AccessToken,
updateSubmodules,
false,
NextProgressReporter("Checkout"),
cancellationToken);
await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin
@@ -579,6 +573,7 @@ namespace Tgstation.Server.Host.Components.Repository
currentModel.AccessUser,
currentModel.AccessToken,
true,
false,
progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0),
default);
@@ -588,5 +583,86 @@ namespace Tgstation.Server.Host.Components.Repository
throw;
}
}
/// <summary>
/// The job entrypoint used by <see cref="Controllers.RepositoryController"/> to reclone a repository.
/// </summary>
/// <param name="instance">The <see cref="IInstanceCore"/> the job is running on.</param>
/// <param name="databaseContextFactory">The <see cref="IDatabaseContextFactory"/> for the operation.</param>
/// <param name="progressReporter">The <see cref="JobProgressReporter"/> for the job.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
public async ValueTask RepositoryRecloneJob(
IInstanceCore? instance,
IDatabaseContextFactory databaseContextFactory,
JobProgressReporter progressReporter,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(instance);
ArgumentNullException.ThrowIfNull(databaseContextFactory);
ArgumentNullException.ThrowIfNull(progressReporter);
progressReporter.StageName = "Loading Old Repository";
Uri origin;
string? oldReference;
string oldSha;
ValueTask deleteTask;
using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken))
{
if (oldRepo == null)
throw new JobException(ErrorCode.RepoMissing);
origin = oldRepo.Origin;
oldSha = oldRepo.Head;
oldReference = oldRepo.Reference;
if (oldReference == Repository.NoReference)
oldReference = null;
progressReporter.StageName = "Deleting Old Repository";
deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken);
}
await deleteTask;
progressReporter.ReportProgress(0.1);
IRepository newRepo;
try
{
newRepo = await instance.RepositoryManager.CloneRepository(
origin,
oldReference,
currentModel.AccessUser,
currentModel.AccessToken,
progressReporter.CreateSection("Cloning New Repository", 0.8),
true, // TODO: Make configurable maybe...
cancellationToken)
?? throw new JobException("A race condition occurred while recloning the repository. Somehow, it was fully cloned instantly after being deleted!"); // I'll take lines of code that should never be hit for $10k
}
catch (Exception ex) when (ex is not JobException)
{
logger.LogWarning("Reclone failed, clearing credentials!");
// need to clear credentials here
await databaseContextFactory.UseContextTaskReturn(context =>
{
context.RepositorySettings.Attach(currentModel);
currentModel.AccessUser = null;
currentModel.AccessToken = null;
return context.Save(CancellationToken.None); // DCT: Must always run
});
throw;
}
using (newRepo)
await newRepo.CheckoutObject(
oldSha,
currentModel.AccessUser,
currentModel.AccessToken,
false,
oldReference != null,
progressReporter.CreateSection("Checking out previous Detached Commit", 0.1),
cancellationToken);
}
}
}
@@ -121,6 +121,9 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
public FifoSemaphore TopicSendSemaphore { get; }
/// <inheritdoc />
public long MemoryUsage => process.MemoryUsage;
/// <summary>
/// The <see cref="Byond.TopicSender.ITopicClient"/> for the <see cref="SessionController"/>.
/// </summary>
@@ -24,6 +24,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
WatchdogStatus Status { get; }
/// <summary>
/// Gets the memory usage of the game server in bytes.
/// </summary>
long? MemoryUsage { get; }
/// <summary>
/// If the alpha server is the active server.
/// </summary>
@@ -50,6 +50,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
}
/// <inheritdoc />
public long? MemoryUsage => GetActiveController()?.MemoryUsage;
/// <inheritdoc />
public abstract bool AlphaIsActive { get; }
@@ -29,5 +29,10 @@
/// If <see langword="true"/>, deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined.
/// </summary>
public bool DelayCleaningFailedDeployments { get; set; }
/// <summary>
/// If set dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used.
/// </summary>
public bool ForceUseDreamDaemonExe { get; set; }
}
}
@@ -7,6 +7,7 @@ using System.Threading.Tasks;
using System.Web;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -42,6 +43,11 @@ namespace Tgstation.Server.Host.Controllers
/// </summary>
const string OctokitException = "Bad GitHub API response, check configuration!";
/// <summary>
/// The <see cref="IMemoryCache"/> key for <see cref="Read(bool?, CancellationToken)"/>.
/// </summary>
static readonly object ReadCacheKey = new();
/// <summary>
/// The <see cref="IGitHubServiceFactory"/> for the <see cref="AdministrationController"/>.
/// </summary>
@@ -77,6 +83,11 @@ namespace Tgstation.Server.Host.Controllers
/// </summary>
readonly IFileTransferTicketProvider fileTransferService;
/// <summary>
/// The <see cref="IMemoryCache"/> for the <see cref="AdministrationController"/>.
/// </summary>
readonly IMemoryCache cacheService;
/// <summary>
/// The <see cref="FileLoggingConfiguration"/> for the <see cref="AdministrationController"/>.
/// </summary>
@@ -94,6 +105,7 @@ namespace Tgstation.Server.Host.Controllers
/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
/// <param name="fileTransferService">The value of <see cref="fileTransferService"/>.</param>
/// <param name="cacheService">The value of <see cref="cacheService"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="ApiController"/>.</param>
/// <param name="fileLoggingConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing value of <see cref="fileLoggingConfiguration"/>.</param>
/// <param name="apiHeadersProvider">The <see cref="IApiHeadersProvider"/> for the <see cref="ApiController"/>.</param>
@@ -107,6 +119,7 @@ namespace Tgstation.Server.Host.Controllers
IIOManager ioManager,
IPlatformIdentifier platformIdentifier,
IFileTransferTicketProvider fileTransferService,
IMemoryCache cacheService,
ILogger<AdministrationController> logger,
IOptions<FileLoggingConfiguration> fileLoggingConfigurationOptions,
IApiHeadersProvider apiHeadersProvider)
@@ -124,12 +137,14 @@ namespace Tgstation.Server.Host.Controllers
this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier));
this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService));
this.cacheService = cacheService ?? throw new ArgumentNullException(nameof(cacheService));
fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions));
}
/// <summary>
/// Get <see cref="AdministrationResponse"/> server information.
/// </summary>
/// <param name="fresh">If <see langword="true"/>, the cache should be bypassed.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="IActionResult"/> for the operation.</returns>
/// <response code="200">Retrieved <see cref="AdministrationResponse"/> data successfully.</response>
@@ -140,39 +155,56 @@ namespace Tgstation.Server.Host.Controllers
[ProducesResponseType(typeof(AdministrationResponse), 200)]
[ProducesResponseType(typeof(ErrorMessageResponse), 424)]
[ProducesResponseType(typeof(ErrorMessageResponse), 429)]
public async ValueTask<IActionResult> Read(CancellationToken cancellationToken)
public async ValueTask<IActionResult> Read([FromQuery] bool? fresh, CancellationToken cancellationToken)
{
try
{
Version? greatestVersion = null;
Uri? repoUrl = null;
try
async Task<JsonResult> CacheFactory()
{
var gitHubService = gitHubServiceFactory.CreateService();
var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken);
var releases = await gitHubService.GetTgsReleases(cancellationToken);
foreach (var kvp in releases)
Version? greatestVersion = null;
Uri? repoUrl = null;
try
{
var version = kvp.Key;
var release = kvp.Value;
if (version.Major > 3 // Forward/backward compatible but not before TGS4
&& (greatestVersion == null || version > greatestVersion))
greatestVersion = version;
var gitHubService = gitHubServiceFactory.CreateService();
var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken);
var releases = await gitHubService.GetTgsReleases(cancellationToken);
foreach (var kvp in releases)
{
var version = kvp.Key;
var release = kvp.Value;
if (version.Major > 3 // Forward/backward compatible but not before TGS4
&& (greatestVersion == null || version > greatestVersion))
greatestVersion = version;
}
repoUrl = await repositoryUrlTask;
}
catch (NotFoundException e)
{
Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!");
}
repoUrl = await repositoryUrlTask;
}
catch (NotFoundException e)
{
Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!");
return Json(new AdministrationResponse
{
LatestVersion = greatestVersion,
TrackedRepositoryUrl = repoUrl,
GeneratedAt = DateTimeOffset.UtcNow,
});
}
return Json(new AdministrationResponse
var ttl = TimeSpan.FromMinutes(30);
Task<JsonResult> task;
if (fresh == true || !cacheService.TryGetValue(ReadCacheKey, out var rawCacheObject))
{
LatestVersion = greatestVersion,
TrackedRepositoryUrl = repoUrl,
});
using var entry = cacheService.CreateEntry(ReadCacheKey);
entry.AbsoluteExpirationRelativeToNow = ttl;
entry.Value = task = CacheFactory();
}
else
task = (Task<JsonResult>)rawCacheObject!;
return await task;
}
catch (RateLimitExceededException e)
{
@@ -317,11 +317,12 @@ namespace Tgstation.Server.Host.Controllers
/// <param name="knownForcedReboot">If there was a settings change made that forced a switch to <see cref="RebootState.Restart"/>.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="IActionResult"/> of the operation.</returns>
#pragma warning disable CA1502 // TODO: Decomplexify
ValueTask<IActionResult> ReadImpl(DreamDaemonSettings? settings, bool knownForcedReboot, CancellationToken cancellationToken)
#pragma warning restore CA1502
=> WithComponentInstance(async instance =>
{
var dd = instance.Watchdog;
var metadata = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadMetadata) != 0;
var revision = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadRevision) != 0;
@@ -372,6 +373,7 @@ namespace Tgstation.Server.Host.Controllers
result.Visibility = settings.Visibility!.Value;
result.SoftRestart = rstate == RebootState.Restart;
result.SoftShutdown = rstate == RebootState.Shutdown;
result.ImmediateMemoryUsage = dd.MemoryUsage;
if (rstate == RebootState.Normal && knownForcedReboot)
result.SoftRestart = true;
@@ -223,6 +223,42 @@ namespace Tgstation.Server.Host.Controllers
return Accepted(api);
}
/// <summary>
/// Delete the repository.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="IActionResult"/> of the operation.</returns>
/// <response code="202">Job to delete the repository created successfully.</response>
/// <response code="410">The database entity for the requested instance could not be retrieved. The instance was likely detached.</response>
[HttpPatch]
[TgsAuthorize(RepositoryRights.Reclone)]
[ProducesResponseType(typeof(RepositoryResponse), 202)]
[ProducesResponseType(typeof(ErrorMessageResponse), 410)]
public async ValueTask<IActionResult> Reclone(CancellationToken cancellationToken)
{
var currentModel = await DatabaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == Instance.Id)
.FirstOrDefaultAsync(cancellationToken);
if (currentModel == default)
return this.Gone();
Logger.LogInformation("Instance {instanceId} repository reclone initiated by user {userId}", Instance.Id, AuthenticationContext.User.Require(x => x.Id));
var repositoryUpdater = CreateRepositoryUpdateService(currentModel);
var job = Job.Create(JobCode.RepositoryReclone, AuthenticationContext.User, Instance);
var api = currentModel.ToApi();
await jobManager.RegisterOperation(
job,
(core, databaseContextFactory, paramJob, progressReporter, ct) => repositoryUpdater.RepositoryRecloneJob(core, databaseContextFactory, progressReporter, ct),
cancellationToken);
api.ActiveJob = job.ToApi();
return Accepted(api);
}
/// <summary>
/// Get the repository's status.
/// </summary>
@@ -443,17 +479,12 @@ namespace Tgstation.Server.Host.Controllers
var job = Job.Create(JobCode.RepositoryUpdate, AuthenticationContext.User, Instance, RepositoryRights.CancelPendingChanges);
job.Description = description;
var repositoryUpdater = new RepositoryUpdateService(
model,
currentModel,
AuthenticationContext.User,
loggerFactory.CreateLogger<RepositoryUpdateService>(),
Instance.Require(x => x.Id));
var repositoryUpdater = CreateRepositoryUpdateService(currentModel);
// Time to access git, do it in a job
await jobManager.RegisterOperation(
job,
repositoryUpdater.RepositoryUpdateJob,
(instance, databaseContextFactory, _, progressReporter, jobToken) => repositoryUpdater.RepositoryUpdateJob(model, instance, databaseContextFactory, progressReporter, jobToken),
cancellationToken);
api.ActiveJob = job.ToApi();
@@ -494,5 +525,17 @@ namespace Tgstation.Server.Host.Controllers
cancellationToken);
return needsDbUpdate;
}
/// <summary>
/// Creates a <see cref="RepositoryUpdateService"/>.
/// </summary>
/// <param name="currentModel">The current <see cref="RepositorySettings"/>.</param>
/// <returns>A new <see cref="RepositoryUpdateService"/>.</returns>
RepositoryUpdateService CreateRepositoryUpdateService(RepositorySettings currentModel)
=> new(
currentModel,
AuthenticationContext.User,
loggerFactory.CreateLogger<RepositoryUpdateService>(),
Instance.Require(x => x.Id));
}
}
@@ -13,6 +13,11 @@ namespace Tgstation.Server.Host.System
/// </summary>
Task<int?> Lifetime { get; }
/// <summary>
/// Gets the process' memory usage in bytes.
/// </summary>
long MemoryUsage { get; }
/// <summary>
/// Set's the owned <see cref="global::System.Diagnostics.Process.PriorityClass"/> to a non-normal value.
/// </summary>
@@ -22,6 +22,9 @@ namespace Tgstation.Server.Host.System
/// <inheritdoc />
public Task<int?> Lifetime { get; }
/// <inheritdoc />
public long MemoryUsage => handle.VirtualMemorySize64;
/// <summary>
/// The <see cref="IProcessFeatures"/> for the <see cref="Process"/>.
/// </summary>
@@ -23,6 +23,7 @@ Session:
HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes
LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes
DelayCleaningFailedDeployments: false # If true, deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined
ForceUseDreamDaemonExe: false # If true, dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used.
FileLogging:
Directory: # Directory in which log files are stored. Windows default: %PROGRAMDATA%/tgstation-server. Linux default: /var/log/tgstation-server
Disable: true # Disable file logging entirely
@@ -1,10 +1,10 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Client;
@@ -54,10 +54,24 @@ namespace Tgstation.Server.Tests.Live
async Task TestRead(CancellationToken cancellationToken)
{
var model = await client.Read(cancellationToken);
var model = await client.Read(false, cancellationToken);
//we've released a few 5.x versions now, check the release checker is at least somewhat functional
Assert.IsTrue(4 < model.LatestVersion.Major);
Assert.IsNotNull(model.TrackedRepositoryUrl);
Assert.IsTrue(model.GeneratedAt.HasValue);
Assert.IsTrue(model.GeneratedAt.Value <= DateTimeOffset.UtcNow);
// test the cache
var newerModel = await client.Read(false, cancellationToken);
Assert.AreEqual(model.GeneratedAt, newerModel.GeneratedAt);
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
var newestModel = await client.Read(true, cancellationToken);
Assert.AreNotEqual(model.GeneratedAt, newestModel.GeneratedAt);
Assert.IsNotNull(newestModel.GeneratedAt);
Assert.IsTrue(model.GeneratedAt < newestModel.GeneratedAt);
}
}
}
@@ -262,7 +262,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.AreEqual(HubConnectionState.Disconnected, permlessConn.State);
// force token refreshs
await Task.WhenAll(permedUser.Administration.Read(cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask());
await Task.WhenAll(permedUser.Administration.Read(false, cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask());
if (!permlessPsId.HasValue)
{
@@ -116,6 +116,8 @@ namespace Tgstation.Server.Tests.Live.Instance
await ApiAssert.ThrowsException<ApiConflictException, RepositoryResponse>(() => Checkout(new RepositoryUpdateRequest { Reference = "master", CheckoutSha = "286bb75" }, false, false, cancellationToken), ErrorCode.RepoMismatchShaAndReference);
var updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "286bb75" }, false, false, cancellationToken);
await RecloneTest(cancellationToken);
// Fake SHA
updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, true, false, cancellationToken);
@@ -142,6 +144,23 @@ namespace Tgstation.Server.Tests.Live.Instance
await TestMergeTests(updated, prNumber, cancellationToken);
}
async ValueTask RecloneTest(CancellationToken cancellationToken)
{
var initialState = await repositoryClient.Read(cancellationToken);
Assert.IsNotNull(initialState.Reference);
Assert.IsNotNull(initialState.RevisionInformation);
Assert.IsNotNull(initialState.RevisionInformation.CommitSha);
Assert.IsNotNull(initialState.RevisionInformation.OriginCommitSha);
var reclone = await repositoryClient.Reclone(cancellationToken);
await WaitForJob(reclone.ActiveJob, 70, false, null, cancellationToken);
var newState = await repositoryClient.Read(cancellationToken);
Assert.AreEqual(initialState.Reference, newState.Reference);
Assert.AreEqual(initialState.RevisionInformation.CommitSha, newState.RevisionInformation.CommitSha);
Assert.AreEqual(initialState.RevisionInformation.OriginCommitSha, newState.RevisionInformation.OriginCommitSha);
}
async ValueTask<RepositoryResponse> Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken)
{
var newRef = isRef ? updated.Reference : updated.CheckoutSha;
@@ -150,6 +150,7 @@ namespace Tgstation.Server.Tests.Live.Instance
}, cancellationToken);
Assert.AreEqual<ushort?>(47, updated.OpenDreamTopicPort);
Assert.IsFalse(updated.ImmediateMemoryUsage.HasValue);
}
catch (ConflictException ex) when (ex.ErrorCode == ErrorCode.PortNotAvailable)
{
@@ -721,6 +722,8 @@ namespace Tgstation.Server.Tests.Live.Instance
await CheckDDPriority();
Assert.AreEqual(false, daemonStatus.SoftRestart);
Assert.AreEqual(false, daemonStatus.SoftShutdown);
Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue);
Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value);
await GracefulWatchdogShutdown(cancellationToken);
@@ -1365,6 +1368,9 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsNotNull(daemonStatus.ActiveCompileJob);
ValidateSessionId(daemonStatus, true);
Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue);
Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value);
Assert.AreEqual(initialStatus.ActiveCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
var newerCompileJob = daemonStatus.StagedCompileJob;
Assert.AreNotEqual(daemonStatus.ActiveCompileJob.EngineVersion, newerCompileJob.EngineVersion);
@@ -216,7 +216,7 @@ namespace Tgstation.Server.Tests.Live
};
var badClient = clientFactory.CreateFromToken(serverClient.Url, newToken);
await ApiAssert.ThrowsException<UnauthorizedException, AdministrationResponse>(() => badClient.Administration.Read(cancellationToken));
await ApiAssert.ThrowsException<UnauthorizedException, AdministrationResponse>(() => badClient.Administration.Read(false, cancellationToken));
await ApiAssert.ThrowsException<UnauthorizedException, ServerInformationResponse>(() => badClient.ServerInformation(cancellationToken));
}
@@ -670,7 +670,7 @@ namespace Tgstation.Server.Tests.Live
"asdfasdfasdfasdf");
await using var node1BadClient = clientFactory.CreateFromToken(node1.RootUrl, controllerUserClient.Token);
await ApiAssert.ThrowsException<UnauthorizedException, AdministrationResponse>(() => node1BadClient.Administration.Read(cancellationToken));
await ApiAssert.ThrowsException<UnauthorizedException, AdministrationResponse>(() => node1BadClient.Administration.Read(false, cancellationToken));
// check instance info is not shared
var controllerInstance = await controllerClient.Instances.CreateOrAttach(
@@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests
() => { });
const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a";
await repo.CheckoutObject(StartSha, null, null, true, new JobProgressReporter(Mock.Of<ILogger<JobProgressReporter>>(), null, (stage, progress) => { }), CancellationToken.None);
await repo.CheckoutObject(StartSha, null, null, true, false, new JobProgressReporter(Mock.Of<ILogger<JobProgressReporter>>(), null, (stage, progress) => { }), CancellationToken.None);
Assert.AreEqual(Host.Components.Repository.Repository.NoReference, repo.Reference);