diff --git a/docs/API.dox b/docs/API.dox index 6beed23239..f33f901cb2 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -162,6 +162,20 @@ I GET "/InstanceUser" => @ref Tgstation.Server.Api.Models.InstanceUser See individual documentation of each permission enum for their usage +@section api_transfer File Transfers + +Certain responses inherit from @ref Tgstation.Server.Api.Models.FileTicketResult. These are special in that, having that model's @ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket field populated indicates there is a pending file download or upload to take place. These are handled in the "/Transfer" endpoint. + +To perform a file download make the following request: + +GET "/Transfer?ticket=<@ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket>" => application/octet-stream + +To perform a file upload make the following request: + +PUT "/Transfer?ticket=<@ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket>" application/octet-stream => OK + +File tickets are only valid for a short time after the initial request is made and should be dealt with immediately. Ensure that the file tickets are properly URL encoded. + @section api_user User Management TGS start with one user: "Admin". The password is "ISolemlySwearToDeleteTheDataDirectory" diff --git a/src/Tgstation.Server.Api/Models/Byond.cs b/src/Tgstation.Server.Api/Models/Byond.cs index 89e7db2a91..6241e28948 100644 --- a/src/Tgstation.Server.Api/Models/Byond.cs +++ b/src/Tgstation.Server.Api/Models/Byond.cs @@ -1,12 +1,11 @@ -using System; -using Tgstation.Server.Api.Models.Internal; +using System; namespace Tgstation.Server.Api.Models { /// - /// Represents a BYOND installation. is used to upload custom BYOND version zip files, though must still be set. + /// Represents a BYOND installation. is used to upload custom BYOND version zip files, though must still be set. /// - public sealed class Byond : RawData + public sealed class Byond : FileTicketResult { /// /// The of the installation used for new compiles. Will be if the user does not have permission to view it or there is no BYOND version installed. Only considers the and numbers. @@ -17,5 +16,10 @@ namespace Tgstation.Server.Api.Models /// The being used to install a new /// public Job? InstallJob { get; set; } + + /// + /// If a custom BYOND version is to be uploaded. + /// + public bool? UploadCustomZip { get; set; } } } diff --git a/src/Tgstation.Server.Api/Models/ConfigurationFile.cs b/src/Tgstation.Server.Api/Models/ConfigurationFile.cs index 83f952d9ca..7d5e0168e9 100644 --- a/src/Tgstation.Server.Api/Models/ConfigurationFile.cs +++ b/src/Tgstation.Server.Api/Models/ConfigurationFile.cs @@ -1,12 +1,11 @@ -using System.ComponentModel.DataAnnotations; -using Tgstation.Server.Api.Models.Internal; +using System.ComponentModel.DataAnnotations; namespace Tgstation.Server.Api.Models { /// /// Represents a game configuration file. Create and delete actions uncerimonuously overwrite/delete files /// - public sealed class ConfigurationFile : RawData + public sealed class ConfigurationFile : FileTicketResult { /// /// The path to the file diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 4c08b12b30..c372458735 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -166,7 +166,7 @@ namespace Tgstation.Server.Api.Models RequiresPosixSystemIdentity, /// - /// A was attem updated + /// A was updated. /// [Description("This existing file hash does not match, the file has beeen updated!")] ConfigurationFileUpdated, @@ -594,5 +594,11 @@ namespace Tgstation.Server.Api.Models /// [Description("The requested OAuth provider is disabled via configuration!")] OAuthProviderDisabled, + + /// + /// A requiring a file upload did not receive it before timing out. + /// + [Description("The job did not receive a required upload before timing out!")] + FileUploadExpired, } } diff --git a/src/Tgstation.Server.Api/Models/FileTicketResult.cs b/src/Tgstation.Server.Api/Models/FileTicketResult.cs new file mode 100644 index 0000000000..87d22a78b2 --- /dev/null +++ b/src/Tgstation.Server.Api/Models/FileTicketResult.cs @@ -0,0 +1,13 @@ +namespace Tgstation.Server.Api.Models +{ + /// + /// Response for when file transfers are necessary. + /// + public class FileTicketResult + { + /// + /// The ticket to use to access the controller. + /// + public string? FileTicket { get; set; } + } +} diff --git a/src/Tgstation.Server.Api/Models/Internal/RawData.cs b/src/Tgstation.Server.Api/Models/Internal/RawData.cs deleted file mode 100644 index 4414e9bba0..0000000000 --- a/src/Tgstation.Server.Api/Models/Internal/RawData.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Tgstation.Server.Api.Models.Internal -{ - /// - /// Represents raw bytes. - /// - public abstract class RawData - { - /// - /// The bytes of the . - /// -#pragma warning disable CA1819, SA1011 // Properties should not return arrays, Closing square bracket should be followed by a space - public byte[]? Content { get; set; } -#pragma warning restore CA1819, SA1011 // Properties should not return arrays, Closing square bracket should be followed by a space - } -} diff --git a/src/Tgstation.Server.Api/Models/Limits.cs b/src/Tgstation.Server.Api/Models/Limits.cs index c7108f2f25..6acd8a35bf 100644 --- a/src/Tgstation.Server.Api/Models/Limits.cs +++ b/src/Tgstation.Server.Api/Models/Limits.cs @@ -1,3 +1,5 @@ +using System; + namespace Tgstation.Server.Api.Models { /// @@ -19,5 +21,10 @@ namespace Tgstation.Server.Api.Models /// Length limit for git commit SHAs. /// public const int MaximumCommitShaLength = 40; + + /// + /// The maximum size for file transfers. + /// + public const int MaximumFileTransferSize = Int32.MaxValue; } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Api/Models/LogFile.cs b/src/Tgstation.Server.Api/Models/LogFile.cs index 3f67f93d11..873528fcb4 100644 --- a/src/Tgstation.Server.Api/Models/LogFile.cs +++ b/src/Tgstation.Server.Api/Models/LogFile.cs @@ -1,12 +1,11 @@ using System; -using Tgstation.Server.Api.Models.Internal; namespace Tgstation.Server.Api.Models { /// /// Represents a server log file. /// - public sealed class LogFile : RawData + public sealed class LogFile : FileTicketResult { /// /// The name of the log file. diff --git a/src/Tgstation.Server.Api/Routes.cs b/src/Tgstation.Server.Api/Routes.cs index 231475f728..64e46067ba 100644 --- a/src/Tgstation.Server.Api/Routes.cs +++ b/src/Tgstation.Server.Api/Routes.cs @@ -88,6 +88,11 @@ namespace Tgstation.Server.Api /// public const string Jobs = Root + nameof(Models.Job); + /// + /// The transfer controller. + /// + public const string Transfer = Root + "Transfer"; + /// /// The postfix for list operations /// @@ -109,7 +114,7 @@ namespace Tgstation.Server.Api public static string ListRoute(string route) => String.Format(CultureInfo.InvariantCulture, "{0}/{1}", route, List); /// - /// Sanitize a path for use in a GET . + /// Sanitize a path for use in a GET . /// /// The path to sanitize. /// The sanitized path. diff --git a/src/Tgstation.Server.Client/AdministrationClient.cs b/src/Tgstation.Server.Client/AdministrationClient.cs index ec1d99513d..ca00100e8b 100644 --- a/src/Tgstation.Server.Client/AdministrationClient.cs +++ b/src/Tgstation.Server.Client/AdministrationClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -38,10 +39,25 @@ namespace Tgstation.Server.Client public Task> ListLogs(CancellationToken cancellationToken) => apiClient.Read>(Routes.Logs, cancellationToken); /// - public Task GetLog(LogFile logFile, CancellationToken cancellationToken) => apiClient.Read( - Routes.Logs + Routes.SanitizeGetPath( - HttpUtility.UrlEncode( - logFile?.Name ?? throw new ArgumentNullException(nameof(logFile)))), - cancellationToken); + public async Task> GetLog(LogFile logFile, CancellationToken cancellationToken) + { + var resultFile = await apiClient.Read( + Routes.Logs + Routes.SanitizeGetPath( + HttpUtility.UrlEncode( + logFile?.Name ?? throw new ArgumentNullException(nameof(logFile)))), + cancellationToken) + .ConfigureAwait(false); + + var stream = await apiClient.Download(resultFile, cancellationToken).ConfigureAwait(false); + try + { + return Tuple.Create(resultFile, stream); + } + catch + { + stream.Dispose(); + throw; + } + } } } diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index 0e107e91f8..558628d0e3 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -1,15 +1,18 @@ -using Newtonsoft.Json; +using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Mime; using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Web; using Tgstation.Server.Api; using Tgstation.Server.Api.Models; @@ -143,29 +146,71 @@ namespace Tgstation.Server.Client /// If this is a token refresh operation. /// The for the operation /// A resulting in the response on success - async Task RunRequest(string route, object? body, HttpMethod method, long? instanceId, bool tokenRefresh, CancellationToken cancellationToken) + Task RunRequest( + string route, + object? body, + HttpMethod method, + long? instanceId, + bool tokenRefresh, + CancellationToken cancellationToken) + { + HttpContent? content = null; + if(body != null) + content = new StringContent( + JsonConvert.SerializeObject(body, GetSerializerSettings()), + Encoding.UTF8, + MediaTypeNames.Application.Json); + + return RunRequest( + route, + content, + method, + instanceId, + tokenRefresh, + cancellationToken); + } + + /// + /// Main request method + /// + /// The resulting POCO type + /// The route to run + /// The of the request if any. + /// The method of the request + /// The optional instance for the request + /// If this is a token refresh operation. + /// The for the operation + /// A resulting in the response on success + async Task RunRequest( + string route, + HttpContent? content, + HttpMethod method, + long? instanceId, + bool tokenRefresh, + CancellationToken cancellationToken) { if (route == null) throw new ArgumentNullException(nameof(route)); if (method == null) throw new ArgumentNullException(nameof(method)); - if (body == null && (method == HttpMethod.Post || method == HttpMethod.Put)) - throw new InvalidOperationException("Body cannot be null for POST or PUT!"); + if (content == null && (method == HttpMethod.Post || method == HttpMethod.Put)) + throw new InvalidOperationException("content cannot be null for POST or PUT!"); HttpResponseMessage response; var fullUri = new Uri(Url, route); var serializerSettings = GetSerializerSettings(); + var fileDownload = typeof(TResult) == typeof(Stream); using (var request = new HttpRequestMessage(method, fullUri)) { - if (body != null) - request.Content = new StringContent( - JsonConvert.SerializeObject(body, serializerSettings), - Encoding.UTF8, - MediaTypeNames.Application.Json); + if (content != null) + request.Content = content; var headersToUse = tokenRefresh ? tokenRefreshHeaders! : headers; headersToUse.SetRequestHeaders(request.Headers, instanceId); + if (fileDownload) + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Octet)); + await Task.WhenAll(requestLoggers.Select(x => x.LogRequest(request, cancellationToken))).ConfigureAwait(false); response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); @@ -175,6 +220,12 @@ namespace Tgstation.Server.Client { await Task.WhenAll(requestLoggers.Select(x => x.LogResponse(response, cancellationToken))).ConfigureAwait(false); + if (fileDownload && response.IsSuccessStatusCode) + { + // just stream + return (TResult)(object)await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + } + var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false); if (!response.IsSuccessStatusCode) @@ -182,7 +233,7 @@ namespace Tgstation.Server.Client if (!tokenRefresh && response.StatusCode == HttpStatusCode.Unauthorized && await RefreshToken(cancellationToken).ConfigureAwait(false)) - return await RunRequest(route, body, method, instanceId, false, cancellationToken).ConfigureAwait(false); + return await RunRequest(route, content, method, instanceId, false, cancellationToken).ConfigureAwait(false); HandleBadResponse(response, json); } @@ -277,5 +328,41 @@ namespace Tgstation.Server.Client /// public void AddRequestLogger(IRequestLogger requestLogger) => requestLoggers.Add(requestLogger ?? throw new ArgumentNullException(nameof(requestLogger))); + + /// + public Task Download(FileTicketResult ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + return RunRequest( + $"{Routes.Transfer}?ticket={HttpUtility.UrlEncode(ticket.FileTicket)}", + null, + HttpMethod.Get, + null, + false, + cancellationToken); + } + + /// + public async Task Upload(FileTicketResult ticket, Stream? uploadStream, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + MemoryStream? memoryStream = null; + if (uploadStream == null) + memoryStream = new MemoryStream(); + + using (memoryStream) + await RunRequest( + $"{Routes.Transfer}?ticket={HttpUtility.UrlEncode(ticket.FileTicket)}", + new StreamContent(uploadStream ?? memoryStream), + HttpMethod.Put, + null, + false, + cancellationToken) + .ConfigureAwait(false); + } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Client/Components/ByondClient.cs b/src/Tgstation.Server.Client/Components/ByondClient.cs index 23d56dc64a..15eb50bb7d 100644 --- a/src/Tgstation.Server.Client/Components/ByondClient.cs +++ b/src/Tgstation.Server.Client/Components/ByondClient.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; @@ -38,6 +39,19 @@ namespace Tgstation.Server.Client.Components public Task> InstalledVersions(CancellationToken cancellationToken) => apiClient.Read>(Routes.ListRoute(Routes.Byond), instance.Id, cancellationToken); /// - public Task SetActiveVersion(Byond byond, CancellationToken cancellationToken) => apiClient.Update(Routes.Byond, byond ?? throw new ArgumentNullException(nameof(byond)), instance.Id, cancellationToken); + public async Task SetActiveVersion(Byond byond, Stream zipFileStream, CancellationToken cancellationToken) + { + var result = await apiClient.Update( + Routes.Byond, + byond ?? throw new ArgumentNullException(nameof(byond)), + instance.Id, + cancellationToken) + .ConfigureAwait(false); + + if (byond.UploadCustomZip == true) + await apiClient.Upload(result, zipFileStream, cancellationToken).ConfigureAwait(false); + + return result; + } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Client/Components/ConfigurationClient.cs b/src/Tgstation.Server.Client/Components/ConfigurationClient.cs index 3134fbcf37..f233d070c4 100644 --- a/src/Tgstation.Server.Client/Components/ConfigurationClient.cs +++ b/src/Tgstation.Server.Client/Components/ConfigurationClient.cs @@ -1,5 +1,9 @@ -using System; +using System; using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; @@ -41,17 +45,61 @@ namespace Tgstation.Server.Client.Components public Task> List(string directory, CancellationToken cancellationToken) => apiClient.Read>(Routes.ListRoute(Routes.Configuration) + Routes.SanitizeGetPath(directory), instance.Id, cancellationToken); /// - public Task Read(ConfigurationFile file, CancellationToken cancellationToken) + public async Task> Read(ConfigurationFile file, CancellationToken cancellationToken) { if (file == null) throw new ArgumentNullException(nameof(file)); - return apiClient.Read( + var configFile = await apiClient.Read( Routes.ConfigurationFile + Routes.SanitizeGetPath(file.Path ?? throw new ArgumentException("file.Path should not be null!", nameof(file))), instance.Id, - cancellationToken); + cancellationToken) + .ConfigureAwait(false); + var downloadStream = await apiClient.Download(configFile, cancellationToken).ConfigureAwait(false); + try + { + return Tuple.Create(configFile, downloadStream); + } + catch + { + downloadStream.Dispose(); + throw; + } } /// - public Task Write(ConfigurationFile file, CancellationToken cancellationToken) => apiClient.Update(Routes.Configuration, file ?? throw new ArgumentNullException(nameof(file)), instance.Id, cancellationToken); + public async Task Write(ConfigurationFile file, Stream uploadStream, CancellationToken cancellationToken) + { + MemoryStream? memoryStream = null; + if (uploadStream != null) + memoryStream = new MemoryStream(); + + using (memoryStream) + { + var configFileTask = apiClient.Update( + Routes.Configuration, + file ?? throw new ArgumentNullException(nameof(file)), + instance.Id, + cancellationToken); + + if (uploadStream != null) + await uploadStream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); + + var configFile = await configFileTask.ConfigureAwait(false); + + // minor improvement to "fix" a lost feature that used to be in API 7 + // since LastReadHash is no longer updated until the next GET request, we can use the same calculations here to generate it. + if (uploadStream != null) +#pragma warning disable CA5350 // Do not use insecure cryptographic algorithm SHA1. + using (var sha1 = new SHA1Managed()) +#pragma warning restore CA5350 // Do not use insecure cryptographic algorithm SHA1. + configFile.LastReadHash = String.Join(String.Empty, sha1.ComputeHash(memoryStream).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + else + configFile.LastReadHash = null; + + await apiClient.Upload(configFile, memoryStream, cancellationToken).ConfigureAwait(false); + + return configFile; + } + } } } diff --git a/src/Tgstation.Server.Client/Components/IByondClient.cs b/src/Tgstation.Server.Client/Components/IByondClient.cs index da4f4d45a3..44c90f64c4 100644 --- a/src/Tgstation.Server.Client/Components/IByondClient.cs +++ b/src/Tgstation.Server.Client/Components/IByondClient.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -28,8 +29,9 @@ namespace Tgstation.Server.Client.Components /// Updates the information /// /// The information to update + /// The for the .zip file if is . /// The for the operation /// A resulting in the updated information - Task SetActiveVersion(Byond byond, CancellationToken cancellationToken); + Task SetActiveVersion(Byond byond, Stream zipFileStream, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Components/IConfigurationClient.cs b/src/Tgstation.Server.Client/Components/IConfigurationClient.cs index d61e426af3..e82220c028 100644 --- a/src/Tgstation.Server.Client/Components/IConfigurationClient.cs +++ b/src/Tgstation.Server.Client/Components/IConfigurationClient.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -23,16 +25,17 @@ namespace Tgstation.Server.Client.Components /// /// The file to read /// The for the operation - /// A representing the running operation - Task Read(ConfigurationFile file, CancellationToken cancellationToken); + /// A resulting in a containing the and downloaded . + Task> Read(ConfigurationFile file, CancellationToken cancellationToken); /// /// Overwrite a file /// /// The file to write + /// The of uploaded data. If , a delete will be attempted. /// The for the operation - /// A resulting in the new - Task Write(ConfigurationFile file, CancellationToken cancellationToken); + /// A resulting in the new . + Task Write(ConfigurationFile file, Stream uploadStream, CancellationToken cancellationToken); /// /// Delete an empty diff --git a/src/Tgstation.Server.Client/IAdministrationClient.cs b/src/Tgstation.Server.Client/IAdministrationClient.cs index bbffab6f0f..b77e7475f7 100644 --- a/src/Tgstation.Server.Client/IAdministrationClient.cs +++ b/src/Tgstation.Server.Client/IAdministrationClient.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -44,7 +46,7 @@ namespace Tgstation.Server.Client /// /// The to download. /// The for the operation - /// A resulting in the downloaded . - Task GetLog(LogFile logFile, CancellationToken cancellationToken); + /// A resulting a containing the downloaded and associated . + Task> GetLog(LogFile logFile, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/IApiClient.cs b/src/Tgstation.Server.Client/IApiClient.cs index bee5ea5328..94cc657f0a 100644 --- a/src/Tgstation.Server.Client/IApiClient.cs +++ b/src/Tgstation.Server.Client/IApiClient.cs @@ -1,7 +1,9 @@ -using System; +using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; +using Tgstation.Server.Api.Models; namespace Tgstation.Server.Client { @@ -113,7 +115,7 @@ namespace Tgstation.Server.Client /// The type of the response body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Create(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -123,7 +125,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Create(string route, long instanceId, CancellationToken cancellationToken); @@ -133,7 +135,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Patch(string route, long instanceId, CancellationToken cancellationToken); @@ -143,7 +145,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Read(string route, long instanceId, CancellationToken cancellationToken); @@ -155,7 +157,7 @@ namespace Tgstation.Server.Client /// The type of the response body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Update(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -164,7 +166,7 @@ namespace Tgstation.Server.Client /// Run an HTTP DELETE request /// /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A representing the running operation Task Delete(string route, long instanceId, CancellationToken cancellationToken); @@ -175,7 +177,7 @@ namespace Tgstation.Server.Client /// The type to of the request body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A representing the running operation Task Delete(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -185,9 +187,26 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Delete(string route, long instanceId, CancellationToken cancellationToken); + + /// + /// Downloads a file for a given . + /// + /// The to download. + /// The for the operation. + /// A resulting in the downloaded . + Task Download(FileTicketResult ticket, CancellationToken cancellationToken); + + /// + /// Uploads a given for a given . + /// + /// The to download. + /// The to upload. represents an empty file. + /// The for the operation. + /// A representing the running operation. + Task Upload(FileTicketResult ticket, Stream? uploadStream, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs index 72ea830c82..5d97995cb6 100644 --- a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Text; @@ -121,10 +122,10 @@ namespace Tgstation.Server.Host.Components.Byond /// Installs a BYOND if it isn't already /// /// The BYOND to install - /// Custom zip file bytes to use. Will cause a number to be added. + /// Custom zip file to use. Will cause a number to be added. /// The for the operation /// A representing the running operation - async Task InstallVersion(Version version, byte[] versionZipBytes, CancellationToken cancellationToken) + async Task InstallVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken) { var ourTcs = new TaskCompletionSource(); Task inProgressTask; @@ -132,7 +133,7 @@ namespace Tgstation.Server.Host.Components.Byond bool installed; lock (installedVersions) { - if (versionZipBytes != null) + if (customVersionStream != null) { int customInstallationNumber = 1; do @@ -157,7 +158,7 @@ namespace Tgstation.Server.Host.Components.Byond return versionKey; } - if (versionZipBytes != null) + if (customVersionStream != null) logger.LogInformation("Installing custom BYOND version as {0}...", versionKey); else if (version.Build > 0) throw new JobException(ErrorCode.ByondNonExistentCustomVersion); @@ -168,26 +169,43 @@ namespace Tgstation.Server.Host.Components.Byond try { await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List { versionKey }, cancellationToken).ConfigureAwait(false); - var zipFileBytesTask = versionZipBytes == null - ? byondInstaller.DownloadVersion(version, cancellationToken) - : Task.FromResult(versionZipBytes); - await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); + var extractPath = ioManager.ResolvePath(versionKey); + async Task DirectoryCleanup() + { + await ioManager.DeleteDirectory(extractPath, cancellationToken).ConfigureAwait(false); + await ioManager.CreateDirectory(extractPath, cancellationToken).ConfigureAwait(false); + } + var directoryCleanupTask = DirectoryCleanup(); try { - versionZipBytes = await zipFileBytesTask.ConfigureAwait(false); - await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false); + Stream versionZipStream; + Stream downloadedStream = null; + if (customVersionStream == null) + { + var bytes = await byondInstaller.DownloadVersion(version, cancellationToken).ConfigureAwait(false); + downloadedStream = new MemoryStream(bytes); + versionZipStream = downloadedStream; + } + else + versionZipStream = customVersionStream; - var extractPath = ioManager.ResolvePath(versionKey); - logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); - await ioManager.ZipToDirectory(extractPath, versionZipBytes, cancellationToken).ConfigureAwait(false); - versionZipBytes = null; + using (downloadedStream) + { + await directoryCleanupTask.ConfigureAwait(false); + logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); + await ioManager.ZipToDirectory(extractPath, versionZipStream, cancellationToken).ConfigureAwait(false); + } await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false); // make sure to do this last because this is what tells us we have a valid version in the future - await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(versionKey), cancellationToken).ConfigureAwait(false); + await ioManager.WriteAllBytes( + ioManager.ConcatPath(versionKey, VersionFileName), + Encoding.UTF8.GetBytes(versionKey), + cancellationToken) + .ConfigureAwait(false); } catch (WebException e) { @@ -220,12 +238,12 @@ namespace Tgstation.Server.Host.Components.Byond } /// - public async Task ChangeVersion(Version version, byte[] customVersionBytes, CancellationToken cancellationToken) + public async Task ChangeVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken) { if (version == null) throw new ArgumentNullException(nameof(version)); - var versionKey = await InstallVersion(version, customVersionBytes, cancellationToken).ConfigureAwait(false); + var versionKey = await InstallVersion(version, customVersionStream, cancellationToken).ConfigureAwait(false); using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { await ioManager.WriteAllBytes(ActiveVersionFileName, Encoding.UTF8.GetBytes(versionKey), cancellationToken).ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs index 650bce0209..129fa54037 100644 --- a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs @@ -1,6 +1,7 @@ -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -25,10 +26,10 @@ namespace Tgstation.Server.Host.Components.Byond /// Change the active BYOND version /// /// The new - /// Optional s of a custom BYOND version zip file. + /// Optional of a custom BYOND version zip file. /// The for the operation /// A representing the running operation - Task ChangeVersion(Version version, byte[] customVersionBytes, CancellationToken cancellationToken); + Task ChangeVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken); /// /// Lock the current installation's location and return a @@ -38,4 +39,4 @@ namespace Tgstation.Server.Host.Components.Byond /// A resulting in the requested Task UseExecutables(Version requiredVersion, CancellationToken cancellationToken); } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 548410eeef..1045c8ed8b 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -19,6 +19,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Components { @@ -125,6 +126,11 @@ namespace Tgstation.Server.Host.Components /// readonly IServerPortProvider serverPortProvider; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for the . /// @@ -153,6 +159,7 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . + /// The value of . /// The containing the value of . public InstanceFactory( IIOManager ioManager, @@ -175,6 +182,7 @@ namespace Tgstation.Server.Host.Components ILibGit2RepositoryFactory repositoryFactory, ILibGit2Commands repositoryCommands, IServerPortProvider serverPortProvider, + IFileTransferTicketProvider fileTransferService, IOptions generalConfigurationOptions) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -197,6 +205,7 @@ namespace Tgstation.Server.Host.Components this.repositoryFactory = repositoryFactory ?? throw new ArgumentNullException(nameof(repositoryFactory)); this.repositoryCommands = repositoryCommands ?? throw new ArgumentNullException(nameof(repositoryCommands)); this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } @@ -214,7 +223,15 @@ namespace Tgstation.Server.Host.Components var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); - var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, platformIdentifier, loggerFactory.CreateLogger()); + var configuration = new StaticFiles.Configuration( + configurationIoManager, + synchronousIOManager, + symlinkFactory, + processExecutor, + postWriteHandler, + platformIdentifier, + fileTransferService, + loggerFactory.CreateLogger()); var eventConsumer = new EventConsumer(configuration); var repoManager = new RepositoryManager( repositoryFactory, diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 5805339a58..6260047ce3 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Components.StaticFiles { @@ -76,6 +77,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// readonly IPlatformIdentifier platformIdentifier; + /// + /// The for . + /// > + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for /// @@ -86,6 +92,16 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// readonly SemaphoreSlim semaphore; + /// + /// The that is triggered when is called. + /// + readonly CancellationTokenSource disposeCts; + + /// + /// The culmination of all upload file transfer callbacks. + /// + Task uploadTasks; + /// /// Construct /// @@ -95,6 +111,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// The value of /// The value of /// The value of + /// The value of . /// The value of public Configuration( IIOManager ioManager, @@ -103,6 +120,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IPlatformIdentifier platformIdentifier, + IFileTransferTicketProvider fileTransferService, ILogger logger) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -111,13 +129,21 @@ namespace Tgstation.Server.Host.Components.StaticFiles this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); semaphore = new SemaphoreSlim(1); + disposeCts = new CancellationTokenSource(); + uploadTasks = Task.CompletedTask; } /// - public void Dispose() => semaphore.Dispose(); + public void Dispose() + { + semaphore.Dispose(); + disposeCts.Cancel(); + disposeCts.Dispose(); + } /// /// Get the proper path to @@ -246,17 +272,39 @@ namespace Tgstation.Server.Host.Components.StaticFiles lock (semaphore) try { - var content = synchronousIOManager.ReadFile(path); - string sha1String; + string GetFileSha() + { + var content = synchronousIOManager.ReadFile(path); #pragma warning disable CA5350 // Do not use insecure cryptographic algorithm SHA1. - using (var sha1 = new SHA1Managed()) + using var sha1 = new SHA1Managed(); #pragma warning restore CA5350 // Do not use insecure cryptographic algorithm SHA1. - sha1String = String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + return String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + } + + var originalSha = GetFileSha(); + + var disposeToken = disposeCts.Token; + var fileTicket = fileTransferService.CreateDownload( + new FileDownloadProvider( + cancellationToken => + { + if (disposeToken.IsCancellationRequested) + return Task.FromResult(ErrorCode.InstanceOffline); + + var newSha = GetFileSha(); + if (newSha != originalSha) + return Task.FromResult(ErrorCode.ConfigurationFileUpdated); + + return Task.FromResult(null); + }, + path, + false)); + result = new ConfigurationFile { - Content = content, + FileTicket = fileTicket.FileTicket, IsDirectory = false, - LastReadHash = sha1String, + LastReadHash = originalSha, AccessDenied = false, Path = configurationRelativePath }; @@ -365,7 +413,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles } /// - public async Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, byte[] data, string previousHash, CancellationToken cancellationToken) + public async Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, string previousHash, CancellationToken cancellationToken) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); var path = ValidateConfigRelativePath(configurationRelativePath); @@ -377,20 +425,45 @@ namespace Tgstation.Server.Host.Components.StaticFiles lock (semaphore) try { - var fileHash = previousHash; - var success = synchronousIOManager.WriteFileChecked(path, data, ref fileHash, cancellationToken); - if (!success) - return; - if (data != null) - postWriteHandler.HandleWrite(path); + var fileTicket = fileTransferService.CreateUpload(); + var uploadCancellationToken = disposeCts.Token; + async Task UploadHandler() + { + using (fileTicket) + { + byte[] data; + var fileHash = previousHash; + using (var ms = new MemoryStream()) + { + using (var stream = await fileTicket.GetResult(uploadCancellationToken).ConfigureAwait(false)) + await stream.CopyToAsync(ms, uploadCancellationToken).ConfigureAwait(false); + data = ms.ToArray(); + if (data.Length == 0) + data = null; + } + + var success = synchronousIOManager.WriteFileChecked(path, data, ref fileHash, cancellationToken); + if (!success) + fileTicket.SetErrorMessage(new ErrorMessage(ErrorCode.ConfigurationFileUpdated) + { + AdditionalData = fileHash + }); + else if(data != null) + postWriteHandler.HandleWrite(path); + } + } + result = new ConfigurationFile { - Content = data, + FileTicket = fileTicket.Ticket.FileTicket, + LastReadHash = previousHash, IsDirectory = false, - LastReadHash = fileHash, AccessDenied = false, Path = configurationRelativePath }; + + lock (disposeCts) + uploadTasks = Task.WhenAll(uploadTasks, UploadHandler()); } catch (UnauthorizedAccessException) { diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs index b57497a4d1..34a42d299e 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using System; using System.Collections.Generic; using System.Threading; @@ -37,7 +37,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// The relative path in the Configuration directory /// The for the operation. If , the operation will be performed as the user of the /// The for the operation - /// A resulting in the s for the items in the directory. and will both be + /// A resulting in the s for the items in the directory. and will both be Task> ListDirectory(string configurationRelativePath, ISystemIdentity systemIdentity, CancellationToken cancellationToken); /// @@ -72,10 +72,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// The relative path in the Configuration directory /// The for the operation. If , the operation will be performed as the user of the - /// The data to write. If , the file is deleted /// The hash any existing file must match in order for the write to succeed /// The for the operation. Usage may result in partial writes /// A resulting in the updated or if the write failed due to conflicts - Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, byte[] data, string previousHash, CancellationToken cancellationToken); + Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, string previousHash, CancellationToken cancellationToken); } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index a41fb7895c..28f13c3c97 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -20,6 +20,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Controllers { @@ -56,6 +57,11 @@ namespace Tgstation.Server.Host.Controllers /// readonly IPlatformIdentifier platformIdentifier; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for the /// @@ -81,6 +87,7 @@ namespace Tgstation.Server.Host.Controllers /// The value of /// The value of /// The value of + /// The value of . /// The for the /// The containing value of /// The containing value of @@ -93,6 +100,7 @@ namespace Tgstation.Server.Host.Controllers IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, IPlatformIdentifier platformIdentifier, + IFileTransferTicketProvider fileTransferService, ILogger logger, IOptions updatesConfigurationOptions, IOptions generalConfigurationOptions, @@ -108,6 +116,7 @@ namespace Tgstation.Server.Host.Controllers this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); updatesConfiguration = updatesConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(updatesConfigurationOptions)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); @@ -386,13 +395,19 @@ namespace Tgstation.Server.Host.Controllers path); try { + var fileTransferTicket = fileTransferService.CreateDownload( + new FileDownloadProvider( + cancellationToken => Task.FromResult(null), + fullPath, + true)); + var readTask = ioManager.ReadAllBytes(fullPath, cancellationToken); return Ok(new LogFile { Name = path, LastModified = await ioManager.GetLastModified(fullPath, cancellationToken).ConfigureAwait(false), - Content = await readTask.ConfigureAwait(false) + FileTicket = fileTransferTicket.FileTicket }); } catch (IOException ex) diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index cab2dd1fb2..93dbd33cbc 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -1,7 +1,8 @@ -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -13,6 +14,7 @@ using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Controllers { @@ -27,6 +29,11 @@ namespace Tgstation.Server.Host.Controllers /// readonly IJobManager jobManager; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// Construct a /// @@ -34,12 +41,14 @@ namespace Tgstation.Server.Host.Controllers /// The for the /// The for the . /// The value of + /// The value of . /// The for the public ByondController( IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, IInstanceManager instanceManager, IJobManager jobManager, + IFileTransferTicketProvider fileTransferService, ILogger logger) : base( instanceManager, @@ -48,6 +57,7 @@ namespace Tgstation.Server.Host.Controllers logger) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); } /// @@ -104,14 +114,16 @@ namespace Tgstation.Server.Host.Controllers if (model == null) throw new ArgumentNullException(nameof(model)); + var uploadingZip = model.UploadCustomZip == true; + if (model.Version == null || model.Version.Revision != -1 - || (model.Content != null && model.Version.Build > 0)) + || (uploadingZip && model.Version.Build > 0)) return BadRequest(new ErrorMessage(ErrorCode.ModelValidationFailure)); var userByondRights = AuthenticationContext.InstanceUser.ByondRights.Value; - if ((!userByondRights.HasFlag(ByondRights.InstallOfficialOrChangeActiveVersion) && model.Content == null) - || (!userByondRights.HasFlag(ByondRights.InstallCustomVersion) && model.Content != null)) + if ((!userByondRights.HasFlag(ByondRights.InstallOfficialOrChangeActiveVersion) && !uploadingZip) + || (!userByondRights.HasFlag(ByondRights.InstallCustomVersion) && uploadingZip)) return Forbid(); // remove cruff fields @@ -120,7 +132,7 @@ namespace Tgstation.Server.Host.Controllers async instance => { var byondManager = instance.ByondManager; - if (model.Content == null && byondManager.InstalledVersions.Any(x => x == model.Version)) + if (!uploadingZip && byondManager.InstalledVersions.Any(x => x == model.Version)) { Logger.LogInformation( "User ID {0} changing instance ID {1} BYOND version to {2}", @@ -146,21 +158,61 @@ namespace Tgstation.Server.Host.Controllers // run the install through the job manager var job = new Models.Job { - Description = $"Install {(model.Content == null ? String.Empty : "custom ")}BYOND version {model.Version.Major}.{model.Version.Minor}", + Description = $"Install {(!uploadingZip ? String.Empty : "custom ")}BYOND version {model.Version.Major}.{model.Version.Minor}", StartedBy = AuthenticationContext.User, CancelRightsType = RightsType.Byond, CancelRight = (ulong)ByondRights.CancelInstall, Instance = Instance }; - await jobManager.RegisterOperation( - job, - (core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => core.ByondManager.ChangeVersion( - model.Version, - model.Content, - jobCancellationToken), - cancellationToken) - .ConfigureAwait(false); - result.InstallJob = job.ToApi(); + + IFileUploadTicket fileUploadTicket = null; + if (uploadingZip) + fileUploadTicket = fileTransferService.CreateUpload(); + + try + { + await jobManager.RegisterOperation( + job, + async (core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => + { + Stream zipFileStream = null; + if (fileUploadTicket != null) + using (fileUploadTicket) + { + var uploadStream = await fileUploadTicket.GetResult(jobCancellationToken).ConfigureAwait(false); + if (uploadStream == null) + throw new JobException(ErrorCode.FileUploadExpired); + + zipFileStream = new MemoryStream(); + try + { + await uploadStream.CopyToAsync(zipFileStream, jobCancellationToken).ConfigureAwait(false); + } + catch + { + await zipFileStream.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + using (zipFileStream) + await core.ByondManager.ChangeVersion( + model.Version, + zipFileStream, + jobCancellationToken) + .ConfigureAwait(false); + }, + cancellationToken) + .ConfigureAwait(false); + + result.InstallJob = job.ToApi(); + result.FileTicket = fileUploadTicket?.Ticket.FileTicket; + } + catch + { + fileUploadTicket?.Dispose(); + throw; + } } if ((AuthenticationContext.GetRight(RightsType.Byond) & (ulong)ByondRights.ReadActive) != 0) diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index 3a76b8bc5b..3d82b23fe8 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -77,11 +77,11 @@ namespace Tgstation.Server.Host.Controllers /// The for the operation. /// A resulting in the for the operation. /// File updated successfully. - /// File created successfully. + /// File upload ticket created successfully. [HttpPost] [TgsAuthorize(ConfigurationRights.Write)] [ProducesResponseType(typeof(ConfigurationFile), 200)] - [ProducesResponseType(typeof(ConfigurationFile), 201)] + [ProducesResponseType(typeof(ConfigurationFile), 202)] public async Task Update([FromBody] ConfigurationFile model, CancellationToken cancellationToken) { if (model == null) @@ -99,16 +99,11 @@ namespace Tgstation.Server.Host.Controllers .Write( model.Path, systemIdentity, - model.Content, model.LastReadHash, cancellationToken) .ConfigureAwait(false); - if (newFile == null) - return Conflict(new ErrorMessage(ErrorCode.ConfigurationFileUpdated)); - newFile.Content = null; - - return model.LastReadHash == null ? (IActionResult)Created(newFile) : Json(newFile); + return model.LastReadHash == null ? (IActionResult)Accepted(newFile) : Json(newFile); }) .ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/TransferController.cs b/src/Tgstation.Server.Host/Controllers/TransferController.cs new file mode 100644 index 0000000000..5dcf554842 --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/TransferController.cs @@ -0,0 +1,126 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; +using System; +using System.Linq; +using System.Net; +using System.Net.Mime; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Transfer; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// for file streaming. + /// + [Route(Routes.Transfer)] + [RequestSizeLimit(Limits.MaximumFileTransferSize)] + public sealed class TransferController : ApiController + { + /// + /// The for the . + /// + readonly IFileTransferStreamHandler fileTransferService; + + /// + /// Initializes a new instance of the . + /// + /// The for the + /// The for the + /// The value of . + /// The for the + public TransferController( + IDatabaseContext databaseContext, + IAuthenticationContextFactory authenticationContextFactory, + IFileTransferStreamHandler fileTransferService, + ILogger logger) + : base( + databaseContext, + authenticationContextFactory, + logger, + true) + { + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); + } + + /// + /// Downloads a file with a given . + /// + /// The for the download. + /// The for the operation. + /// A resulting in the of the method. + /// Started streaming download successfully. + /// The was no longer or was never valid. + [TgsAuthorize] + [HttpGet] + [Produces(MediaTypeNames.Application.Octet, MediaTypeNames.Application.Json)] + public async Task Download([FromQuery] string ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + var streamAccept = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); + if (!Request.GetTypedHeaders().Accept.Any(x => streamAccept.IsSubsetOf(x))) + return StatusCode(HttpStatusCode.NotAcceptable, new ErrorMessage(ErrorCode.BadHeaders) + { + AdditionalData = $"File downloads must accept both {MediaTypeNames.Application.Octet} and {MediaTypeNames.Application.Json}!" + }); + + var fileTicketResult = new FileTicketResult + { + FileTicket = ticket + }; + + var tuple = await fileTransferService.RetrieveDownloadStream(fileTicketResult, cancellationToken).ConfigureAwait(false); + var stream = tuple.Item1; + try + { + if (tuple.Item2 != null) + return Conflict(tuple.Item2); + + if (stream == null) + return Gone(); + + return new FileStreamResult(stream, new MediaTypeHeaderValue(MediaTypeNames.Application.Octet)); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// + /// Uploads a file with a given . + /// + /// The for the upload. + /// The for the operation. + /// A resulting in the of the method. + /// Uploaded file successfully. + /// The was no longer or was never valid. + [TgsAuthorize] + [HttpPut] + public async Task Upload([FromQuery] string ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + var fileTicketResult = new FileTicketResult + { + FileTicket = ticket + }; + + var result = await fileTransferService.SetUploadStream(fileTicketResult, Request.Body, cancellationToken).ConfigureAwait(false); + if (result != null) + return Conflict(result); + + return Created(new object()); + } + } +} diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 80d71e27c0..77cdd26224 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -40,6 +40,7 @@ using Tgstation.Server.Host.Security; using Tgstation.Server.Host.Security.OAuth; using Tgstation.Server.Host.Setup; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Core { @@ -304,12 +305,15 @@ namespace Tgstation.Server.Host.Core } // configure misc services + services.AddScoped(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService()); + services.AddSingleton(x => x.GetRequiredService()); // configure component services services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index d63c32753c..6c5a55ab69 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -122,7 +122,7 @@ namespace Tgstation.Server.Host.IO /// public async Task CopyDirectory(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) { - if (dest == null) + if (src == null) throw new ArgumentNullException(nameof(src)); if (dest == null) throw new ArgumentNullException(nameof(src)); @@ -134,12 +134,7 @@ namespace Tgstation.Server.Host.IO } /// - public string ConcatPath(params string[] paths) - { - if (paths == null) - throw new ArgumentNullException(nameof(paths)); - return Path.Combine(paths); - } + public string ConcatPath(params string[] paths) => Path.Combine(paths); /// public async Task CopyFile(string src, string dest, CancellationToken cancellationToken) @@ -308,14 +303,13 @@ namespace Tgstation.Server.Host.IO } /// - public Task ZipToDirectory(string path, byte[] zipFileBytes, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + public Task ZipToDirectory(string path, Stream zipFile, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { path = ResolvePath(path); - if (zipFileBytes == null) - throw new ArgumentNullException(nameof(zipFileBytes)); + if (zipFile == null) + throw new ArgumentNullException(nameof(zipFile)); - using var ms = new MemoryStream(zipFileBytes); - using var archive = new ZipArchive(ms, ZipArchiveMode.Read); + using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read); archive.ExtractToDirectory(path); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); @@ -329,5 +323,8 @@ namespace Tgstation.Server.Host.IO var fileInfo = new FileInfo(path); return new DateTimeOffset(fileInfo.LastWriteTimeUtc); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + + /// + public Stream GetFileStream(string path, bool shareWrite) => new FileStream(ResolvePath(path), FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), DefaultBufferSize, true); } } diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index d18e117864..2f2c3caac3 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -187,13 +188,13 @@ namespace Tgstation.Server.Host.IO Task DownloadFile(Uri url, CancellationToken cancellationToken); /// - /// Extract a set of to a given + /// Extract a set of to a given /// /// The path to unzip to - /// The s of the + /// The of the /// The for the operation /// A representing the running operation - Task ZipToDirectory(string path, byte[] zipFileBytes, CancellationToken cancellationToken); + Task ZipToDirectory(string path, Stream zipFile, CancellationToken cancellationToken); /// /// Get the of when a given was last modified. @@ -202,5 +203,13 @@ namespace Tgstation.Server.Host.IO /// The for the operation. /// A resulting in the of when the file was last modified. Task GetLastModified(string path, CancellationToken cancellationToken); + + /// + /// Gets the for a given file . + /// + /// The path of the file. + /// If should be used. + /// The of the file. + Stream GetFileStream(string path, bool shareWrite); } } diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 8ce353d0ae..dbfaee0b7a 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -194,8 +194,11 @@ namespace Tgstation.Server.Host var cancellationToken = cancellationTokenSource.Token; logger.LogTrace("Downloading zip package..."); - var updateZipData = await ioManager.DownloadFile(updateZipUrl, cancellationToken).ConfigureAwait(false); - + using var updateZipData = new MemoryStream( + await ioManager.DownloadFile( + updateZipUrl, + cancellationToken) + .ConfigureAwait(false)); try { logger.LogTrace("Exctracting zip package to {0}...", updatePath); diff --git a/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs b/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs new file mode 100644 index 0000000000..a01bdee992 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs @@ -0,0 +1,41 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Represents a file on disk to be downloaded. + /// + public sealed class FileDownloadProvider + { + /// + /// A of a to run before providing the download. If it returns a non-null , a 400 error with that code will be returned instead of a download stream. + /// + public Func> ActivationCallback { get; } + + /// + /// The full path to the file on disk to download. + /// + public string FilePath { get; } + + /// + /// If the file read stream should be allowed to share writes. + /// + public bool ShareWrite { get; } + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of . + /// The value of . + public FileDownloadProvider(Func> activationCallback, string filePath, bool shareWrite) + { + ActivationCallback = activationCallback ?? throw new ArgumentNullException(nameof(activationCallback)); + FilePath = filePath ?? throw new ArgumentNullException(nameof(filePath)); + ShareWrite = shareWrite; + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs new file mode 100644 index 0000000000..9b520f9690 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs @@ -0,0 +1,272 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Implementation of the file transfer service. + /// + sealed class FileTransferService : IFileTransferTicketProvider, IFileTransferStreamHandler, IAsyncDisposable + { + /// + /// Number of minutes before transfer ticket expire. + /// + const int TicketValidityMinutes = 5; + + /// + /// The for the . + /// + readonly ICryptographySuite cryptographySuite; + + /// + /// The for the . + /// + readonly IIOManager ioManager; + + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// of s to upload s. + /// + readonly Dictionary uploadTickets; + + /// + /// of s to s. + /// + readonly Dictionary downloadTickets; + + /// + /// that is triggered when is called. + /// + readonly CancellationTokenSource disposeCts; + + /// + /// used to update . + /// + readonly object synchronizationLock; + + /// + /// Combined of all calls. + /// + Task expireTask; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + public FileTransferService( + ICryptographySuite cryptographySuite, + IIOManager ioManager, + IAsyncDelayer asyncDelayer, + ILogger logger) + { + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + uploadTickets = new Dictionary(); + downloadTickets = new Dictionary(); + + disposeCts = new CancellationTokenSource(); + + expireTask = Task.CompletedTask; + synchronizationLock = new object(); + } + + /// + public async ValueTask DisposeAsync() + { + Task toAwait; + lock (synchronizationLock) + if (expireTask != null) + { + disposeCts.Cancel(); + disposeCts.Dispose(); + toAwait = expireTask; + expireTask = null; + } + else + toAwait = Task.CompletedTask; + + await toAwait.ConfigureAwait(false); + } + + /// + /// Creates a new . + /// + /// A new . + FileTicketResult CreateTicket() => new FileTicketResult + { + FileTicket = cryptographySuite.GetSecureString() + }; + + void QueueExpiry(Action expireAction) + { + Task oldExpireTask = null; + + async Task ExpireAsync() + { + var expireAt = DateTimeOffset.Now + TimeSpan.FromMinutes(TicketValidityMinutes); + try + { + await oldExpireTask.WithToken(disposeCts.Token).ConfigureAwait(false); + + var now = DateTimeOffset.Now; + if (now < expireAt) + await asyncDelayer.Delay(expireAt - now, disposeCts.Token).ConfigureAwait(false); + } + finally + { + expireAction(); + } + } + + lock (synchronizationLock) + { + oldExpireTask = expireTask; + expireTask = ExpireAsync(); + } + } + + /// + public FileTicketResult CreateDownload(FileDownloadProvider downloadProvider) + { + if (downloadProvider == null) + throw new ArgumentNullException(nameof(downloadProvider)); + + logger.LogDebug("Creating download ticket for path {0}", downloadProvider.FilePath); + var ticketResult = CreateTicket(); + + lock (downloadTickets) + downloadTickets.Add(ticketResult.FileTicket, downloadProvider); + + QueueExpiry(() => + { + logger.LogTrace("Expiring download ticket {0}...", ticketResult.FileTicket); + lock (downloadTickets) + downloadTickets.Remove(ticketResult.FileTicket); + }); + + logger.LogTrace("Created download ticket {0}", ticketResult.FileTicket); + + return ticketResult; + } + + /// + public IFileUploadTicket CreateUpload() + { + logger.LogDebug("Creating upload ticket..."); + var uploadTicket = new FileUploadProvider(CreateTicket()); + + lock (uploadTickets) + uploadTickets.Add(uploadTicket.Ticket.FileTicket, uploadTicket); + + QueueExpiry(() => + { + logger.LogTrace("Expiring upload ticket {0}...", uploadTicket.Ticket.FileTicket); + lock (uploadTickets) + uploadTickets.Remove(uploadTicket.Ticket.FileTicket); + + uploadTicket.Expire(); + }); + + logger.LogTrace("Created upload ticket {0}", uploadTicket.Ticket.FileTicket); + + return uploadTicket; + } + + /// + public async Task> RetrieveDownloadStream(FileTicketResult ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + FileDownloadProvider downloadProvider; + lock (downloadTickets) + { + if (!downloadTickets.TryGetValue(ticket.FileTicket, out downloadProvider)) + { + logger.LogTrace("Download ticket {0} not found!", ticket.FileTicket); + return Tuple.Create(null, null); + } + + downloadTickets.Remove(ticket.FileTicket); + } + + var errorCode = await downloadProvider.ActivationCallback(cancellationToken).ConfigureAwait(false); + if (errorCode.HasValue) + { + logger.LogDebug("Download ticket {0} failed activation!", ticket.FileTicket); + return Tuple.Create(null, new ErrorMessage(errorCode.Value)); + } + + Stream stream; + try + { + stream = ioManager.GetFileStream(downloadProvider.FilePath, downloadProvider.ShareWrite); + } + catch (IOException ex) + { + return Tuple.Create( + null, + new ErrorMessage(ErrorCode.IOError) + { + AdditionalData = ex.ToString() + }); + } + + try + { + logger.LogTrace("Ticket {0} downloading...", ticket.FileTicket); + return Tuple.Create(stream, null); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// + public async Task SetUploadStream(FileTicketResult ticket, Stream stream, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + FileUploadProvider uploadProvider; + lock (uploadTickets) + { + if (!uploadTickets.TryGetValue(ticket.FileTicket, out uploadProvider)) + { + logger.LogTrace("Upload ticket {0} not found!", ticket.FileTicket); + return new ErrorMessage(ErrorCode.ResourceNotPresent); + } + + uploadTickets.Remove(ticket.FileTicket); + } + + return await uploadProvider.Completion(stream, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs b/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs new file mode 100644 index 0000000000..21c3f3f142 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs @@ -0,0 +1,108 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Extensions; + +namespace Tgstation.Server.Host.Transfer +{ + /// + sealed class FileUploadProvider : IFileUploadTicket + { + /// + public FileTicketResult Ticket { get; } + + /// + /// The for the ticket duration. + /// + readonly CancellationTokenSource ticketExpiryCts; + + /// + /// The for the . + /// + readonly TaskCompletionSource taskCompletionSource; + + /// + /// The that completes in or when is called. + /// + readonly TaskCompletionSource completionTcs; + + /// + /// The that occurred while processing the upload if any. + /// + ErrorMessage errorMessage; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + public FileUploadProvider(FileTicketResult ticket) + { + Ticket = ticket ?? throw new ArgumentNullException(nameof(ticket)); + + ticketExpiryCts = new CancellationTokenSource(); + taskCompletionSource = new TaskCompletionSource(); + completionTcs = new TaskCompletionSource(); + } + + /// + public void Dispose() + { + ticketExpiryCts.Dispose(); + completionTcs.TrySetResult(null); + } + + /// + public async Task GetResult(CancellationToken cancellationToken) + { + using (cancellationToken.Register(() => taskCompletionSource.TrySetCanceled())) + using (ticketExpiryCts.Token.Register(() => taskCompletionSource.TrySetResult(null))) + return await taskCompletionSource.Task.ConfigureAwait(false); + } + + /// + /// Expire the . + /// + public void Expire() + { + if (!completionTcs.Task.IsCompleted) + ticketExpiryCts.Cancel(); + } + + /// + /// Resolve the for the and awaits the upload. + /// + /// The containing uploaded data. + /// The for the operation. + /// A resulting in , otherwise. + public async Task Completion(Stream stream, CancellationToken cancellationToken) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + if (ticketExpiryCts.IsCancellationRequested) + return new ErrorMessage(ErrorCode.ResourceNotPresent); + + taskCompletionSource.TrySetResult(stream); + + await completionTcs.Task.WithToken(cancellationToken).ConfigureAwait(false); + return errorMessage; + } + + /// + public void SetErrorMessage(ErrorMessage errorMessage) + { + if (errorMessage == null) +#pragma warning disable IDE0016 // Use 'throw' expression + throw new ArgumentNullException(nameof(errorMessage)); +#pragma warning restore IDE0016 // Use 'throw' expression + + if (this.errorMessage != null) + throw new InvalidOperationException("ErrorMessage already set!"); + + this.errorMessage = errorMessage; + completionTcs.TrySetResult(null); + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs b/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs new file mode 100644 index 0000000000..fdf16b455f --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs @@ -0,0 +1,31 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Reads and writes to s associated with s. + /// + public interface IFileTransferStreamHandler + { + /// + /// Sets the for a given associated with a pending upload. + /// + /// The . + /// The with uploaded data. + /// The for the operation. + /// if the upload completed successfully, otherwise. + Task SetUploadStream(FileTicketResult ticket, Stream stream, CancellationToken cancellationToken); + + /// + /// Gets the the for a given associated with a pending download. + /// + /// The . + /// The for the operation. + /// A containing either a containing the data to download or an to return. + Task> RetrieveDownloadStream(FileTicketResult ticket, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs b/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs new file mode 100644 index 0000000000..7e697e841c --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs @@ -0,0 +1,23 @@ +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Service for temporarily storing files to be downloaded or uploaded. + /// + public interface IFileTransferTicketProvider + { + /// + /// Create a for a download. + /// + /// The . + /// A new for a download. + FileTicketResult CreateDownload(FileDownloadProvider fileDownloadProvider); + + /// + /// Create a . + /// + /// A new . + IFileUploadTicket CreateUpload(); + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs b/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs new file mode 100644 index 0000000000..4eb1ea622c --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs @@ -0,0 +1,33 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// A that waits for a pending upload. + /// + public interface IFileUploadTicket : IDisposable + { + /// + /// The . + /// + FileTicketResult Ticket { get; } + + /// + /// Gets the for the uploaded file. + /// + /// The for the operation. + /// A resulting in the uploaded of the file on success, if the ticket timed out. + /// The resulting is short lived and should be buffered if it needs use outside the lifetime of the . + Task GetResult(CancellationToken cancellationToken); + + /// + /// Sets an for the upload. Will be returned in upload request as a 409 error. + /// + /// The to set. + void SetErrorMessage(ErrorMessage errorMessage); + } +} diff --git a/tests/Tgstation.Server.Tests/AdministrationTest.cs b/tests/Tgstation.Server.Tests/AdministrationTest.cs index 578e7f3e91..cddd0b0aa4 100644 --- a/tests/Tgstation.Server.Tests/AdministrationTest.cs +++ b/tests/Tgstation.Server.Tests/AdministrationTest.cs @@ -33,12 +33,12 @@ namespace Tgstation.Server.Tests var logFile = logs.First(); Assert.IsNotNull(logFile); Assert.IsFalse(String.IsNullOrWhiteSpace(logFile.Name)); - Assert.IsNull(logFile.Content); + Assert.IsNull(logFile.FileTicket); - var downloaded = await client.GetLog(logFile, cancellationToken); - Assert.AreEqual(logFile.Name, downloaded.Name); - Assert.IsTrue(logFile.LastModified <= downloaded.LastModified); - Assert.IsNull(logFile.Content); + var downloadedTuple = await client.GetLog(logFile, cancellationToken); + Assert.AreEqual(logFile.Name, downloadedTuple.Item1.Name); + Assert.IsTrue(logFile.LastModified <= downloadedTuple.Item1.LastModified); + Assert.IsNull(logFile.FileTicket); await ApiAssert.ThrowsException(() => client.GetLog(new LogFile { diff --git a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs index 9266e01da6..11a24484e4 100644 --- a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs @@ -1,4 +1,4 @@ -using Castle.Core.Logging; +using Castle.Core.Logging; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Instance { Version = new Version(5011, 1385) }; - var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); + var test = await byondClient.SetActiveVersion(newModel, null, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); await WaitForJob(test.InstallJob, 60, true, ErrorCode.ByondDownloadFail, cancellationToken).ConfigureAwait(false); } @@ -56,7 +56,7 @@ namespace Tgstation.Server.Tests.Instance { Version = TestVersion }; - var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); + var test = await byondClient.SetActiveVersion(newModel, null, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); Assert.IsNull(test.Version); await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); @@ -98,11 +98,18 @@ namespace Tgstation.Server.Tests.Instance Mock.Of>()); // get the bytes for stable - var test = await byondClient.SetActiveVersion(new Api.Models.Byond - { - Version = TestVersion, - Content = await byondInstaller.DownloadVersion(TestVersion, cancellationToken) - }, cancellationToken).ConfigureAwait(false); + using var stableBytesMs = new MemoryStream( + await byondInstaller.DownloadVersion(TestVersion, cancellationToken)); + + var test = await byondClient.SetActiveVersion( + new Api.Models.Byond + { + Version = TestVersion, + UploadCustomZip = true + }, + stableBytesMs, + cancellationToken) + .ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); @@ -114,17 +121,17 @@ namespace Tgstation.Server.Tests.Instance newSettings = await byondClient.SetActiveVersion(new Api.Models.Byond { Version = TestVersion - }, cancellationToken); + }, null, cancellationToken); Assert.IsNull(newSettings.InstallJob); await ApiAssert.ThrowsException(() => byondClient.SetActiveVersion(new Api.Models.Byond { Version = new Version(TestVersion.Major, TestVersion.Minor, 2) - }, cancellationToken), ErrorCode.ByondNonExistentCustomVersion); + }, null, cancellationToken), ErrorCode.ByondNonExistentCustomVersion); newSettings = await byondClient.SetActiveVersion(new Api.Models.Byond { Version = new Version(TestVersion.Major, TestVersion.Minor, 1) - }, cancellationToken); + }, null, cancellationToken); Assert.IsNull(newSettings.InstallJob); } } diff --git a/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs index b30178169f..26abb403c5 100644 --- a/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs @@ -1,4 +1,4 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; using System.Text; @@ -25,7 +25,8 @@ namespace Tgstation.Server.Tests.Instance { var tmp = (file.Path?.StartsWith('/') ?? false) ? '.' + file.Path : file.Path; var path = Path.Combine(instance.Path, "Configuration", tmp); - return File.Exists(path); + var result = File.Exists(path); + return result; } async Task TestDeleteDirectory(CancellationToken cancellationToken) @@ -39,18 +40,21 @@ namespace Tgstation.Server.Tests.Instance await configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken).ConfigureAwait(false); //try to delete non-empty + using var uploadMs = new MemoryStream(Encoding.UTF8.GetBytes("Hello world!")); var file = await configurationClient.Write(new ConfigurationFile { - Content = Encoding.UTF8.GetBytes("Hello world!"), Path = TestDir.Path + "/test.txt" - }, cancellationToken).ConfigureAwait(false); + }, uploadMs, cancellationToken).ConfigureAwait(false); Assert.IsTrue(FileExists(file)); + var updatedFile = await configurationClient.Read(file, cancellationToken).ConfigureAwait(false); + Assert.AreEqual(file.LastReadHash, updatedFile.Item1.LastReadHash); + await ApiAssert.ThrowsException(() => configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken), ErrorCode.ConfigurationDirectoryNotEmpty).ConfigureAwait(false); - file.Content = null; - await configurationClient.Write(file, cancellationToken).ConfigureAwait(false); + file.FileTicket = null; + await configurationClient.Write(file, null, cancellationToken).ConfigureAwait(false); await configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken).ConfigureAwait(false); } diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 0c6e8915df..35dddd9d35 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -396,6 +396,7 @@ namespace Tgstation.Server.Tests.Instance { Version = versionToInstall }, + null, cancellationToken); var byondInstallJob = await byondInstallJobTask;