diff --git a/build/OpenApiValidationSettings.json b/build/OpenApiValidationSettings.json index aa6faf8c1d..2c14eb6066 100644 --- a/build/OpenApiValidationSettings.json +++ b/build/OpenApiValidationSettings.json @@ -6,6 +6,7 @@ "no_summary": "error", "no_array_responses": "off", "parameter_order": "error", + "undefined_tag": "off", "unused_tag": "error", "operation_id_naming_convention": "off" }, @@ -44,12 +45,15 @@ "no_property_description": "off", "description_mentions_json": "error", "array_of_arrays": "error", + "inconsistent_property_type": "error", "property_case_convention": "off", - "enum_case_convention": "error" + "property_case_collision": "error", + "enum_case_convention": "error", + "undefined_required_properties": "error" }, "walker": { "no_empty_descriptions": "error", - "has_circular_references": "off", + "has_circular_references": "error", "$ref_siblings": "error", "duplicate_sibling_description": "error", "incorrect_ref_pattern": "error" @@ -76,7 +80,11 @@ "responses": { "no_response_codes": "error", "no_success_response_codes": "error", + "no_response_body": "error", "ibm_status_code_guidelines": "off" + }, + "schemas": { + "json_or_param_binary_string": "error" } } -} \ No newline at end of file +} diff --git a/build/Version.props b/build/Version.props index 3b64b3ff67..5305d27cc4 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,10 +3,10 @@ - 4.6.0 - 2.1.1 - 7.4.0 - 8.4.0 + 4.7.0 + 2.2.0 + 8.0.0 + 9.0.0 5.2.9 1.1.0 1.2.0 diff --git a/docs/API.dox b/docs/API.dox index 6beed23239..f33f901cb2 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -162,6 +162,20 @@ I GET "/InstanceUser" => @ref Tgstation.Server.Api.Models.InstanceUser See individual documentation of each permission enum for their usage +@section api_transfer File Transfers + +Certain responses inherit from @ref Tgstation.Server.Api.Models.FileTicketResult. These are special in that, having that model's @ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket field populated indicates there is a pending file download or upload to take place. These are handled in the "/Transfer" endpoint. + +To perform a file download make the following request: + +GET "/Transfer?ticket=<@ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket>" => application/octet-stream + +To perform a file upload make the following request: + +PUT "/Transfer?ticket=<@ref Tgstation.Server.Api.Models.FileTicketResult.FileTicket>" application/octet-stream => OK + +File tickets are only valid for a short time after the initial request is made and should be dealt with immediately. Ensure that the file tickets are properly URL encoded. + @section api_user User Management TGS start with one user: "Admin". The password is "ISolemlySwearToDeleteTheDataDirectory" diff --git a/src/Tgstation.Server.Api/Models/Byond.cs b/src/Tgstation.Server.Api/Models/Byond.cs index 89e7db2a91..6241e28948 100644 --- a/src/Tgstation.Server.Api/Models/Byond.cs +++ b/src/Tgstation.Server.Api/Models/Byond.cs @@ -1,12 +1,11 @@ -using System; -using Tgstation.Server.Api.Models.Internal; +using System; namespace Tgstation.Server.Api.Models { /// - /// Represents a BYOND installation. is used to upload custom BYOND version zip files, though must still be set. + /// Represents a BYOND installation. is used to upload custom BYOND version zip files, though must still be set. /// - public sealed class Byond : RawData + public sealed class Byond : FileTicketResult { /// /// The of the installation used for new compiles. Will be if the user does not have permission to view it or there is no BYOND version installed. Only considers the and numbers. @@ -17,5 +16,10 @@ namespace Tgstation.Server.Api.Models /// The being used to install a new /// public Job? InstallJob { get; set; } + + /// + /// If a custom BYOND version is to be uploaded. + /// + public bool? UploadCustomZip { get; set; } } } diff --git a/src/Tgstation.Server.Api/Models/ConfigurationFile.cs b/src/Tgstation.Server.Api/Models/ConfigurationFile.cs index 83f952d9ca..7d5e0168e9 100644 --- a/src/Tgstation.Server.Api/Models/ConfigurationFile.cs +++ b/src/Tgstation.Server.Api/Models/ConfigurationFile.cs @@ -1,12 +1,11 @@ -using System.ComponentModel.DataAnnotations; -using Tgstation.Server.Api.Models.Internal; +using System.ComponentModel.DataAnnotations; namespace Tgstation.Server.Api.Models { /// /// Represents a game configuration file. Create and delete actions uncerimonuously overwrite/delete files /// - public sealed class ConfigurationFile : RawData + public sealed class ConfigurationFile : FileTicketResult { /// /// The path to the file diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 4c08b12b30..c372458735 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -166,7 +166,7 @@ namespace Tgstation.Server.Api.Models RequiresPosixSystemIdentity, /// - /// A was attem updated + /// A was updated. /// [Description("This existing file hash does not match, the file has beeen updated!")] ConfigurationFileUpdated, @@ -594,5 +594,11 @@ namespace Tgstation.Server.Api.Models /// [Description("The requested OAuth provider is disabled via configuration!")] OAuthProviderDisabled, + + /// + /// A requiring a file upload did not receive it before timing out. + /// + [Description("The job did not receive a required upload before timing out!")] + FileUploadExpired, } } diff --git a/src/Tgstation.Server.Api/Models/FileTicketResult.cs b/src/Tgstation.Server.Api/Models/FileTicketResult.cs new file mode 100644 index 0000000000..87d22a78b2 --- /dev/null +++ b/src/Tgstation.Server.Api/Models/FileTicketResult.cs @@ -0,0 +1,13 @@ +namespace Tgstation.Server.Api.Models +{ + /// + /// Response for when file transfers are necessary. + /// + public class FileTicketResult + { + /// + /// The ticket to use to access the controller. + /// + public string? FileTicket { get; set; } + } +} diff --git a/src/Tgstation.Server.Api/Models/Internal/RawData.cs b/src/Tgstation.Server.Api/Models/Internal/RawData.cs deleted file mode 100644 index 4414e9bba0..0000000000 --- a/src/Tgstation.Server.Api/Models/Internal/RawData.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Tgstation.Server.Api.Models.Internal -{ - /// - /// Represents raw bytes. - /// - public abstract class RawData - { - /// - /// The bytes of the . - /// -#pragma warning disable CA1819, SA1011 // Properties should not return arrays, Closing square bracket should be followed by a space - public byte[]? Content { get; set; } -#pragma warning restore CA1819, SA1011 // Properties should not return arrays, Closing square bracket should be followed by a space - } -} diff --git a/src/Tgstation.Server.Api/Models/Limits.cs b/src/Tgstation.Server.Api/Models/Limits.cs index c7108f2f25..6acd8a35bf 100644 --- a/src/Tgstation.Server.Api/Models/Limits.cs +++ b/src/Tgstation.Server.Api/Models/Limits.cs @@ -1,3 +1,5 @@ +using System; + namespace Tgstation.Server.Api.Models { /// @@ -19,5 +21,10 @@ namespace Tgstation.Server.Api.Models /// Length limit for git commit SHAs. /// public const int MaximumCommitShaLength = 40; + + /// + /// The maximum size for file transfers. + /// + public const int MaximumFileTransferSize = Int32.MaxValue; } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Api/Models/LogFile.cs b/src/Tgstation.Server.Api/Models/LogFile.cs index 3f67f93d11..873528fcb4 100644 --- a/src/Tgstation.Server.Api/Models/LogFile.cs +++ b/src/Tgstation.Server.Api/Models/LogFile.cs @@ -1,12 +1,11 @@ using System; -using Tgstation.Server.Api.Models.Internal; namespace Tgstation.Server.Api.Models { /// /// Represents a server log file. /// - public sealed class LogFile : RawData + public sealed class LogFile : FileTicketResult { /// /// The name of the log file. diff --git a/src/Tgstation.Server.Api/Routes.cs b/src/Tgstation.Server.Api/Routes.cs index 231475f728..64e46067ba 100644 --- a/src/Tgstation.Server.Api/Routes.cs +++ b/src/Tgstation.Server.Api/Routes.cs @@ -88,6 +88,11 @@ namespace Tgstation.Server.Api /// public const string Jobs = Root + nameof(Models.Job); + /// + /// The transfer controller. + /// + public const string Transfer = Root + "Transfer"; + /// /// The postfix for list operations /// @@ -109,7 +114,7 @@ namespace Tgstation.Server.Api public static string ListRoute(string route) => String.Format(CultureInfo.InvariantCulture, "{0}/{1}", route, List); /// - /// Sanitize a path for use in a GET . + /// Sanitize a path for use in a GET . /// /// The path to sanitize. /// The sanitized path. diff --git a/src/Tgstation.Server.Client/AdministrationClient.cs b/src/Tgstation.Server.Client/AdministrationClient.cs index ec1d99513d..ca00100e8b 100644 --- a/src/Tgstation.Server.Client/AdministrationClient.cs +++ b/src/Tgstation.Server.Client/AdministrationClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -38,10 +39,25 @@ namespace Tgstation.Server.Client public Task> ListLogs(CancellationToken cancellationToken) => apiClient.Read>(Routes.Logs, cancellationToken); /// - public Task GetLog(LogFile logFile, CancellationToken cancellationToken) => apiClient.Read( - Routes.Logs + Routes.SanitizeGetPath( - HttpUtility.UrlEncode( - logFile?.Name ?? throw new ArgumentNullException(nameof(logFile)))), - cancellationToken); + public async Task> GetLog(LogFile logFile, CancellationToken cancellationToken) + { + var resultFile = await apiClient.Read( + Routes.Logs + Routes.SanitizeGetPath( + HttpUtility.UrlEncode( + logFile?.Name ?? throw new ArgumentNullException(nameof(logFile)))), + cancellationToken) + .ConfigureAwait(false); + + var stream = await apiClient.Download(resultFile, cancellationToken).ConfigureAwait(false); + try + { + return Tuple.Create(resultFile, stream); + } + catch + { + stream.Dispose(); + throw; + } + } } } diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index 0e107e91f8..3252782576 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -1,15 +1,18 @@ -using Newtonsoft.Json; +using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Mime; using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Web; using Tgstation.Server.Api; using Tgstation.Server.Api.Models; @@ -36,7 +39,7 @@ namespace Tgstation.Server.Client } /// - /// The for the + /// The for the /// readonly IHttpClient httpClient; @@ -143,38 +146,92 @@ namespace Tgstation.Server.Client /// If this is a token refresh operation. /// The for the operation /// A resulting in the response on success - async Task RunRequest(string route, object? body, HttpMethod method, long? instanceId, bool tokenRefresh, CancellationToken cancellationToken) + Task RunRequest( + string route, + object? body, + HttpMethod method, + long? instanceId, + bool tokenRefresh, + CancellationToken cancellationToken) + { + HttpContent? content = null; + if(body != null) + content = new StringContent( + JsonConvert.SerializeObject(body, GetSerializerSettings()), + Encoding.UTF8, + MediaTypeNames.Application.Json); + + return RunRequest( + route, + content, + method, + instanceId, + tokenRefresh, + cancellationToken); + } + + /// + /// Main request method + /// + /// The resulting POCO type + /// The route to run + /// The of the request if any. + /// The method of the request + /// The optional instance for the request + /// If this is a token refresh operation. + /// The for the operation + /// A resulting in the response on success + async Task RunRequest( + string route, + HttpContent? content, + HttpMethod method, + long? instanceId, + bool tokenRefresh, + CancellationToken cancellationToken) { if (route == null) throw new ArgumentNullException(nameof(route)); if (method == null) throw new ArgumentNullException(nameof(method)); - if (body == null && (method == HttpMethod.Post || method == HttpMethod.Put)) - throw new InvalidOperationException("Body cannot be null for POST or PUT!"); + if (content == null && (method == HttpMethod.Post || method == HttpMethod.Put)) + throw new InvalidOperationException("content cannot be null for POST or PUT!"); HttpResponseMessage response; var fullUri = new Uri(Url, route); var serializerSettings = GetSerializerSettings(); + var fileDownload = typeof(TResult) == typeof(Stream); using (var request = new HttpRequestMessage(method, fullUri)) { - if (body != null) - request.Content = new StringContent( - JsonConvert.SerializeObject(body, serializerSettings), - Encoding.UTF8, - MediaTypeNames.Application.Json); + if (content != null) + request.Content = content; var headersToUse = tokenRefresh ? tokenRefreshHeaders! : headers; headersToUse.SetRequestHeaders(request.Headers, instanceId); + if (fileDownload) + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Octet)); + await Task.WhenAll(requestLoggers.Select(x => x.LogRequest(request, cancellationToken))).ConfigureAwait(false); response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); } - using (response) + try { await Task.WhenAll(requestLoggers.Select(x => x.LogResponse(response, cancellationToken))).ConfigureAwait(false); + // just stream + if (fileDownload && response.IsSuccessStatusCode) + return (TResult)(object)await CachedResponseStream.Create(response).ConfigureAwait(false); + } + catch + { + response.Dispose(); + throw; + } + + using (response) + { var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false); if (!response.IsSuccessStatusCode) @@ -182,7 +239,7 @@ namespace Tgstation.Server.Client if (!tokenRefresh && response.StatusCode == HttpStatusCode.Unauthorized && await RefreshToken(cancellationToken).ConfigureAwait(false)) - return await RunRequest(route, body, method, instanceId, false, cancellationToken).ConfigureAwait(false); + return await RunRequest(route, content, method, instanceId, false, cancellationToken).ConfigureAwait(false); HandleBadResponse(response, json); } @@ -277,5 +334,41 @@ namespace Tgstation.Server.Client /// public void AddRequestLogger(IRequestLogger requestLogger) => requestLoggers.Add(requestLogger ?? throw new ArgumentNullException(nameof(requestLogger))); + + /// + public Task Download(FileTicketResult ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + return RunRequest( + $"{Routes.Transfer}?ticket={HttpUtility.UrlEncode(ticket.FileTicket)}", + null, + HttpMethod.Get, + null, + false, + cancellationToken); + } + + /// + public async Task Upload(FileTicketResult ticket, Stream? uploadStream, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + MemoryStream? memoryStream = null; + if (uploadStream == null) + memoryStream = new MemoryStream(); + + using (memoryStream) + await RunRequest( + $"{Routes.Transfer}?ticket={HttpUtility.UrlEncode(ticket.FileTicket)}", + new StreamContent(uploadStream ?? memoryStream), + HttpMethod.Put, + null, + false, + cancellationToken) + .ConfigureAwait(false); + } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Client/ApiClientFactory.cs b/src/Tgstation.Server.Client/ApiClientFactory.cs index d3f137baeb..02ff485b3a 100644 --- a/src/Tgstation.Server.Client/ApiClientFactory.cs +++ b/src/Tgstation.Server.Client/ApiClientFactory.cs @@ -7,6 +7,6 @@ namespace Tgstation.Server.Client sealed class ApiClientFactory : IApiClientFactory { /// - public IApiClient CreateApiClient(Uri url, ApiHeaders apiHeaders, ApiHeaders? tokenRefreshHeaders) => new ApiClient(new HttpClient(), url, apiHeaders, tokenRefreshHeaders); + public IApiClient CreateApiClient(Uri url, ApiHeaders apiHeaders, ApiHeaders? tokenRefreshHeaders) => new ApiClient(new HttpClientImplementation(), url, apiHeaders, tokenRefreshHeaders); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Client/CachedResponseStream.cs b/src/Tgstation.Server.Client/CachedResponseStream.cs new file mode 100644 index 0000000000..9ad446f75c --- /dev/null +++ b/src/Tgstation.Server.Client/CachedResponseStream.cs @@ -0,0 +1,96 @@ +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Tgstation.Server.Client +{ + /// + /// Caches the from a for later use. + /// + sealed class CachedResponseStream : Stream + { + /// + /// The for the . + /// + readonly HttpResponseMessage response; + + /// + /// The reponse content . + /// + readonly Stream responseStream; + + /// + /// Asyncronously creates a new . + /// + /// The to build from. + /// A resulting in a new . + public static async Task Create(HttpResponseMessage response) + { + var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + return new CachedResponseStream(response, stream); + } + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of . + CachedResponseStream(HttpResponseMessage response, Stream responseStream) + { + this.response = response; + this.responseStream = responseStream; + } + + /// + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + if (!disposing) + return; + responseStream.Dispose(); + response.Dispose(); + } + + /// + public override async ValueTask DisposeAsync() + { + await base.DisposeAsync().ConfigureAwait(false); + await responseStream.DisposeAsync().ConfigureAwait(false); + response.Dispose(); + } + + /// + public override bool CanRead => responseStream.CanRead; + + /// + public override bool CanSeek => responseStream.CanSeek; + + /// + public override bool CanWrite => responseStream.CanWrite; + + /// + public override long Length => responseStream.Length; + + /// + public override long Position + { + get => responseStream.Position; + set => responseStream.Position = value; + } + + /// + public override void Flush() => responseStream.Flush(); + + /// + public override int Read(byte[] buffer, int offset, int count) => responseStream.Read(buffer, offset, count); + + /// + public override long Seek(long offset, SeekOrigin origin) => responseStream.Seek(offset, origin); + + /// + public override void SetLength(long value) => responseStream.SetLength(value); + + /// + public override void Write(byte[] buffer, int offset, int count) => responseStream.Write(buffer, offset, count); + } +} diff --git a/src/Tgstation.Server.Client/Components/ByondClient.cs b/src/Tgstation.Server.Client/Components/ByondClient.cs index 23d56dc64a..15eb50bb7d 100644 --- a/src/Tgstation.Server.Client/Components/ByondClient.cs +++ b/src/Tgstation.Server.Client/Components/ByondClient.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; @@ -38,6 +39,19 @@ namespace Tgstation.Server.Client.Components public Task> InstalledVersions(CancellationToken cancellationToken) => apiClient.Read>(Routes.ListRoute(Routes.Byond), instance.Id, cancellationToken); /// - public Task SetActiveVersion(Byond byond, CancellationToken cancellationToken) => apiClient.Update(Routes.Byond, byond ?? throw new ArgumentNullException(nameof(byond)), instance.Id, cancellationToken); + public async Task SetActiveVersion(Byond byond, Stream zipFileStream, CancellationToken cancellationToken) + { + var result = await apiClient.Update( + Routes.Byond, + byond ?? throw new ArgumentNullException(nameof(byond)), + instance.Id, + cancellationToken) + .ConfigureAwait(false); + + if (byond.UploadCustomZip == true) + await apiClient.Upload(result, zipFileStream, cancellationToken).ConfigureAwait(false); + + return result; + } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Client/Components/ConfigurationClient.cs b/src/Tgstation.Server.Client/Components/ConfigurationClient.cs index 3134fbcf37..25f8a0060d 100644 --- a/src/Tgstation.Server.Client/Components/ConfigurationClient.cs +++ b/src/Tgstation.Server.Client/Components/ConfigurationClient.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; @@ -41,17 +42,56 @@ namespace Tgstation.Server.Client.Components public Task> List(string directory, CancellationToken cancellationToken) => apiClient.Read>(Routes.ListRoute(Routes.Configuration) + Routes.SanitizeGetPath(directory), instance.Id, cancellationToken); /// - public Task Read(ConfigurationFile file, CancellationToken cancellationToken) + public async Task> Read(ConfigurationFile file, CancellationToken cancellationToken) { if (file == null) throw new ArgumentNullException(nameof(file)); - return apiClient.Read( + var configFile = await apiClient.Read( Routes.ConfigurationFile + Routes.SanitizeGetPath(file.Path ?? throw new ArgumentException("file.Path should not be null!", nameof(file))), instance.Id, - cancellationToken); + cancellationToken) + .ConfigureAwait(false); + var downloadStream = await apiClient.Download(configFile, cancellationToken).ConfigureAwait(false); + try + { + return Tuple.Create(configFile, downloadStream); + } + catch + { + downloadStream.Dispose(); + throw; + } } /// - public Task Write(ConfigurationFile file, CancellationToken cancellationToken) => apiClient.Update(Routes.Configuration, file ?? throw new ArgumentNullException(nameof(file)), instance.Id, cancellationToken); + public async Task Write(ConfigurationFile file, Stream uploadStream, CancellationToken cancellationToken) + { + long initialStreamPosition = 0; + MemoryStream? memoryStream = null; + if (uploadStream?.CanSeek == false) + memoryStream = new MemoryStream(); + else if (uploadStream != null) + initialStreamPosition = uploadStream.Position; + + using (memoryStream) + { + var configFileTask = apiClient.Update( + Routes.Configuration, + file ?? throw new ArgumentNullException(nameof(file)), + instance.Id, + cancellationToken); + + if (memoryStream != null) + await uploadStream!.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); + + var configFile = await configFileTask.ConfigureAwait(false); + + var streamUsed = memoryStream ?? uploadStream; + streamUsed?.Seek(initialStreamPosition, SeekOrigin.Begin); + await apiClient.Upload(configFile, streamUsed, cancellationToken).ConfigureAwait(false); + + return configFile; + } + } } } diff --git a/src/Tgstation.Server.Client/Components/IByondClient.cs b/src/Tgstation.Server.Client/Components/IByondClient.cs index da4f4d45a3..44c90f64c4 100644 --- a/src/Tgstation.Server.Client/Components/IByondClient.cs +++ b/src/Tgstation.Server.Client/Components/IByondClient.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -28,8 +29,9 @@ namespace Tgstation.Server.Client.Components /// Updates the information /// /// The information to update + /// The for the .zip file if is . /// The for the operation /// A resulting in the updated information - Task SetActiveVersion(Byond byond, CancellationToken cancellationToken); + Task SetActiveVersion(Byond byond, Stream zipFileStream, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Components/IConfigurationClient.cs b/src/Tgstation.Server.Client/Components/IConfigurationClient.cs index d61e426af3..e82220c028 100644 --- a/src/Tgstation.Server.Client/Components/IConfigurationClient.cs +++ b/src/Tgstation.Server.Client/Components/IConfigurationClient.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -23,16 +25,17 @@ namespace Tgstation.Server.Client.Components /// /// The file to read /// The for the operation - /// A representing the running operation - Task Read(ConfigurationFile file, CancellationToken cancellationToken); + /// A resulting in a containing the and downloaded . + Task> Read(ConfigurationFile file, CancellationToken cancellationToken); /// /// Overwrite a file /// /// The file to write + /// The of uploaded data. If , a delete will be attempted. /// The for the operation - /// A resulting in the new - Task Write(ConfigurationFile file, CancellationToken cancellationToken); + /// A resulting in the new . + Task Write(ConfigurationFile file, Stream uploadStream, CancellationToken cancellationToken); /// /// Delete an empty diff --git a/src/Tgstation.Server.Client/HttpClient.cs b/src/Tgstation.Server.Client/HttpClientImplementation.cs similarity index 68% rename from src/Tgstation.Server.Client/HttpClient.cs rename to src/Tgstation.Server.Client/HttpClientImplementation.cs index c01042ae2d..5a449b1441 100644 --- a/src/Tgstation.Server.Client/HttpClient.cs +++ b/src/Tgstation.Server.Client/HttpClientImplementation.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -6,7 +6,7 @@ using System.Threading.Tasks; namespace Tgstation.Server.Client { /// - sealed class HttpClient : IHttpClient + sealed class HttpClientImplementation : IHttpClient { /// public TimeSpan Timeout @@ -16,16 +16,16 @@ namespace Tgstation.Server.Client } /// - /// The real + /// The real /// - readonly System.Net.Http.HttpClient httpClient; + readonly HttpClient httpClient; /// - /// Construct an + /// Construct an /// - public HttpClient() + public HttpClientImplementation() { - httpClient = new System.Net.Http.HttpClient(); + httpClient = new HttpClient(); } /// diff --git a/src/Tgstation.Server.Client/IAdministrationClient.cs b/src/Tgstation.Server.Client/IAdministrationClient.cs index bbffab6f0f..b77e7475f7 100644 --- a/src/Tgstation.Server.Client/IAdministrationClient.cs +++ b/src/Tgstation.Server.Client/IAdministrationClient.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; @@ -44,7 +46,7 @@ namespace Tgstation.Server.Client /// /// The to download. /// The for the operation - /// A resulting in the downloaded . - Task GetLog(LogFile logFile, CancellationToken cancellationToken); + /// A resulting a containing the downloaded and associated . + Task> GetLog(LogFile logFile, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/IApiClient.cs b/src/Tgstation.Server.Client/IApiClient.cs index bee5ea5328..94cc657f0a 100644 --- a/src/Tgstation.Server.Client/IApiClient.cs +++ b/src/Tgstation.Server.Client/IApiClient.cs @@ -1,7 +1,9 @@ -using System; +using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api; +using Tgstation.Server.Api.Models; namespace Tgstation.Server.Client { @@ -113,7 +115,7 @@ namespace Tgstation.Server.Client /// The type of the response body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Create(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -123,7 +125,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Create(string route, long instanceId, CancellationToken cancellationToken); @@ -133,7 +135,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Patch(string route, long instanceId, CancellationToken cancellationToken); @@ -143,7 +145,7 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Read(string route, long instanceId, CancellationToken cancellationToken); @@ -155,7 +157,7 @@ namespace Tgstation.Server.Client /// The type of the response body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Update(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -164,7 +166,7 @@ namespace Tgstation.Server.Client /// Run an HTTP DELETE request /// /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A representing the running operation Task Delete(string route, long instanceId, CancellationToken cancellationToken); @@ -175,7 +177,7 @@ namespace Tgstation.Server.Client /// The type to of the request body /// The server route to make the request to /// The request body - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A representing the running operation Task Delete(string route, TBody body, long instanceId, CancellationToken cancellationToken); @@ -185,9 +187,26 @@ namespace Tgstation.Server.Client /// /// The type of the response body /// The server route to make the request to - /// The instance to make the request to + /// The instance to make the request to /// The for the operation /// A resulting in the response body as a Task Delete(string route, long instanceId, CancellationToken cancellationToken); + + /// + /// Downloads a file for a given . + /// + /// The to download. + /// The for the operation. + /// A resulting in the downloaded . + Task Download(FileTicketResult ticket, CancellationToken cancellationToken); + + /// + /// Uploads a given for a given . + /// + /// The to download. + /// The to upload. represents an empty file. + /// The for the operation. + /// A representing the running operation. + Task Upload(FileTicketResult ticket, Stream? uploadStream, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Properties/AssemblyInfo.cs b/src/Tgstation.Server.Client/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..7f876c9048 --- /dev/null +++ b/src/Tgstation.Server.Client/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Tgstation.Server.Tests")] diff --git a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs index 72ea830c82..5d97995cb6 100644 --- a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Text; @@ -121,10 +122,10 @@ namespace Tgstation.Server.Host.Components.Byond /// Installs a BYOND if it isn't already /// /// The BYOND to install - /// Custom zip file bytes to use. Will cause a number to be added. + /// Custom zip file to use. Will cause a number to be added. /// The for the operation /// A representing the running operation - async Task InstallVersion(Version version, byte[] versionZipBytes, CancellationToken cancellationToken) + async Task InstallVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken) { var ourTcs = new TaskCompletionSource(); Task inProgressTask; @@ -132,7 +133,7 @@ namespace Tgstation.Server.Host.Components.Byond bool installed; lock (installedVersions) { - if (versionZipBytes != null) + if (customVersionStream != null) { int customInstallationNumber = 1; do @@ -157,7 +158,7 @@ namespace Tgstation.Server.Host.Components.Byond return versionKey; } - if (versionZipBytes != null) + if (customVersionStream != null) logger.LogInformation("Installing custom BYOND version as {0}...", versionKey); else if (version.Build > 0) throw new JobException(ErrorCode.ByondNonExistentCustomVersion); @@ -168,26 +169,43 @@ namespace Tgstation.Server.Host.Components.Byond try { await eventConsumer.HandleEvent(EventType.ByondInstallStart, new List { versionKey }, cancellationToken).ConfigureAwait(false); - var zipFileBytesTask = versionZipBytes == null - ? byondInstaller.DownloadVersion(version, cancellationToken) - : Task.FromResult(versionZipBytes); - await ioManager.DeleteDirectory(versionKey, cancellationToken).ConfigureAwait(false); + var extractPath = ioManager.ResolvePath(versionKey); + async Task DirectoryCleanup() + { + await ioManager.DeleteDirectory(extractPath, cancellationToken).ConfigureAwait(false); + await ioManager.CreateDirectory(extractPath, cancellationToken).ConfigureAwait(false); + } + var directoryCleanupTask = DirectoryCleanup(); try { - versionZipBytes = await zipFileBytesTask.ConfigureAwait(false); - await ioManager.CreateDirectory(versionKey, cancellationToken).ConfigureAwait(false); + Stream versionZipStream; + Stream downloadedStream = null; + if (customVersionStream == null) + { + var bytes = await byondInstaller.DownloadVersion(version, cancellationToken).ConfigureAwait(false); + downloadedStream = new MemoryStream(bytes); + versionZipStream = downloadedStream; + } + else + versionZipStream = customVersionStream; - var extractPath = ioManager.ResolvePath(versionKey); - logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); - await ioManager.ZipToDirectory(extractPath, versionZipBytes, cancellationToken).ConfigureAwait(false); - versionZipBytes = null; + using (downloadedStream) + { + await directoryCleanupTask.ConfigureAwait(false); + logger.LogTrace("Extracting downloaded BYOND zip to {0}...", extractPath); + await ioManager.ZipToDirectory(extractPath, versionZipStream, cancellationToken).ConfigureAwait(false); + } await byondInstaller.InstallByond(extractPath, version, cancellationToken).ConfigureAwait(false); // make sure to do this last because this is what tells us we have a valid version in the future - await ioManager.WriteAllBytes(ioManager.ConcatPath(versionKey, VersionFileName), Encoding.UTF8.GetBytes(versionKey), cancellationToken).ConfigureAwait(false); + await ioManager.WriteAllBytes( + ioManager.ConcatPath(versionKey, VersionFileName), + Encoding.UTF8.GetBytes(versionKey), + cancellationToken) + .ConfigureAwait(false); } catch (WebException e) { @@ -220,12 +238,12 @@ namespace Tgstation.Server.Host.Components.Byond } /// - public async Task ChangeVersion(Version version, byte[] customVersionBytes, CancellationToken cancellationToken) + public async Task ChangeVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken) { if (version == null) throw new ArgumentNullException(nameof(version)); - var versionKey = await InstallVersion(version, customVersionBytes, cancellationToken).ConfigureAwait(false); + var versionKey = await InstallVersion(version, customVersionStream, cancellationToken).ConfigureAwait(false); using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { await ioManager.WriteAllBytes(ActiveVersionFileName, Encoding.UTF8.GetBytes(versionKey), cancellationToken).ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs index 650bce0209..129fa54037 100644 --- a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs @@ -1,6 +1,7 @@ -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -25,10 +26,10 @@ namespace Tgstation.Server.Host.Components.Byond /// Change the active BYOND version /// /// The new - /// Optional s of a custom BYOND version zip file. + /// Optional of a custom BYOND version zip file. /// The for the operation /// A representing the running operation - Task ChangeVersion(Version version, byte[] customVersionBytes, CancellationToken cancellationToken); + Task ChangeVersion(Version version, Stream customVersionStream, CancellationToken cancellationToken); /// /// Lock the current installation's location and return a @@ -38,4 +39,4 @@ namespace Tgstation.Server.Host.Components.Byond /// A resulting in the requested Task UseExecutables(Version requiredVersion, CancellationToken cancellationToken); } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 548410eeef..1045c8ed8b 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -19,6 +19,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Components { @@ -125,6 +126,11 @@ namespace Tgstation.Server.Host.Components /// readonly IServerPortProvider serverPortProvider; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for the . /// @@ -153,6 +159,7 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . + /// The value of . /// The containing the value of . public InstanceFactory( IIOManager ioManager, @@ -175,6 +182,7 @@ namespace Tgstation.Server.Host.Components ILibGit2RepositoryFactory repositoryFactory, ILibGit2Commands repositoryCommands, IServerPortProvider serverPortProvider, + IFileTransferTicketProvider fileTransferService, IOptions generalConfigurationOptions) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -197,6 +205,7 @@ namespace Tgstation.Server.Host.Components this.repositoryFactory = repositoryFactory ?? throw new ArgumentNullException(nameof(repositoryFactory)); this.repositoryCommands = repositoryCommands ?? throw new ArgumentNullException(nameof(repositoryCommands)); this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } @@ -214,7 +223,15 @@ namespace Tgstation.Server.Host.Components var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); - var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, platformIdentifier, loggerFactory.CreateLogger()); + var configuration = new StaticFiles.Configuration( + configurationIoManager, + synchronousIOManager, + symlinkFactory, + processExecutor, + postWriteHandler, + platformIdentifier, + fileTransferService, + loggerFactory.CreateLogger()); var eventConsumer = new EventConsumer(configuration); var repoManager = new RepositoryManager( repositoryFactory, diff --git a/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs b/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs index 65c1f21049..58158078ab 100644 --- a/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs +++ b/src/Tgstation.Server.Host/Components/Repository/LibGit2Commands.cs @@ -1,4 +1,4 @@ -using LibGit2Sharp; +using LibGit2Sharp; using System; using System.Collections.Generic; @@ -22,13 +22,10 @@ namespace Tgstation.Server.Host.Components.Repository if (libGit2Repo == null) throw new ArgumentNullException(nameof(libGit2Repo)); - if (!(libGit2Repo is LibGit2Sharp.Repository concreteRepo)) - throw new ArgumentException("libGit2Repo must be an instance of LibGit2Sharp.Repository!", nameof(libGit2Repo)); - if (remote == null) throw new ArgumentNullException(nameof(remote)); - Commands.Fetch(concreteRepo, remote.Name, refSpecs, fetchOptions, logMessage); + Commands.Fetch((LibGit2Sharp.Repository)libGit2Repo, remote.Name, refSpecs, fetchOptions, logMessage); } } } diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 5805339a58..427636e6b3 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Components.StaticFiles { @@ -76,6 +77,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// readonly IPlatformIdentifier platformIdentifier; + /// + /// The for . + /// > + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for /// @@ -86,6 +92,16 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// readonly SemaphoreSlim semaphore; + /// + /// The that is triggered when is called. + /// + readonly CancellationTokenSource disposeCts; + + /// + /// The culmination of all upload file transfer callbacks. + /// + Task uploadTasks; + /// /// Construct /// @@ -95,6 +111,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// The value of /// The value of /// The value of + /// The value of . /// The value of public Configuration( IIOManager ioManager, @@ -103,6 +120,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IPlatformIdentifier platformIdentifier, + IFileTransferTicketProvider fileTransferService, ILogger logger) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -111,13 +129,21 @@ namespace Tgstation.Server.Host.Components.StaticFiles this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); semaphore = new SemaphoreSlim(1); + disposeCts = new CancellationTokenSource(); + uploadTasks = Task.CompletedTask; } /// - public void Dispose() => semaphore.Dispose(); + public void Dispose() + { + semaphore.Dispose(); + disposeCts.Cancel(); + disposeCts.Dispose(); + } /// /// Get the proper path to @@ -246,17 +272,55 @@ namespace Tgstation.Server.Host.Components.StaticFiles lock (semaphore) try { - var content = synchronousIOManager.ReadFile(path); - string sha1String; + string GetFileSha() + { + var content = synchronousIOManager.ReadFile(path); #pragma warning disable CA5350 // Do not use insecure cryptographic algorithm SHA1. - using (var sha1 = new SHA1Managed()) + using var sha1 = new SHA1Managed(); #pragma warning restore CA5350 // Do not use insecure cryptographic algorithm SHA1. - sha1String = String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + return String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + } + + var originalSha = GetFileSha(); + + var disposeToken = disposeCts.Token; + var fileTicket = fileTransferService.CreateDownload( + new FileDownloadProvider( + () => + { + if (disposeToken.IsCancellationRequested) + return ErrorCode.InstanceOffline; + + var newSha = GetFileSha(); + if (newSha != originalSha) + return ErrorCode.ConfigurationFileUpdated; + + return null; + }, + async cancellationToken => + { + FileStream result = null; + void GetFileStream() + { + result = ioManager.GetFileStream(path, false); + } + + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + if (systemIdentity == null) + await Task.Factory.StartNew(GetFileStream, cancellationToken, DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current).ConfigureAwait(false); + else + await systemIdentity.RunImpersonated(GetFileStream, cancellationToken).ConfigureAwait(false); + + return result; + }, + path, + false)); + result = new ConfigurationFile { - Content = content, + FileTicket = fileTicket.FileTicket, IsDirectory = false, - LastReadHash = sha1String, + LastReadHash = originalSha, AccessDenied = false, Path = configurationRelativePath }; @@ -365,7 +429,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles } /// - public async Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, byte[] data, string previousHash, CancellationToken cancellationToken) + public async Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, string previousHash, CancellationToken cancellationToken) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); var path = ValidateConfigRelativePath(configurationRelativePath); @@ -377,20 +441,47 @@ namespace Tgstation.Server.Host.Components.StaticFiles lock (semaphore) try { - var fileHash = previousHash; - var success = synchronousIOManager.WriteFileChecked(path, data, ref fileHash, cancellationToken); - if (!success) - return; - if (data != null) - postWriteHandler.HandleWrite(path); + var fileTicket = fileTransferService.CreateUpload(true); + var uploadCancellationToken = disposeCts.Token; + async Task UploadHandler() + { + using (fileTicket) + { + var fileHash = previousHash; + using var uploadStream = await fileTicket.GetResult(uploadCancellationToken).ConfigureAwait(false); + bool success = false; + void WriteCallback() + { + success = synchronousIOManager.WriteFileChecked(path, uploadStream, ref fileHash, cancellationToken); + } + + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + if (systemIdentity == null) + await Task.Factory.StartNew(WriteCallback, cancellationToken, DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current).ConfigureAwait(false); + else + await systemIdentity.RunImpersonated(WriteCallback, cancellationToken).ConfigureAwait(false); + + if (!success) + fileTicket.SetErrorMessage(new ErrorMessage(ErrorCode.ConfigurationFileUpdated) + { + AdditionalData = fileHash + }); + else if(uploadStream.Length > 0) + postWriteHandler.HandleWrite(path); + } + } + result = new ConfigurationFile { - Content = data, + FileTicket = fileTicket.Ticket.FileTicket, + LastReadHash = previousHash, IsDirectory = false, - LastReadHash = fileHash, AccessDenied = false, Path = configurationRelativePath }; + + lock (disposeCts) + uploadTasks = Task.WhenAll(uploadTasks, UploadHandler()); } catch (UnauthorizedAccessException) { diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs index b57497a4d1..34a42d299e 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting; using System; using System.Collections.Generic; using System.Threading; @@ -37,7 +37,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// The relative path in the Configuration directory /// The for the operation. If , the operation will be performed as the user of the /// The for the operation - /// A resulting in the s for the items in the directory. and will both be + /// A resulting in the s for the items in the directory. and will both be Task> ListDirectory(string configurationRelativePath, ISystemIdentity systemIdentity, CancellationToken cancellationToken); /// @@ -72,10 +72,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// The relative path in the Configuration directory /// The for the operation. If , the operation will be performed as the user of the - /// The data to write. If , the file is deleted /// The hash any existing file must match in order for the write to succeed /// The for the operation. Usage may result in partial writes /// A resulting in the updated or if the write failed due to conflicts - Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, byte[] data, string previousHash, CancellationToken cancellationToken); + Task Write(string configurationRelativePath, ISystemIdentity systemIdentity, string previousHash, CancellationToken cancellationToken); } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index a41fb7895c..e920efacb3 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -20,6 +20,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Controllers { @@ -56,6 +57,11 @@ namespace Tgstation.Server.Host.Controllers /// readonly IPlatformIdentifier platformIdentifier; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// The for the /// @@ -81,6 +87,7 @@ namespace Tgstation.Server.Host.Controllers /// The value of /// The value of /// The value of + /// The value of . /// The for the /// The containing value of /// The containing value of @@ -93,6 +100,7 @@ namespace Tgstation.Server.Host.Controllers IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, IPlatformIdentifier platformIdentifier, + IFileTransferTicketProvider fileTransferService, ILogger logger, IOptions updatesConfigurationOptions, IOptions generalConfigurationOptions, @@ -108,6 +116,7 @@ namespace Tgstation.Server.Host.Controllers this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); updatesConfiguration = updatesConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(updatesConfigurationOptions)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); @@ -386,13 +395,20 @@ namespace Tgstation.Server.Host.Controllers path); try { + var fileTransferTicket = fileTransferService.CreateDownload( + new FileDownloadProvider( + () => null, + null, + fullPath, + true)); + var readTask = ioManager.ReadAllBytes(fullPath, cancellationToken); return Ok(new LogFile { Name = path, LastModified = await ioManager.GetLastModified(fullPath, cancellationToken).ConfigureAwait(false), - Content = await readTask.ConfigureAwait(false) + FileTicket = fileTransferTicket.FileTicket }); } catch (IOException ex) diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index cab2dd1fb2..8c503ba47b 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -1,7 +1,8 @@ -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -13,6 +14,7 @@ using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Controllers { @@ -27,6 +29,11 @@ namespace Tgstation.Server.Host.Controllers /// readonly IJobManager jobManager; + /// + /// The for the . + /// + readonly IFileTransferTicketProvider fileTransferService; + /// /// Construct a /// @@ -34,12 +41,14 @@ namespace Tgstation.Server.Host.Controllers /// The for the /// The for the . /// The value of + /// The value of . /// The for the public ByondController( IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, IInstanceManager instanceManager, IJobManager jobManager, + IFileTransferTicketProvider fileTransferService, ILogger logger) : base( instanceManager, @@ -48,6 +57,7 @@ namespace Tgstation.Server.Host.Controllers logger) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); } /// @@ -104,14 +114,16 @@ namespace Tgstation.Server.Host.Controllers if (model == null) throw new ArgumentNullException(nameof(model)); + var uploadingZip = model.UploadCustomZip == true; + if (model.Version == null || model.Version.Revision != -1 - || (model.Content != null && model.Version.Build > 0)) + || (uploadingZip && model.Version.Build > 0)) return BadRequest(new ErrorMessage(ErrorCode.ModelValidationFailure)); var userByondRights = AuthenticationContext.InstanceUser.ByondRights.Value; - if ((!userByondRights.HasFlag(ByondRights.InstallOfficialOrChangeActiveVersion) && model.Content == null) - || (!userByondRights.HasFlag(ByondRights.InstallCustomVersion) && model.Content != null)) + if ((!userByondRights.HasFlag(ByondRights.InstallOfficialOrChangeActiveVersion) && !uploadingZip) + || (!userByondRights.HasFlag(ByondRights.InstallCustomVersion) && uploadingZip)) return Forbid(); // remove cruff fields @@ -120,7 +132,7 @@ namespace Tgstation.Server.Host.Controllers async instance => { var byondManager = instance.ByondManager; - if (model.Content == null && byondManager.InstalledVersions.Any(x => x == model.Version)) + if (!uploadingZip && byondManager.InstalledVersions.Any(x => x == model.Version)) { Logger.LogInformation( "User ID {0} changing instance ID {1} BYOND version to {2}", @@ -146,21 +158,61 @@ namespace Tgstation.Server.Host.Controllers // run the install through the job manager var job = new Models.Job { - Description = $"Install {(model.Content == null ? String.Empty : "custom ")}BYOND version {model.Version.Major}.{model.Version.Minor}", + Description = $"Install {(!uploadingZip ? String.Empty : "custom ")}BYOND version {model.Version.Major}.{model.Version.Minor}", StartedBy = AuthenticationContext.User, CancelRightsType = RightsType.Byond, CancelRight = (ulong)ByondRights.CancelInstall, Instance = Instance }; - await jobManager.RegisterOperation( - job, - (core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => core.ByondManager.ChangeVersion( - model.Version, - model.Content, - jobCancellationToken), - cancellationToken) - .ConfigureAwait(false); - result.InstallJob = job.ToApi(); + + IFileUploadTicket fileUploadTicket = null; + if (uploadingZip) + fileUploadTicket = fileTransferService.CreateUpload(false); + + try + { + await jobManager.RegisterOperation( + job, + async (core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => + { + Stream zipFileStream = null; + if (fileUploadTicket != null) + using (fileUploadTicket) + { + var uploadStream = await fileUploadTicket.GetResult(jobCancellationToken).ConfigureAwait(false); + if (uploadStream == null) + throw new JobException(ErrorCode.FileUploadExpired); + + zipFileStream = new MemoryStream(); + try + { + await uploadStream.CopyToAsync(zipFileStream, jobCancellationToken).ConfigureAwait(false); + } + catch + { + await zipFileStream.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + using (zipFileStream) + await core.ByondManager.ChangeVersion( + model.Version, + zipFileStream, + jobCancellationToken) + .ConfigureAwait(false); + }, + cancellationToken) + .ConfigureAwait(false); + + result.InstallJob = job.ToApi(); + result.FileTicket = fileUploadTicket?.Ticket.FileTicket; + } + catch + { + fileUploadTicket?.Dispose(); + throw; + } } if ((AuthenticationContext.GetRight(RightsType.Byond) & (ulong)ByondRights.ReadActive) != 0) diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index 3a76b8bc5b..3d82b23fe8 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -77,11 +77,11 @@ namespace Tgstation.Server.Host.Controllers /// The for the operation. /// A resulting in the for the operation. /// File updated successfully. - /// File created successfully. + /// File upload ticket created successfully. [HttpPost] [TgsAuthorize(ConfigurationRights.Write)] [ProducesResponseType(typeof(ConfigurationFile), 200)] - [ProducesResponseType(typeof(ConfigurationFile), 201)] + [ProducesResponseType(typeof(ConfigurationFile), 202)] public async Task Update([FromBody] ConfigurationFile model, CancellationToken cancellationToken) { if (model == null) @@ -99,16 +99,11 @@ namespace Tgstation.Server.Host.Controllers .Write( model.Path, systemIdentity, - model.Content, model.LastReadHash, cancellationToken) .ConfigureAwait(false); - if (newFile == null) - return Conflict(new ErrorMessage(ErrorCode.ConfigurationFileUpdated)); - newFile.Content = null; - - return model.LastReadHash == null ? (IActionResult)Created(newFile) : Json(newFile); + return model.LastReadHash == null ? (IActionResult)Accepted(newFile) : Json(newFile); }) .ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResult.cs b/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResult.cs new file mode 100644 index 0000000000..eefafb2aa6 --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResult.cs @@ -0,0 +1,44 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.IO; +using System.Net.Mime; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// Very similar to except it's contains a fix for https://github.com/dotnet/aspnetcore/issues/28189. + /// + public sealed class LimitedFileStreamResult : FileResult + { + /// + /// The representing the file to download. + /// + public FileStream FileStream { get; } + + /// + /// Initializes a new instance of the . + /// + /// The value of . + public LimitedFileStreamResult(FileStream stream) + : base(MediaTypeNames.Application.Octet) + { + FileStream = stream ?? throw new ArgumentNullException(nameof(stream)); + } + + /// + public override Task ExecuteResultAsync(ActionContext context) + { + if (context == null) + throw new ArgumentNullException(nameof(context)); + + var executor = context + .HttpContext + .RequestServices + .GetRequiredService>(); + return executor.ExecuteAsync(context, this); + } + } +} diff --git a/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResultExecutor.cs b/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResultExecutor.cs new file mode 100644 index 0000000000..19df00ae1e --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/LimitedFileStreamResultExecutor.cs @@ -0,0 +1,76 @@ +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.Extensions.Logging; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// for s. + /// + public class LimitedFileStreamResultExecutor : FileResultExecutorBase, IActionResultExecutor + { + /// + /// Initializes a new instance of the . + /// + /// The for the . + public LimitedFileStreamResultExecutor(ILogger logger) + : base(logger) + { + } + + /// + public async Task ExecuteAsync(ActionContext context, LimitedFileStreamResult result) + { + if (context == null) + throw new ArgumentNullException(nameof(context)); + + if (result == null) + throw new ArgumentNullException(nameof(result)); + + using (result.FileStream) + { + var contentLength = result.FileStream.Length; + var (range, rangeLength, serveBody) = SetHeadersAndLog(context, result, contentLength, result.EnableRangeProcessing); + if (!serveBody) + return; + + try + { + var cancellationToken = context.HttpContext.RequestAborted; + var outputStream = context.HttpContext.Response.Body; + if (range == null) + { + await StreamCopyOperation.CopyToAsync( + result.FileStream, + outputStream, + contentLength, + BufferSize, + cancellationToken) + .ConfigureAwait(false); + } + else + { + result.FileStream.Seek(range.From.Value, SeekOrigin.Begin); + await StreamCopyOperation.CopyToAsync( + result.FileStream, + outputStream, + rangeLength, + BufferSize, + cancellationToken) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Don't throw this exception, it's most likely caused by the client disconnecting. + // However, if it was cancelled for any other reason we need to prevent empty responses. + context.HttpContext.Abort(); + } + } + } + } +} diff --git a/src/Tgstation.Server.Host/Controllers/TransferController.cs b/src/Tgstation.Server.Host/Controllers/TransferController.cs new file mode 100644 index 0000000000..26521a7159 --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/TransferController.cs @@ -0,0 +1,133 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; +using System; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Net; +using System.Net.Mime; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Transfer; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// for file streaming. + /// + [Route(Routes.Transfer)] + [RequestSizeLimit(Limits.MaximumFileTransferSize)] + public sealed class TransferController : ApiController + { + /// + /// The for the . + /// + readonly IFileTransferStreamHandler fileTransferService; + + /// + /// Initializes a new instance of the . + /// + /// The for the + /// The for the + /// The value of . + /// The for the + public TransferController( + IDatabaseContext databaseContext, + IAuthenticationContextFactory authenticationContextFactory, + IFileTransferStreamHandler fileTransferService, + ILogger logger) + : base( + databaseContext, + authenticationContextFactory, + logger, + true) + { + this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); + } + + /// + /// Downloads a file with a given . + /// + /// The for the download. + /// The for the operation. + /// A resulting in the of the method. + /// Started streaming download successfully. + /// The was no longer or was never valid. + [TgsAuthorize] + [HttpGet] + [ProducesResponseType(200, Type = typeof(LimitedFileStreamResult))] + [ProducesResponseType(410, Type = typeof(ErrorMessage))] + public async Task Download([Required, FromQuery] string ticket, CancellationToken cancellationToken) + { + if (ticket == null) + return BadRequest(new ErrorMessage(ErrorCode.ModelValidationFailure)); + + var streamAccept = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet); + if (!Request.GetTypedHeaders().Accept.Any(x => streamAccept.IsSubsetOf(x))) + return StatusCode(HttpStatusCode.NotAcceptable, new ErrorMessage(ErrorCode.BadHeaders) + { + AdditionalData = $"File downloads must accept both {MediaTypeNames.Application.Octet} and {MediaTypeNames.Application.Json}!" + }); + + var fileTicketResult = new FileTicketResult + { + FileTicket = ticket + }; + + var tuple = await fileTransferService.RetrieveDownloadStream(fileTicketResult, cancellationToken).ConfigureAwait(false); + var stream = tuple.Item1; + try + { + if (tuple.Item2 != null) + return Conflict(tuple.Item2); + + if (stream == null) + return Gone(); + + return new LimitedFileStreamResult(stream); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// + /// Uploads a file with a given . + /// + /// The for the upload. + /// The for the operation. + /// A resulting in the of the method. + /// Uploaded file successfully. + /// An error occurred during the upload. + /// The was no longer or was never valid. + [TgsAuthorize] + [HttpPut] + [ProducesResponseType(204)] + [ProducesResponseType(410, Type = typeof(ErrorMessage))] + public async Task Upload([Required, FromQuery] string ticket, CancellationToken cancellationToken) + { + if (ticket == null) + return BadRequest(new ErrorMessage(ErrorCode.ModelValidationFailure)); + + var fileTicketResult = new FileTicketResult + { + FileTicket = ticket + }; + + var result = await fileTransferService.SetUploadStream(fileTicketResult, Request.Body, cancellationToken).ConfigureAwait(false); + if (result != null) + return result.ErrorCode == ErrorCode.ResourceNotPresent + ? Gone() + : Conflict(result); + + return NoContent(); + } + } +} diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 80d71e27c0..cf80be7bfd 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Identity; +using Microsoft.AspNetCore.Mvc.Infrastructure; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -31,6 +32,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Controllers; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; @@ -40,6 +42,7 @@ using Tgstation.Server.Host.Security; using Tgstation.Server.Host.Security.OAuth; using Tgstation.Server.Host.Setup; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Core { @@ -304,12 +307,16 @@ namespace Tgstation.Server.Host.Core } // configure misc services + services.AddScoped(); + services.AddTransient, LimitedFileStreamResultExecutor>(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService()); + services.AddSingleton(x => x.GetRequiredService()); // configure component services services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/Core/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Core/SwaggerConfiguration.cs index 2d5a07a408..67d2dccc61 100644 --- a/src/Tgstation.Server.Host/Core/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Core/SwaggerConfiguration.cs @@ -25,6 +25,11 @@ namespace Tgstation.Server.Host.Core /// const string PasswordSecuritySchemeId = "Password_Login_Scheme"; + /// + /// The name for OAuth 2.0 authentication. + /// + const string OAuthSecuritySchemeId = "OAuth_Login_Scheme"; + /// /// The name for token authentication. /// @@ -170,6 +175,14 @@ namespace Tgstation.Server.Host.Core Scheme = ApiHeaders.BasicAuthenticationScheme }); + swaggerGenOptions.AddSecurityDefinition(OAuthSecuritySchemeId, new OpenApiSecurityScheme + { + In = ParameterLocation.Header, + Type = SecuritySchemeType.Http, + Name = HeaderNames.Authorization, + Scheme = ApiHeaders.OAuthAuthenticationScheme + }); + swaggerGenOptions.AddSecurityDefinition(TokenSecuritySchemeId, new OpenApiSecurityScheme { BearerFormat = "JWT", @@ -236,10 +249,35 @@ namespace Tgstation.Server.Host.Core Id = ApiHeaders.InstanceIdHeader } }); + else if (typeof(TransferController).IsAssignableFrom(context.MethodInfo.DeclaringType)) + if (context.MethodInfo.Name == nameof(TransferController.Upload)) + operation.RequestBody = new OpenApiRequestBody + { + Content = new Dictionary + { + { + MediaTypeNames.Application.Octet, + new OpenApiMediaType + { + Schema = new OpenApiSchema + { + Type = "string", + Format = "binary" + } + } + } + } + }; + else if (context.MethodInfo.Name == nameof(TransferController.Download)) + { + var twoHundredResponseContents = operation.Responses["200"].Content; + var fileContent = twoHundredResponseContents[MediaTypeNames.Application.Json]; + twoHundredResponseContents.Remove(MediaTypeNames.Application.Json); + twoHundredResponseContents.Add(MediaTypeNames.Application.Octet, fileContent); + } } - else + else if (context.MethodInfo.Name == nameof(HomeController.CreateToken)) { - // HomeController.CreateToken var passwordScheme = new OpenApiSecurityScheme { Reference = new OpenApiReference @@ -249,6 +287,28 @@ namespace Tgstation.Server.Host.Core } }; + var oAuthScheme = new OpenApiSecurityScheme + { + Reference = new OpenApiReference + { + Type = ReferenceType.SecurityScheme, + Id = OAuthSecuritySchemeId + } + }; + + operation.Parameters.Add(new OpenApiParameter + { + In = ParameterLocation.Header, + Name = ApiHeaders.OAuthProviderHeader, + Description = "The external OAuth service provider.", + Style = ParameterStyle.Simple, + Example = new OpenApiString("Discord"), + Schema = new OpenApiSchema + { + Type = "string" + } + }); + operation.Security = new List { new OpenApiSecurityRequirement @@ -256,6 +316,10 @@ namespace Tgstation.Server.Host.Core { passwordScheme, new List() + }, + { + oAuthScheme, + new List() } } }; @@ -311,17 +375,24 @@ namespace Tgstation.Server.Host.Core Schema = productHeaderSchema }); - string bridgeOperationPath = null; + var pathsToRemove = new List(); + var filteredControllers = new string[] + { + nameof(BridgeController), + nameof(ControlPanelController), + }; + foreach (var path in swaggerDoc.Paths) foreach (var operation in path.Value.Operations.Select(x => x.Value)) { - if (operation.OperationId.Equals($"{nameof(BridgeController)}.{nameof(BridgeController.Process)}", StringComparison.Ordinal)) + if (filteredControllers.Any( + x => operation.OperationId.StartsWith(x, StringComparison.Ordinal))) { - bridgeOperationPath = path.Key; + pathsToRemove.Add(path.Key); continue; } - operation.Parameters.Add(new OpenApiParameter + operation.Parameters.Insert(0, new OpenApiParameter { Reference = new OpenApiReference { @@ -330,7 +401,7 @@ namespace Tgstation.Server.Host.Core }, }); - operation.Parameters.Add(new OpenApiParameter + operation.Parameters.Insert(1, new OpenApiParameter { Reference = new OpenApiReference { @@ -340,7 +411,8 @@ namespace Tgstation.Server.Host.Core }); } - swaggerDoc.Paths.Remove(bridgeOperationPath); + foreach (var filteredPath in pathsToRemove) + swaggerDoc.Paths.Remove(filteredPath); AddDefaultResponses(swaggerDoc); } diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index d63c32753c..b79fa1640a 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -122,7 +122,7 @@ namespace Tgstation.Server.Host.IO /// public async Task CopyDirectory(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) { - if (dest == null) + if (src == null) throw new ArgumentNullException(nameof(src)); if (dest == null) throw new ArgumentNullException(nameof(src)); @@ -134,12 +134,7 @@ namespace Tgstation.Server.Host.IO } /// - public string ConcatPath(params string[] paths) - { - if (paths == null) - throw new ArgumentNullException(nameof(paths)); - return Path.Combine(paths); - } + public string ConcatPath(params string[] paths) => Path.Combine(paths); /// public async Task CopyFile(string src, string dest, CancellationToken cancellationToken) @@ -150,6 +145,8 @@ namespace Tgstation.Server.Host.IO throw new ArgumentNullException(nameof(dest)); using var srcStream = new FileStream(ResolvePath(src), FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, DefaultBufferSize, true); using var destStream = new FileStream(ResolvePath(dest), FileMode.Create, FileAccess.Write, FileShare.ReadWrite | FileShare.Delete, DefaultBufferSize, true); + + // value taken from documentation await srcStream.CopyToAsync(destStream, 81920, cancellationToken).ConfigureAwait(false); } @@ -308,14 +305,13 @@ namespace Tgstation.Server.Host.IO } /// - public Task ZipToDirectory(string path, byte[] zipFileBytes, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + public Task ZipToDirectory(string path, Stream zipFile, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { path = ResolvePath(path); - if (zipFileBytes == null) - throw new ArgumentNullException(nameof(zipFileBytes)); + if (zipFile == null) + throw new ArgumentNullException(nameof(zipFile)); - using var ms = new MemoryStream(zipFileBytes); - using var archive = new ZipArchive(ms, ZipArchiveMode.Read); + using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read); archive.ExtractToDirectory(path); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); @@ -329,5 +325,8 @@ namespace Tgstation.Server.Host.IO var fileInfo = new FileInfo(path); return new DateTimeOffset(fileInfo.LastWriteTimeUtc); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + + /// + public FileStream GetFileStream(string path, bool shareWrite) => new FileStream(ResolvePath(path), FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), DefaultBufferSize, true); } } diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index d18e117864..90c13b8cdb 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; @@ -187,13 +188,13 @@ namespace Tgstation.Server.Host.IO Task DownloadFile(Uri url, CancellationToken cancellationToken); /// - /// Extract a set of to a given + /// Extract a set of to a given /// /// The path to unzip to - /// The s of the + /// The of the /// The for the operation /// A representing the running operation - Task ZipToDirectory(string path, byte[] zipFileBytes, CancellationToken cancellationToken); + Task ZipToDirectory(string path, Stream zipFile, CancellationToken cancellationToken); /// /// Get the of when a given was last modified. @@ -202,5 +203,14 @@ namespace Tgstation.Server.Host.IO /// The for the operation. /// A resulting in the of when the file was last modified. Task GetLastModified(string path, CancellationToken cancellationToken); + + /// + /// Gets the for a given file . + /// + /// The path of the file. + /// If should be used. + /// The of the file. + /// This function is sychronous. + FileStream GetFileStream(string path, bool shareWrite); } } diff --git a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs index 69f80050b5..20e6656aec 100644 --- a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Generic; +using System.IO; using System.Threading; namespace Tgstation.Server.Host.IO @@ -50,11 +51,11 @@ namespace Tgstation.Server.Host.IO /// Write to a file at a given /// /// The path to the file to write - /// The new contents of the file + /// A containing the new contents of the file /// The function only succeeds if this parameter matches the SHA-1 hash of the contents of the current file. Contains the SHA1 of the file on disk once the function returns /// The for the operation /// on success, if the operation failed due to not matching the file's contents - bool WriteFileChecked(string path, byte[] data, ref string sha1InOut, CancellationToken cancellationToken); + bool WriteFileChecked(string path, Stream data, ref string sha1InOut, CancellationToken cancellationToken); /// /// Checks if a given is a directory diff --git a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs index 2a6530faa0..c3c66bf92e 100644 --- a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -74,30 +74,24 @@ namespace Tgstation.Server.Host.IO } /// - public bool WriteFileChecked(string path, byte[] data, ref string sha1InOut, CancellationToken cancellationToken) + public bool WriteFileChecked(string path, Stream data, ref string sha1InOut, CancellationToken cancellationToken) { if (path == null) throw new ArgumentNullException(nameof(path)); + if (data == null) + throw new ArgumentNullException(nameof(data)); + cancellationToken.ThrowIfCancellationRequested(); var directory = Path.GetDirectoryName(path); + Directory.CreateDirectory(directory); cancellationToken.ThrowIfCancellationRequested(); using (var file = File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) { cancellationToken.ThrowIfCancellationRequested(); - // as nice as it would be to not have to arrayify the memory stream, we have to - // because, oddly enough sha1(memorystream) != sha1(memorystream.ToArray()) - // vOv - byte[] originalBytes; - using (var readMs = new MemoryStream()) - { - file.CopyTo(readMs); - originalBytes = readMs.ToArray(); - } - // no sha1? no write - if (originalBytes.Length != 0 && sha1InOut == null) + if (file.Length != 0 && sha1InOut == null) return false; // suppressed due to only using for consistency checks @@ -105,8 +99,8 @@ namespace Tgstation.Server.Host.IO using (var sha1 = new SHA1Managed()) #pragma warning restore CA5350 // Do not use insecure cryptographic algorithm SHA1. { - string GetSha1(byte[] dataToHash) => dataToHash != null && dataToHash.Length != 0 ? String.Join(String.Empty, sha1.ComputeHash(dataToHash).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))) : null; - var originalSha1 = GetSha1(originalBytes); + string GetSha1(Stream dataToHash) => dataToHash != null && dataToHash.Length != 0 ? String.Join(String.Empty, sha1.ComputeHash(dataToHash).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))) : null; + var originalSha1 = GetSha1(file); if (originalSha1 != sha1InOut) { sha1InOut = originalSha1; @@ -118,17 +112,18 @@ namespace Tgstation.Server.Host.IO cancellationToken.ThrowIfCancellationRequested(); - if (data != null) + if (data.Length != 0) { file.Seek(0, SeekOrigin.Begin); + data.Seek(0, SeekOrigin.Begin); cancellationToken.ThrowIfCancellationRequested(); file.SetLength(data.Length); - file.Write(data, 0, data.Length); + data.CopyTo(file); } } - if (data == null) + if (data.Length == 0) File.Delete(path); return true; } diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 8ce353d0ae..dbfaee0b7a 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -194,8 +194,11 @@ namespace Tgstation.Server.Host var cancellationToken = cancellationTokenSource.Token; logger.LogTrace("Downloading zip package..."); - var updateZipData = await ioManager.DownloadFile(updateZipUrl, cancellationToken).ConfigureAwait(false); - + using var updateZipData = new MemoryStream( + await ioManager.DownloadFile( + updateZipUrl, + cancellationToken) + .ConfigureAwait(false)); try { logger.LogTrace("Exctracting zip package to {0}...", updatePath); diff --git a/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs b/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs new file mode 100644 index 0000000000..16f78aad1a --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileDownloadProvider.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Represents a file on disk to be downloaded. + /// + public sealed class FileDownloadProvider + { + /// + /// A to run before providing the download. If it returns a non-null , a 400 error with that code will be returned instead of a download stream. + /// + public Func ActivationCallback { get; } + + /// + /// A to specially provide a returning the . + /// + public Func> FileStreamProvider { get; } + + /// + /// The full path to the file on disk to download. + /// + public string FilePath { get; } + + /// + /// If the file read stream should be allowed to share writes. If this is set, the entire file will be buffered to avoid Content-Length mismatches. + /// + public bool ShareWrite { get; } + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The optional value of . + /// The value of . + /// The value of . + public FileDownloadProvider( + Func activationCallback, + Func> fileStreamProvider, + string filePath, + bool shareWrite) + { + ActivationCallback = activationCallback ?? throw new ArgumentNullException(nameof(activationCallback)); + FileStreamProvider = fileStreamProvider; + FilePath = filePath ?? throw new ArgumentNullException(nameof(filePath)); + ShareWrite = shareWrite; + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs new file mode 100644 index 0000000000..812e2c11f7 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs @@ -0,0 +1,277 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Implementation of the file transfer service. + /// + sealed class FileTransferService : IFileTransferTicketProvider, IFileTransferStreamHandler, IAsyncDisposable + { + /// + /// Number of minutes before transfer ticket expire. + /// + const int TicketValidityMinutes = 5; + + /// + /// The for the . + /// + readonly ICryptographySuite cryptographySuite; + + /// + /// The for the . + /// + readonly IIOManager ioManager; + + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// of s to upload s. + /// + readonly Dictionary uploadTickets; + + /// + /// of s to s. + /// + readonly Dictionary downloadTickets; + + /// + /// that is triggered when is called. + /// + readonly CancellationTokenSource disposeCts; + + /// + /// used to update . + /// + readonly object synchronizationLock; + + /// + /// Combined of all calls. + /// + Task expireTask; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + public FileTransferService( + ICryptographySuite cryptographySuite, + IIOManager ioManager, + IAsyncDelayer asyncDelayer, + ILogger logger) + { + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + uploadTickets = new Dictionary(); + downloadTickets = new Dictionary(); + + disposeCts = new CancellationTokenSource(); + + expireTask = Task.CompletedTask; + synchronizationLock = new object(); + } + + /// + public async ValueTask DisposeAsync() + { + Task toAwait; + lock (synchronizationLock) + if (expireTask != null) + { + disposeCts.Cancel(); + disposeCts.Dispose(); + toAwait = expireTask; + expireTask = null; + } + else + toAwait = Task.CompletedTask; + + await toAwait.ConfigureAwait(false); + } + + /// + /// Creates a new . + /// + /// A new . + FileTicketResult CreateTicket() => new FileTicketResult + { + FileTicket = cryptographySuite.GetSecureString() + }; + + void QueueExpiry(Action expireAction) + { + Task oldExpireTask = null; + + async Task ExpireAsync() + { + var expireAt = DateTimeOffset.Now + TimeSpan.FromMinutes(TicketValidityMinutes); + try + { + await oldExpireTask.WithToken(disposeCts.Token).ConfigureAwait(false); + + var now = DateTimeOffset.Now; + if (now < expireAt) + await asyncDelayer.Delay(expireAt - now, disposeCts.Token).ConfigureAwait(false); + } + finally + { + expireAction(); + } + } + + lock (synchronizationLock) + { + oldExpireTask = expireTask; + expireTask = ExpireAsync(); + } + } + + /// + public FileTicketResult CreateDownload(FileDownloadProvider downloadProvider) + { + if (downloadProvider == null) + throw new ArgumentNullException(nameof(downloadProvider)); + + logger.LogDebug("Creating download ticket for path {0}", downloadProvider.FilePath); + var ticketResult = CreateTicket(); + + lock (downloadTickets) + downloadTickets.Add(ticketResult.FileTicket, downloadProvider); + + QueueExpiry(() => + { + lock (downloadTickets) + if(downloadTickets.Remove(ticketResult.FileTicket)) + logger.LogTrace("Expired download ticket {0}...", ticketResult.FileTicket); + }); + + logger.LogTrace("Created download ticket {0}", ticketResult.FileTicket); + + return ticketResult; + } + + /// + public IFileUploadTicket CreateUpload(bool requireSynchronousIO) + { + logger.LogDebug("Creating upload ticket..."); + var uploadTicket = new FileUploadProvider(CreateTicket(), requireSynchronousIO); + + lock (uploadTickets) + uploadTickets.Add(uploadTicket.Ticket.FileTicket, uploadTicket); + + QueueExpiry(() => + { + lock (uploadTickets) + if (uploadTickets.Remove(uploadTicket.Ticket.FileTicket)) + logger.LogTrace("Expired upload ticket {0}...", uploadTicket.Ticket.FileTicket); + else + return; + + uploadTicket.Expire(); + }); + + logger.LogTrace("Created upload ticket {0}", uploadTicket.Ticket.FileTicket); + + return uploadTicket; + } + + /// + public async Task> RetrieveDownloadStream(FileTicketResult ticket, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + FileDownloadProvider downloadProvider; + lock (downloadTickets) + { + if (!downloadTickets.TryGetValue(ticket.FileTicket, out downloadProvider)) + { + logger.LogTrace("Download ticket {0} not found!", ticket.FileTicket); + return Tuple.Create(null, null); + } + + downloadTickets.Remove(ticket.FileTicket); + } + + var errorCode = downloadProvider.ActivationCallback(); + if (errorCode.HasValue) + { + logger.LogDebug("Download ticket {0} failed activation!", ticket.FileTicket); + return Tuple.Create(null, new ErrorMessage(errorCode.Value)); + } + + FileStream stream; + try + { + if (downloadProvider.FileStreamProvider != null) + stream = await downloadProvider.FileStreamProvider(cancellationToken).ConfigureAwait(false); + else + stream = ioManager.GetFileStream(downloadProvider.FilePath, downloadProvider.ShareWrite); + } + catch (IOException ex) + { + return Tuple.Create( + null, + new ErrorMessage(ErrorCode.IOError) + { + AdditionalData = ex.ToString() + }); + } + + try + { + logger.LogTrace("Ticket {0} downloading...", ticket.FileTicket); + return Tuple.Create(stream, null); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// + public async Task SetUploadStream(FileTicketResult ticket, Stream stream, CancellationToken cancellationToken) + { + if (ticket == null) + throw new ArgumentNullException(nameof(ticket)); + + FileUploadProvider uploadProvider; + lock (uploadTickets) + { + if (!uploadTickets.TryGetValue(ticket.FileTicket, out uploadProvider)) + { + logger.LogTrace("Upload ticket {0} not found!", ticket.FileTicket); + return new ErrorMessage(ErrorCode.ResourceNotPresent); + } + + uploadTickets.Remove(ticket.FileTicket); + } + + return await uploadProvider.Completion(stream, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs b/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs new file mode 100644 index 0000000000..ea5e47a245 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/FileUploadProvider.cs @@ -0,0 +1,128 @@ +using Microsoft.AspNetCore.WebUtilities; +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; + +namespace Tgstation.Server.Host.Transfer +{ + /// + sealed class FileUploadProvider : IFileUploadTicket + { + /// + public FileTicketResult Ticket { get; } + + /// + /// The for the ticket duration. + /// + readonly CancellationTokenSource ticketExpiryCts; + + /// + /// The for the . + /// + readonly TaskCompletionSource taskCompletionSource; + + /// + /// The that completes in or when is called. + /// + readonly TaskCompletionSource completionTcs; + + /// + /// If synchronous IO is required. Uses a as a backend if set. + /// + readonly bool requireSynchronousIO; + + /// + /// The that occurred while processing the upload if any. + /// + ErrorMessage errorMessage; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of + public FileUploadProvider(FileTicketResult ticket, bool requireSynchronousIO) + { + Ticket = ticket ?? throw new ArgumentNullException(nameof(ticket)); + + ticketExpiryCts = new CancellationTokenSource(); + taskCompletionSource = new TaskCompletionSource(); + completionTcs = new TaskCompletionSource(); + this.requireSynchronousIO = requireSynchronousIO; + } + + /// + public void Dispose() + { + ticketExpiryCts.Dispose(); + completionTcs.TrySetResult(null); + } + + /// + public async Task GetResult(CancellationToken cancellationToken) + { + using (cancellationToken.Register(() => taskCompletionSource.TrySetCanceled())) + using (ticketExpiryCts.Token.Register(() => taskCompletionSource.TrySetResult(null))) + return await taskCompletionSource.Task.ConfigureAwait(false); + } + + /// + /// Expire the . + /// + public void Expire() + { + if (!completionTcs.Task.IsCompleted) + ticketExpiryCts.Cancel(); + } + + /// + /// Resolve the for the and awaits the upload. + /// + /// The containing uploaded data. + /// The for the operation. + /// A resulting in , otherwise. + public async Task Completion(Stream stream, CancellationToken cancellationToken) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + if (ticketExpiryCts.IsCancellationRequested) + return new ErrorMessage(ErrorCode.ResourceNotPresent); + + Stream bufferedStream = null; + if (requireSynchronousIO) + { + // big reads, we should buffer to disk + bufferedStream = new FileBufferingReadStream(stream, DefaultIOManager.DefaultBufferSize); + await bufferedStream.DrainAsync(cancellationToken).ConfigureAwait(false); + } + + using (bufferedStream) + { + taskCompletionSource.TrySetResult(bufferedStream ?? stream); + + await completionTcs.Task.WithToken(cancellationToken).ConfigureAwait(false); + return errorMessage; + } + } + + /// + public void SetErrorMessage(ErrorMessage errorMessage) + { + if (errorMessage == null) +#pragma warning disable IDE0016 // Use 'throw' expression + throw new ArgumentNullException(nameof(errorMessage)); +#pragma warning restore IDE0016 // Use 'throw' expression + + if (this.errorMessage != null) + throw new InvalidOperationException("ErrorMessage already set!"); + + this.errorMessage = errorMessage; + completionTcs.TrySetResult(null); + } + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs b/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs new file mode 100644 index 0000000000..fd8a051332 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileTransferStreamHandler.cs @@ -0,0 +1,31 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Reads and writes to s associated with s. + /// + public interface IFileTransferStreamHandler + { + /// + /// Sets the for a given associated with a pending upload. + /// + /// The . + /// The with uploaded data. + /// The for the operation. + /// if the upload completed successfully, otherwise. + Task SetUploadStream(FileTicketResult ticket, Stream stream, CancellationToken cancellationToken); + + /// + /// Gets the the for a given associated with a pending download. + /// + /// The . + /// The for the operation. + /// A containing either a containing the data to download or an to return. + Task> RetrieveDownloadStream(FileTicketResult ticket, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs b/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs new file mode 100644 index 0000000000..5edfb86cd7 --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileTransferTicketProvider.cs @@ -0,0 +1,24 @@ +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// Service for temporarily storing files to be downloaded or uploaded. + /// + public interface IFileTransferTicketProvider + { + /// + /// Create a for a download. + /// + /// The . + /// A new for a download. + FileTicketResult CreateDownload(FileDownloadProvider fileDownloadProvider); + + /// + /// Create a . + /// + /// If synchronous IO is required on the provided stream. + /// A new . + IFileUploadTicket CreateUpload(bool requiresSynchronousIO); + } +} diff --git a/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs b/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs new file mode 100644 index 0000000000..4eb1ea622c --- /dev/null +++ b/src/Tgstation.Server.Host/Transfer/IFileUploadTicket.cs @@ -0,0 +1,33 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Host.Transfer +{ + /// + /// A that waits for a pending upload. + /// + public interface IFileUploadTicket : IDisposable + { + /// + /// The . + /// + FileTicketResult Ticket { get; } + + /// + /// Gets the for the uploaded file. + /// + /// The for the operation. + /// A resulting in the uploaded of the file on success, if the ticket timed out. + /// The resulting is short lived and should be buffered if it needs use outside the lifetime of the . + Task GetResult(CancellationToken cancellationToken); + + /// + /// Sets an for the upload. Will be returned in upload request as a 409 error. + /// + /// The to set. + void SetErrorMessage(ErrorMessage errorMessage); + } +} diff --git a/tests/Tgstation.Server.Tests/AdministrationTest.cs b/tests/Tgstation.Server.Tests/AdministrationTest.cs index 578e7f3e91..cddd0b0aa4 100644 --- a/tests/Tgstation.Server.Tests/AdministrationTest.cs +++ b/tests/Tgstation.Server.Tests/AdministrationTest.cs @@ -33,12 +33,12 @@ namespace Tgstation.Server.Tests var logFile = logs.First(); Assert.IsNotNull(logFile); Assert.IsFalse(String.IsNullOrWhiteSpace(logFile.Name)); - Assert.IsNull(logFile.Content); + Assert.IsNull(logFile.FileTicket); - var downloaded = await client.GetLog(logFile, cancellationToken); - Assert.AreEqual(logFile.Name, downloaded.Name); - Assert.IsTrue(logFile.LastModified <= downloaded.LastModified); - Assert.IsNull(logFile.Content); + var downloadedTuple = await client.GetLog(logFile, cancellationToken); + Assert.AreEqual(logFile.Name, downloadedTuple.Item1.Name); + Assert.IsTrue(logFile.LastModified <= downloadedTuple.Item1.LastModified); + Assert.IsNull(logFile.FileTicket); await ApiAssert.ThrowsException(() => client.GetLog(new LogFile { diff --git a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs index 9266e01da6..11a24484e4 100644 --- a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs @@ -1,4 +1,4 @@ -using Castle.Core.Logging; +using Castle.Core.Logging; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Instance { Version = new Version(5011, 1385) }; - var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); + var test = await byondClient.SetActiveVersion(newModel, null, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); await WaitForJob(test.InstallJob, 60, true, ErrorCode.ByondDownloadFail, cancellationToken).ConfigureAwait(false); } @@ -56,7 +56,7 @@ namespace Tgstation.Server.Tests.Instance { Version = TestVersion }; - var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); + var test = await byondClient.SetActiveVersion(newModel, null, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); Assert.IsNull(test.Version); await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); @@ -98,11 +98,18 @@ namespace Tgstation.Server.Tests.Instance Mock.Of>()); // get the bytes for stable - var test = await byondClient.SetActiveVersion(new Api.Models.Byond - { - Version = TestVersion, - Content = await byondInstaller.DownloadVersion(TestVersion, cancellationToken) - }, cancellationToken).ConfigureAwait(false); + using var stableBytesMs = new MemoryStream( + await byondInstaller.DownloadVersion(TestVersion, cancellationToken)); + + var test = await byondClient.SetActiveVersion( + new Api.Models.Byond + { + Version = TestVersion, + UploadCustomZip = true + }, + stableBytesMs, + cancellationToken) + .ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); @@ -114,17 +121,17 @@ namespace Tgstation.Server.Tests.Instance newSettings = await byondClient.SetActiveVersion(new Api.Models.Byond { Version = TestVersion - }, cancellationToken); + }, null, cancellationToken); Assert.IsNull(newSettings.InstallJob); await ApiAssert.ThrowsException(() => byondClient.SetActiveVersion(new Api.Models.Byond { Version = new Version(TestVersion.Major, TestVersion.Minor, 2) - }, cancellationToken), ErrorCode.ByondNonExistentCustomVersion); + }, null, cancellationToken), ErrorCode.ByondNonExistentCustomVersion); newSettings = await byondClient.SetActiveVersion(new Api.Models.Byond { Version = new Version(TestVersion.Major, TestVersion.Minor, 1) - }, cancellationToken); + }, null, cancellationToken); Assert.IsNull(newSettings.InstallJob); } } diff --git a/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs index b30178169f..b9e353a632 100644 --- a/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ConfigurationTest.cs @@ -1,6 +1,9 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; +using System.Net.Http; +using System.Net.Mime; +using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -25,7 +28,8 @@ namespace Tgstation.Server.Tests.Instance { var tmp = (file.Path?.StartsWith('/') ?? false) ? '.' + file.Path : file.Path; var path = Path.Combine(instance.Path, "Configuration", tmp); - return File.Exists(path); + var result = File.Exists(path); + return result; } async Task TestDeleteDirectory(CancellationToken cancellationToken) @@ -39,20 +43,43 @@ namespace Tgstation.Server.Tests.Instance await configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken).ConfigureAwait(false); //try to delete non-empty + const string TestString = "Hello world!"; + using var uploadMs = new MemoryStream(Encoding.UTF8.GetBytes(TestString)); var file = await configurationClient.Write(new ConfigurationFile { - Content = Encoding.UTF8.GetBytes("Hello world!"), Path = TestDir.Path + "/test.txt" - }, cancellationToken).ConfigureAwait(false); + }, uploadMs, cancellationToken).ConfigureAwait(false); Assert.IsTrue(FileExists(file)); + Assert.IsNull(file.LastReadHash); + + var updatedFileTuple = await configurationClient.Read(file, cancellationToken).ConfigureAwait(false); + var updatedFile = updatedFileTuple.Item1; + Assert.IsNotNull(updatedFile.LastReadHash); + using (var downloadMemoryStream = new MemoryStream()) + { + using (var downloadStream = updatedFileTuple.Item2) + { + var requestStream = downloadStream as CachedResponseStream; + Assert.IsNotNull(requestStream); + var response = (HttpResponseMessage)requestStream.GetType().GetField("response", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(requestStream); + Assert.AreEqual(response.Content.Headers.ContentType.MediaType, MediaTypeNames.Application.Octet); + await downloadStream.CopyToAsync(downloadMemoryStream); + } + Assert.AreEqual(TestString, Encoding.UTF8.GetString(downloadMemoryStream.ToArray()).Trim()); + } await ApiAssert.ThrowsException(() => configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken), ErrorCode.ConfigurationDirectoryNotEmpty).ConfigureAwait(false); - file.Content = null; - await configurationClient.Write(file, cancellationToken).ConfigureAwait(false); + file.FileTicket = null; + await configurationClient.Write(updatedFile, null, cancellationToken).ConfigureAwait(false); + Assert.IsFalse(FileExists(file)); await configurationClient.DeleteEmptyDirectory(TestDir, cancellationToken).ConfigureAwait(false); + + var tmp = (TestDir.Path?.StartsWith('/') ?? false) ? '.' + TestDir.Path : TestDir.Path; + var path = Path.Combine(instance.Path, "Configuration", tmp); + Assert.IsFalse(Directory.Exists(path)); } public async Task Run(CancellationToken cancellationToken) diff --git a/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs index def6f1385d..02b3251622 100644 --- a/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs @@ -79,7 +79,7 @@ namespace Tgstation.Server.Tests.Instance clone = await repositoryClient.Clone(initalRepo, cancellationToken).ConfigureAwait(false); - await WaitForJob(clone.ActiveJob, 180, false, null, cancellationToken).ConfigureAwait(false); + await WaitForJob(clone.ActiveJob, 600, false, null, cancellationToken).ConfigureAwait(false); var cloned = await repositoryClient.Read(cancellationToken); Assert.AreEqual(Origin, cloned.Origin); diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 0c6e8915df..35dddd9d35 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -396,6 +396,7 @@ namespace Tgstation.Server.Tests.Instance { Version = versionToInstall }, + null, cancellationToken); var byondInstallJob = await byondInstallJobTask; diff --git a/tests/Tgstation.Server.Tests/RootTest.cs b/tests/Tgstation.Server.Tests/RootTest.cs index be2b87cb95..eccf9c0889 100644 --- a/tests/Tgstation.Server.Tests/RootTest.cs +++ b/tests/Tgstation.Server.Tests/RootTest.cs @@ -194,11 +194,86 @@ namespace Tgstation.Server.Tests Assert.AreEqual(HttpStatusCode.Unauthorized, response.StatusCode); } } + async Task TestInvalidTransfers(IServerClient serverClient, CancellationToken cancellationToken) + { + var url = serverClient.Url; + var token = serverClient.Token.Bearer; + // check that 400s are returned appropriately + using var httpClient = new HttpClient(); + + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.Transfer.Substring(1))) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); + request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/" + ApiHeaders.Version); + request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); + using var response = await httpClient.SendAsync(request, cancellationToken); + Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); + var content = await response.Content.ReadAsStringAsync(); + var message = JsonConvert.DeserializeObject(content); + Assert.AreEqual(MediaTypeNames.Application.Json, response.Content.Headers.ContentType.MediaType); + Assert.AreEqual(ErrorCode.ModelValidationFailure, message.ErrorCode); + } + + using (var request = new HttpRequestMessage(HttpMethod.Put, url.ToString() + Routes.Transfer.Substring(1))) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); + request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/" + ApiHeaders.Version); + request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); + using var response = await httpClient.SendAsync(request, cancellationToken); + Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); + var content = await response.Content.ReadAsStringAsync(); + var message = JsonConvert.DeserializeObject(content); + Assert.AreEqual(MediaTypeNames.Application.Json, response.Content.Headers.ContentType.MediaType); + Assert.AreEqual(ErrorCode.ModelValidationFailure, message.ErrorCode); + } + + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.Transfer.Substring(1) + "?ticket=veryfaketicket")) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); + request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/" + ApiHeaders.Version); + request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); + using var response = await httpClient.SendAsync(request, cancellationToken); + Assert.AreEqual(MediaTypeNames.Application.Json, response.Content.Headers.ContentType.MediaType); + Assert.AreEqual(HttpStatusCode.NotAcceptable, response.StatusCode); + } + + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.Transfer.Substring(1) + "?ticket=veryfaketicket")) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Octet)); + request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/" + ApiHeaders.Version); + request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); + using var response = await httpClient.SendAsync(request, cancellationToken); + Assert.AreEqual(MediaTypeNames.Application.Json, response.Content.Headers.ContentType.MediaType); + Assert.AreEqual(HttpStatusCode.Gone, response.StatusCode); + } + + using (var request = new HttpRequestMessage(HttpMethod.Put, url.ToString() + Routes.Transfer.Substring(1) + "?ticket=veryfaketicket")) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Json)); + request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/" + ApiHeaders.Version); + request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); + using var response = await httpClient.SendAsync(request, cancellationToken); + Assert.AreEqual(MediaTypeNames.Application.Json, response.Content.Headers.ContentType.MediaType); + Assert.AreEqual(HttpStatusCode.Gone, response.StatusCode); + } + } public Task Run(IServerClientFactory clientFactory, IServerClient serverClient, CancellationToken cancellationToken) => Task.WhenAll( TestRequestValidation(serverClient, cancellationToken), TestOAuthFails(serverClient, cancellationToken), - TestServerInformation(clientFactory, serverClient, cancellationToken)); + TestServerInformation(clientFactory, serverClient, cancellationToken), + TestInvalidTransfers(serverClient, cancellationToken)); } }