Merge pull request #1340 from tgstation/LogsLogsLogs [TGSDeploy]

Improve logging of failed OAuth handshakes
This commit is contained in:
Jordan Brown
2021-10-13 12:56:38 -04:00
committed by GitHub
14 changed files with 112 additions and 92 deletions
+1 -5
View File
@@ -187,9 +187,7 @@ jobs:
env:
TGS_TEST_DATABASE_TYPE: SqlServer
TGS_TEST_DUMP_API_SPEC: yes
concurrency: integration-windows-${{ github.head_ref }}
strategy:
max-parallel: 2
matrix:
watchdog-type: [ 'Basic', 'System' ]
configuration: [ 'Debug', 'Release' ]
@@ -303,9 +301,7 @@ jobs:
--health-interval=10s
--health-timeout=5s
--health-retries=3
concurrency: integration-linux-${{ github.head_ref }}
strategy:
max-parallel: 2
matrix:
database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ]
watchdog-type: [ 'System' ]
@@ -773,7 +769,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y xmlstarlet
xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props >> $Env:GITHUB_ENV
echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $Env:GITHUB_ENV
- name: Docker Build and Push
uses: elgohr/Publish-Docker-Github-Action@master
+1 -1
View File
@@ -3,7 +3,7 @@
<!-- Integration tests will ensure they match across the board -->
<Import Project="ControlPanelVersion.props" />
<PropertyGroup>
<TgsCoreVersion>4.15.3</TgsCoreVersion>
<TgsCoreVersion>4.15.4</TgsCoreVersion>
<TgsConfigVersion>4.0.0</TgsConfigVersion>
<TgsApiVersion>9.3.0</TgsApiVersion>
<TgsApiLibraryVersion>9.3.1</TgsApiLibraryVersion>
@@ -190,9 +190,16 @@ namespace Tgstation.Server.Host.Components.Chat
throw new ArgumentNullException(nameof(newChannels));
logger.LogTrace("ChangeChannels {0}...", connectionId);
var provider = await RemoveProvider(connectionId, false, cancellationToken).ConfigureAwait(false);
var provider = await RemoveProviderChannels(connectionId, false, cancellationToken).ConfigureAwait(false);
if (provider == null)
return;
if (!provider.Connected)
{
logger.LogDebug("Cannot map channels, provider {providerId} disconnected!", connectionId);
return;
}
var results = await provider.MapChannels(newChannels, cancellationToken).ConfigureAwait(false);
lock (activeChatBots)
{
@@ -260,29 +267,14 @@ namespace Tgstation.Server.Host.Components.Chat
throw new ArgumentNullException(nameof(newSettings));
logger.LogTrace("ChangeSettings...");
IProvider provider;
async Task DisconnectProvider(IProvider p)
{
try
{
await p.Disconnect(cancellationToken).ConfigureAwait(false);
}
finally
{
await p.DisposeAsync().ConfigureAwait(false);
}
}
Task disconnectTask;
IProvider provider = null;
lock (providers)
{
// raw settings changes forces a rebuild of the provider
if (providers.TryGetValue(newSettings.Id.Value, out provider))
{
providers.Remove(newSettings.Id.Value);
disconnectTask = DisconnectProvider(provider);
}
if (providers.ContainsKey(newSettings.Id.Value))
disconnectTask = DeleteConnection(newSettings.Id.Value, cancellationToken);
else
disconnectTask = Task.CompletedTask;
if (newSettings.Enabled.Value)
@@ -430,7 +422,6 @@ namespace Tgstation.Server.Host.Components.Chat
builtinCommands.Add(tgsCommand.Name.ToUpperInvariant(), tgsCommand);
var initialChatBots = activeChatBots.ToList();
await Task.WhenAll(initialChatBots.Select(x => ChangeSettings(x, cancellationToken))).ConfigureAwait(false);
await Task.WhenAll(initialChatBots.Select(x => ChangeChannels(x.Id.Value, x.Channels, cancellationToken))).ConfigureAwait(false);
initialProviderConnectionsTask = InitialConnection();
chatHandler = MonitorMessages(handlerCts.Token);
}
@@ -441,7 +432,7 @@ namespace Tgstation.Server.Host.Components.Chat
handlerCts.Cancel();
if (chatHandler != null)
await chatHandler.ConfigureAwait(false);
await Task.WhenAll(providers.Select(x => x.Value).Select(x => x.Disconnect(cancellationToken))).ConfigureAwait(false);
await Task.WhenAll(providers.Select(x => x.Key).Select(x => DeleteConnection(x, cancellationToken))).ConfigureAwait(false);
await messageSendTask.ConfigureAwait(false);
}
@@ -480,7 +471,7 @@ namespace Tgstation.Server.Host.Components.Chat
/// <inheritdoc />
public async Task DeleteConnection(long connectionId, CancellationToken cancellationToken)
{
var provider = await RemoveProvider(connectionId, true, cancellationToken).ConfigureAwait(false);
var provider = await RemoveProviderChannels(connectionId, true, cancellationToken).ConfigureAwait(false);
if (provider != null)
try
{
@@ -510,23 +501,28 @@ namespace Tgstation.Server.Host.Components.Chat
}
/// <summary>
/// Remove a <see cref="IProvider"/> from <see cref="providers"/> and <see cref="mappedChannels"/> optionally updating the <see cref="trackingContexts"/> as well.
/// Remove a <see cref="IProvider"/> from <see cref="mappedChannels"/> optionally removing the provider itself from <see cref="providers"/> and updating the <see cref="trackingContexts"/> as well.
/// </summary>
/// <param name="connectionId">The <see cref="Api.Models.EntityId.Id"/> of the <see cref="IProvider"/> to delete.</param>
/// <param name="updateTrackings">If <see cref="trackingContexts"/> should be update.</param>
/// <param name="removeProvider">If the provider should be removed from <see cref="providers"/> and <see cref="trackingContexts"/> should be update.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="IProvider"/> being removed if it exists, <see langword="null"/> otherwise.</returns>
async Task<IProvider> RemoveProvider(long connectionId, bool updateTrackings, CancellationToken cancellationToken)
async Task<IProvider> RemoveProviderChannels(long connectionId, bool removeProvider, CancellationToken cancellationToken)
{
logger.LogTrace("RemoveProvider {0}...", connectionId);
logger.LogTrace("RemoveProviderChannels {0}...", connectionId);
IProvider provider;
lock (providers)
{
if (!providers.TryGetValue(connectionId, out provider))
{
logger.LogTrace("Aborted, no such provider!");
return null;
}
if (removeProvider)
providers.Remove(connectionId);
}
Task trackingContextsUpdateTask;
lock (mappedChannels)
{
@@ -535,7 +531,7 @@ namespace Tgstation.Server.Host.Components.Chat
var newMappedChannels = mappedChannels.Select(y => y.Value.Channel).ToList();
if (updateTrackings)
if (removeProvider)
lock (trackingContexts)
trackingContextsUpdateTask = Task.WhenAll(trackingContexts.Select(x => x.UpdateChannels(newMappedChannels, cancellationToken)));
else
@@ -607,6 +603,7 @@ namespace Tgstation.Server.Host.Components.Chat
.First();
mappedChannel = mappedChannels
.Where(x => x.Value.ProviderId == providerId && x.Value.ProviderChannelId == providerChannelId)
.Select(x => (KeyValuePair<ulong, ChannelMapping>?)x)
.FirstOrDefault();
}
@@ -652,12 +649,11 @@ namespace Tgstation.Server.Host.Components.Chat
return;
}
var mappingNonNullableKvp = mappedChannel.Value;
var mapping = mappingNonNullableKvp.Value;
var mappingChannelRepresentation = mappedChannel.Value.Value.Channel;
message.User.Channel.Id = mapping.Channel.Id;
message.User.Channel.Tag = mapping.Channel.Tag;
message.User.Channel.IsAdminChannel = mapping.Channel.IsAdminChannel;
message.User.Channel.Id = mappingChannelRepresentation.Id;
message.User.Channel.Tag = mappingChannelRepresentation.Tag;
message.User.Channel.IsAdminChannel = mappingChannelRepresentation.IsAdminChannel;
}
var splits = new List<string>(message.Content.Trim().Split(' '));
@@ -93,6 +93,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
/// </summary>
Snowflake currentUserId;
/// <summary>
/// The bot's username at the time of connection.
/// </summary>
string initialUserName;
/// <summary>
/// Normalize a discord mention string.
/// </summary>
@@ -194,20 +199,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
if (channels == null)
throw new ArgumentNullException(nameof(channels));
if (!Connected)
{
Logger.LogWarning("Cannot map channels, provider disconnected!");
return Array.Empty<ChannelRepresentation>();
}
var usersClient = serviceProvider.GetRequiredService<IDiscordRestUserAPI>();
var currentUserResponse = await usersClient.GetCurrentUserAsync(cancellationToken).ConfigureAwait(false);
if (!currentUserResponse.IsSuccess)
{
Logger.LogWarning("Error retrieving current Discord user: {0}", currentUserResponse.Error.Message);
return Array.Empty<ChannelRepresentation>();
}
bool remapRequired = false;
async Task<ChannelRepresentation> GetModelChannelFromDBChannel(Api.Models.ChatChannel channelFromDB)
{
@@ -215,14 +207,12 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
throw new InvalidOperationException("ChatChannel missing DiscordChannelId!");
var channelId = channelFromDB.DiscordChannelId.Value;
ulong discordChannelId;
string connectionName;
string friendlyName;
if (channelId == 0)
{
connectionName = currentUserResponse.Entity.Username;
connectionName = initialUserName;
friendlyName = "(Unmapped accessible channels)";
discordChannelId = 0;
}
else
{
@@ -231,6 +221,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
if (!discordChannelResponse.IsSuccess)
{
Logger.LogWarning("Error retrieving discord channel {0}: {1}", channelId, discordChannelResponse.Error.Message);
remapRequired = true;
return null;
}
@@ -240,7 +231,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
return null;
}
discordChannelId = discordChannelResponse.Entity.ID.Value;
friendlyName = discordChannelResponse.Entity.Name.Value;
var guildsClient = serviceProvider.GetRequiredService<IDiscordRestGuildAPI>();
@@ -254,6 +244,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
"Error retrieving discord guild {0}: {1}",
discordChannelResponse.Entity.GuildID.Value,
discordChannelResponse.Error.Message);
remapRequired = true;
return null;
}
@@ -262,7 +253,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
var channelModel = new ChannelRepresentation
{
RealId = discordChannelId,
RealId = channelId,
IsAdminChannel = channelFromDB.IsAdminChannel == true,
ConnectionName = connectionName,
FriendlyName = friendlyName,
@@ -276,13 +267,13 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
var tasks = channels
.Select(x => GetModelChannelFromDBChannel(x))
.Where(x => x != null)
.ToList();
await Task.WhenAll(tasks);
var enumerator = tasks
.Select(x => x.Result)
.Where(x => x != null)
.ToList();
lock (mappedChannels)
@@ -291,6 +282,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
mappedChannels.AddRange(enumerator.Select(x => x.RealId));
}
if (remapRequired)
EnqueueMessage(null);
return enumerator;
}
@@ -613,6 +607,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
gatewayReadyTcs = new TaskCompletionSource<object>();
using var gatewayConnectionAbortRegistration = cancellationToken.Register(() => gatewayReadyTcs.TrySetCanceled());
gatewayCancellationToken.Register(() => Logger.LogTrace("Stopping gateway client..."));
// reconnects keep happening until we stop or it faults, our auto-reconnector will handle the latter
localGatewayTask = gatewayClient.RunAsync(gatewayCancellationToken);
@@ -634,6 +629,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
}
currentUserId = currentUserResult.Entity.ID;
initialUserName = currentUserResult.Entity.Username;
}
finally
{
@@ -107,6 +107,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
if (Connected)
{
Logger.LogTrace("Disconnecting...");
await DisconnectImpl(cancellationToken).ConfigureAwait(false);
Logger.LogTrace("Disconnected");
}
@@ -183,6 +184,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
/// <param name="message">The <see cref="Message"/> to queue.</param>
protected void EnqueueMessage(Message message)
{
if (message == null)
Logger.LogTrace("Requesting channel remap...");
lock (messageQueue)
{
messageQueue.Enqueue(message);
@@ -1,4 +1,7 @@
using LibGit2Sharp.Handlers;
using LibGit2Sharp;
using LibGit2Sharp.Handlers;
using Tgstation.Server.Host.Jobs;
namespace Tgstation.Server.Host.Components.Repository
{
@@ -14,5 +17,11 @@ namespace Tgstation.Server.Host.Components.Repository
/// <param name="password">The optional password to use in the <see cref="CredentialsHandler"/>.</param>
/// <returns>A new <see cref="CredentialsHandler"/>.</returns>
CredentialsHandler GenerateCredentialsHandler(string username, string password);
/// <summary>
/// Rethrow the authentication failure message as a <see cref="JobException"/> if it is one.
/// </summary>
/// <param name="exception">The current <see cref="LibGit2SharpException"/>.</param>
public void CheckBadCredentialsException(LibGit2SharpException exception);
}
}
@@ -71,6 +71,11 @@ namespace Tgstation.Server.Host.Components.Repository
logger.LogTrace(ex, "Suppressing clone cancellation exception");
cancellationToken.ThrowIfCancellationRequested();
}
catch (LibGit2SharpException ex)
{
CheckBadCredentialsException(ex);
throw;
}
},
cancellationToken,
DefaultIOManager.BlockingTaskCreationOptions,
@@ -99,5 +104,22 @@ namespace Tgstation.Server.Host.Components.Repository
throw new JobException(ErrorCode.RepoCannotAuthenticate);
};
/// <inheritdoc />
public void CheckBadCredentialsException(LibGit2SharpException exception)
{
if (exception == null)
throw new ArgumentNullException(nameof(exception));
if (exception.Message == "too many redirects or authentication replays")
throw new JobException("Bad git credentials exchange!", exception);
if (exception.Message == ErrorCode.RepoCredentialsRequired.Describe())
throw new JobException(ErrorCode.RepoCredentialsRequired);
// submodule recursion
if (exception.InnerException is LibGit2SharpException innerException)
CheckBadCredentialsException(innerException);
}
}
}
@@ -111,19 +111,6 @@ namespace Tgstation.Server.Host.Components.Repository
/// </summary>
bool disposed;
/// <summary>
/// Rethrow the authentication failure message as a <see cref="JobException"/> if it is one.
/// </summary>
/// <param name="exception">The current <see cref="LibGit2SharpException"/>.</param>
static void CheckBadCredentialsException(LibGit2SharpException exception)
{
if (exception.Message == "too many redirects or authentication replays")
throw new JobException("Bad git credentials exchange!", exception);
if (exception.Message == ErrorCode.RepoCredentialsRequired.Describe())
throw new JobException(ErrorCode.RepoCredentialsRequired);
}
/// <summary>
/// Initializes a new instance of the <see cref="Repository"/> class.
/// </summary>
@@ -262,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Repository
}
catch (LibGit2SharpException ex)
{
CheckBadCredentialsException(ex);
credentialsProvider.CheckBadCredentialsException(ex);
}
cancellationToken.ThrowIfCancellationRequested();
@@ -289,7 +276,7 @@ namespace Tgstation.Server.Host.Components.Repository
FastForwardStrategy = FastForwardStrategy.NoFastForward,
SkipReuc = true,
OnCheckoutProgress = CheckoutProgressHandler(
(lambdaStage, progress) => progressReporter(lambdaStage, 50 + (progress / 2)),
(lambdaStage, progress) => progressReporter(lambdaStage, progress.HasValue ? (int?)(50 + (progress.Value / 2)) : null),
$"Merge {testMergeParameters.TargetCommitSha}"),
});
}
@@ -439,7 +426,7 @@ namespace Tgstation.Server.Host.Components.Repository
}
catch (LibGit2SharpException ex)
{
CheckBadCredentialsException(ex);
credentialsProvider.CheckBadCredentialsException(ex);
}
},
cancellationToken,
@@ -659,7 +646,7 @@ namespace Tgstation.Server.Host.Components.Repository
{
libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions
{
OnCheckoutProgress = CheckoutProgressHandler((stage, progress) => progressReporter(stage, progress / 10), "Hard reset and remove untracked files"),
OnCheckoutProgress = CheckoutProgressHandler((stage, progress) => progressReporter(stage, progress.HasValue ? (int?)(progress.Value / 10) : null), "Hard reset and remove untracked files"),
});
cancellationToken.ThrowIfCancellationRequested();
libGitRepo.RemoveUntrackedFiles();
@@ -952,7 +939,7 @@ namespace Tgstation.Server.Host.Components.Repository
var factor = 100 / submoduleCount;
foreach (var submodule in libGitRepo.Submodules)
{
void LocalProgressReporter(string stage, int percentage) => progressReporter(stage, (iteration * factor) + (percentage / submoduleCount));
void LocalProgressReporter(string stage, int? percentage) => progressReporter(stage, percentage.HasValue ? (int?)((iteration * factor) + (percentage.Value / submoduleCount)) : null);
var submoduleUpdateOptions = new SubmoduleUpdateOptions
{
Init = true,
@@ -964,7 +951,7 @@ namespace Tgstation.Server.Host.Components.Repository
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
OnCheckoutProgress = CheckoutProgressHandler(
(stage, progress) => LocalProgressReporter(stage, 50 + (progress.Value / 2)),
(stage, progress) => LocalProgressReporter(stage, progress.HasValue ? (int?)(50 + (progress.Value / 2)) : null),
$"Checkout submodule {submodule.Name}"),
};
@@ -982,7 +969,7 @@ namespace Tgstation.Server.Host.Components.Repository
{
// workaround for https://github.com/libgit2/libgit2/issues/3820
// kill off the modules/ folder in .git and try again
CheckBadCredentialsException(ex);
credentialsProvider.CheckBadCredentialsException(ex);
logger.LogWarning(ex, "Initial update of submodule {0} failed. Deleting submodule directories and re-attempting...", submodule.Name);
await Task.WhenAll(
@@ -1001,7 +988,7 @@ namespace Tgstation.Server.Host.Components.Repository
}
catch (LibGit2SharpException ex2)
{
CheckBadCredentialsException(ex2);
credentialsProvider.CheckBadCredentialsException(ex2);
logger.LogTrace(ex2, "Retried update of submodule {0} failed!", submodule.Name);
throw new AggregateException(ex, ex2);
}
@@ -1020,7 +1007,11 @@ namespace Tgstation.Server.Host.Components.Repository
CheckoutProgressHandler CheckoutProgressHandler(JobProgressReporter progressReporter, string stage) => (a, completedSteps, totalSteps) =>
{
int? percentage;
if (totalSteps < completedSteps || totalSteps == 0)
// short circuit initialization where totalSteps is 0
if (completedSteps == 0)
percentage = 0;
else if (totalSteps < completedSteps || totalSteps == 0)
percentage = null;
else
{
@@ -162,9 +162,9 @@ namespace Tgstation.Server.Host.Components.Repository
// DCT: Cancellation token is for job, operation must run regardless
await ioManager.DeleteDirectory(repositoryPath, default).ConfigureAwait(false);
}
catch (Exception e)
catch (Exception innerException)
{
logger.LogDebug(e, "Error deleting partially cloned repository!");
logger.LogError(innerException, "Error deleting partially cloned repository!");
}
throw;
@@ -519,7 +519,7 @@ namespace Tgstation.Server.Host.Controllers
{
IQueryable<Models.Instance> GetBaseQuery()
{
IQueryable<Models.Instance> query = DatabaseContext
var query = DatabaseContext
.Instances
.AsQueryable()
.Where(x => x.SwarmIdentifer == swarmConfiguration.Identifier);
@@ -55,6 +55,8 @@ namespace Tgstation.Server.Host.Security.OAuth
public override async Task<string> ValidateResponseCode(string code, CancellationToken cancellationToken)
{
using var httpClient = CreateHttpClient();
string tokenResponsePayload = null;
string userInformationPayload = null;
try
{
Logger.LogTrace("Validating response code...");
@@ -71,8 +73,8 @@ namespace Tgstation.Server.Host.Security.OAuth
tokenRequest.Content = new FormUrlEncodedContent(tokenRequestDictionary);
var tokenResponse = await httpClient.SendAsync(tokenRequest, cancellationToken).ConfigureAwait(false);
tokenResponsePayload = await tokenResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
tokenResponse.EnsureSuccessStatusCode();
var tokenResponsePayload = await tokenResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
var tokenResponseJson = JObject.Parse(tokenResponsePayload);
var accessToken = DecodeTokenPayload(tokenResponseJson);
@@ -89,16 +91,20 @@ namespace Tgstation.Server.Host.Security.OAuth
accessToken);
var userInformationResponse = await httpClient.SendAsync(userInformationRequest, cancellationToken).ConfigureAwait(false);
userInformationPayload = await userInformationResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
userInformationResponse.EnsureSuccessStatusCode();
var userInformationPayload = await userInformationResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
var userInformationJson = JObject.Parse(userInformationPayload);
return DecodeUserInformationPayload(userInformationJson);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Error while completing OAuth handshake!");
Logger.LogWarning(
ex,
"Error while completing OAuth handshake! Payload:{newLine}{responsePayload}",
Environment.NewLine,
userInformationPayload ?? tokenResponsePayload);
return null;
}
}
@@ -87,7 +87,7 @@
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="3.1.11" />
<PackageReference Include="Octokit" Version="0.50.0" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.2.6" />
<PackageReference Include="Remora.Discord" Version="3.0.58" />
<PackageReference Include="Remora.Discord" Version="3.0.71" />
<PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
<PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.0.0" />
@@ -1,4 +1,4 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Text;
@@ -20,7 +20,7 @@ namespace Tgstation.Server.Tests.Instance
this.JobsClient = jobsClient;
}
public async Task<JobResponse> WaitForJob(JobResponse originalJob, int timeout, bool expectFailure, ErrorCode? expectedCode, CancellationToken cancellationToken)
public async Task<JobResponse> WaitForJob(JobResponse originalJob, int timeout, bool? expectFailure, ErrorCode? expectedCode, CancellationToken cancellationToken)
{
var job = originalJob;
do
@@ -37,7 +37,7 @@ namespace Tgstation.Server.Tests.Instance
Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!");
}
if (expectFailure ^ job.ExceptionDetails != null)
if(expectFailure.HasValue && (expectFailure.Value ^ job.ExceptionDetails != null))
Assert.Fail(job.ExceptionDetails
?? $"Expected job \"{job.Id}\" \"{job.Description}\" to fail {(expectedCode.HasValue ? $"with ErrorCode \"{expectedCode.Value}\" " : String.Empty)}but it didn't");
@@ -958,7 +958,7 @@ namespace Tgstation.Server.Tests
foreach (var job in jobs)
{
Assert.IsTrue(job.StartedAt.Value >= preStartupTime);
await jrt.WaitForJob(job, 140, false, null, cancellationToken);
await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? (bool?)null : (bool?)false, null, cancellationToken);
}
var dd = await instanceClient.DreamDaemon.Read(cancellationToken);