Lot of origin tracking fixes

This commit is contained in:
Jordan Brown
2020-08-05 01:49:45 -04:00
parent 63572f10ad
commit 90e8940177
4 changed files with 271 additions and 223 deletions
+247 -221
View File
@@ -16,6 +16,7 @@ using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Components.Watchdog;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.Models;
@@ -162,13 +163,209 @@ namespace Tgstation.Server.Host.Components
}
}
Task RepositoryAutoUpdateJob(
IInstanceCore core,
IDatabaseContextFactory databaseContextFactory,
Job job,
Action<int> progressReporter,
CancellationToken cancellationToken)
=> databaseContextFactory.UseContext(
async databaseContext =>
{
if (core != this)
throw new InvalidOperationException(DifferentCoreExceptionMessage);
// assume 5 steps with synchronize
const int ProgressSections = 7;
const int ProgressStep = 100 / ProgressSections;
var repositorySettingsTask = databaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.FirstAsync(cancellationToken);
const int NumSteps = 3;
var doneSteps = 0;
Action<int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return progress => progressReporter((progress + (100 * tmpDoneSteps)) / NumSteps);
}
using var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false);
if (repo == null)
{
logger.LogTrace("Aborting repo update, no repository!");
return;
}
var startSha = repo.Head;
if (!repo.Tracking)
{
logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!");
return;
}
var repositorySettings = await repositorySettingsTask.ConfigureAwait(false);
// the main point of auto update is to pull the remote
await repo.FetchOrigin(
repositorySettings.AccessUser,
repositorySettings.AccessToken,
NextProgressReporter(),
cancellationToken)
.ConfigureAwait(false);
RevisionInformation currentRevInfo = null;
Task<RevisionInformation> LoadRevInfo() => databaseContext.RevisionInformations
.AsQueryable()
.Where(x => x.CommitSha == startSha && x.Instance.Id == metadata.Id)
.Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge)
.FirstOrDefaultAsync(cancellationToken);
var hasDbChanges = false;
// take appropriate auto update actions
var shouldSyncTracked = false;
var currentRevInfoTask = LoadRevInfo();
var result = await repo.MergeOrigin(
repositorySettings.CommitterName,
repositorySettings.CommitterEmail,
NextProgressReporter(),
cancellationToken)
.ConfigureAwait(false);
async Task UpdateRevInfo(string currentHead, bool onOrigin, IEnumerable<RevInfoTestMerge> updatedTestMerges)
{
if (currentRevInfo == null)
currentRevInfo = await LoadRevInfo().ConfigureAwait(false);
if (currentRevInfo == default)
{
logger.LogInformation(Repository.Repository.OriginTrackingErrorTemplate, currentHead);
onOrigin = true;
}
var attachedInstance = new Models.Instance
{
Id = metadata.Id
};
var oldRevInfo = currentRevInfo;
currentRevInfo = new RevisionInformation
{
CommitSha = currentHead,
OriginCommitSha = onOrigin
? currentHead
: await repo.GetOriginSha(cancellationToken).ConfigureAwait(false),
Instance = attachedInstance
};
if (!onOrigin)
currentRevInfo.ActiveTestMerges = new List<RevInfoTestMerge>(
updatedTestMerges ?? oldRevInfo.ActiveTestMerges);
databaseContext.Instances.Attach(attachedInstance);
databaseContext.RevisionInformations.Add(currentRevInfo);
hasDbChanges = true;
}
var preserveTestMerges = repositorySettings.AutoUpdatesKeepTestMerges.Value;
if (result.HasValue)
{
currentRevInfo = await currentRevInfoTask.ConfigureAwait(false);
var updatedTestMerges = await RemoveMergedPullRequests(
repo,
repositorySettings,
currentRevInfo,
cancellationToken)
.ConfigureAwait(false);
if (updatedTestMerges.Count == 0)
{
logger.LogTrace("All test merges have been merged on remote");
preserveTestMerges = false;
}
else
{
var lastRevInfoWasOriginCommit =
currentRevInfo == default
|| currentRevInfo.CommitSha == currentRevInfo.OriginCommitSha;
var stillOnOrigin = result.Value && lastRevInfoWasOriginCommit;
var currentHead = repo.Head;
if (currentHead != startSha)
{
await UpdateRevInfo(currentHead, stillOnOrigin, updatedTestMerges).ConfigureAwait(false);
shouldSyncTracked = stillOnOrigin;
}
}
}
else if (preserveTestMerges)
throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict);
if (!preserveTestMerges)
{
logger.LogTrace("Resetting to origin...");
await repo.ResetToOrigin(NextProgressReporter(), cancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
currentRevInfo = await databaseContext.RevisionInformations
.AsQueryable()
.Where(x => x.CommitSha == currentHead && x.Instance.Id == metadata.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (currentHead != startSha && currentRevInfo == default)
await UpdateRevInfo(currentHead, true, null).ConfigureAwait(false);
shouldSyncTracked = true;
}
// synch if necessary
if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head)
{
var pushedOrigin = await repo.Sychronize(
repositorySettings.AccessUser,
repositorySettings.AccessToken,
repositorySettings.CommitterName,
repositorySettings.CommitterEmail,
NextProgressReporter(),
shouldSyncTracked,
cancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
if (currentHead != currentRevInfo.CommitSha)
await UpdateRevInfo(currentHead, pushedOrigin, null).ConfigureAwait(false);
}
if (hasDbChanges)
try
{
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
}
catch
{
// DCT: Cancellation token is for job, operation must run regardless
await repo.ResetToSha(startSha, progressReporter, default).ConfigureAwait(false);
throw;
}
progressReporter(5 * ProgressStep);
});
/// <summary>
/// Pull the repository and compile for every set of given <paramref name="minutes"/>
/// </summary>
/// <param name="minutes">How many minutes the operation should repeat. Does not include running time</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
#pragma warning disable CA1502 // TODO: Decomplexify
#pragma warning disable CA1502 // TODO: Decomplexify
async Task TimerLoop(uint minutes, CancellationToken cancellationToken)
{
logger.LogDebug("Entering auto-update loop");
@@ -191,236 +388,59 @@ namespace Tgstation.Server.Host.Components
CancelRight = (ulong)RepositoryRights.CancelPendingChanges
};
string deploySha = null;
await jobManager.RegisterOperation(repositoryUpdateJob, async (core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken) =>
{
if (core != this)
throw new InvalidOperationException(DifferentCoreExceptionMessage);
// assume 5 steps with synchronize
const int ProgressSections = 7;
const int ProgressStep = 100 / ProgressSections;
string repoHead = null;
await databaseContextFactory.UseContext(
async databaseContext =>
{
var repositorySettingsTask = databaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.FirstAsync(jobCancellationToken);
const int NumSteps = 3;
var doneSteps = 0;
Action<int> NextProgressReporter()
{
var tmpDoneSteps = doneSteps;
++doneSteps;
return progress => progressReporter((progress + (100 * tmpDoneSteps)) / NumSteps);
}
using var repo = await RepositoryManager.LoadRepository(jobCancellationToken).ConfigureAwait(false);
if (repo == null)
{
logger.LogTrace("Aborting repo update, no repository!");
return;
}
var startSha = repo.Head;
if (!repo.Tracking)
{
logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!");
deploySha = startSha;
return;
}
var repositorySettings = await repositorySettingsTask.ConfigureAwait(false);
// the main point of auto update is to pull the remote
await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
RevisionInformation currentRevInfo = null;
bool hasDbChanges = false;
Task<RevisionInformation> LoadRevInfo() => databaseContext.RevisionInformations
.AsQueryable()
.Where(x => x.CommitSha == startSha && x.Instance.Id == metadata.Id)
.Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge)
.FirstOrDefaultAsync(jobCancellationToken);
async Task UpdateRevInfo(string currentHead, bool onOrigin, IEnumerable<RevInfoTestMerge> updatedTestMerges)
{
if (currentRevInfo == null)
currentRevInfo = await LoadRevInfo().ConfigureAwait(false);
if (currentRevInfo == default)
{
logger.LogInformation(Repository.Repository.OriginTrackingErrorTemplate, currentHead);
onOrigin = true;
}
var attachedInstance = new Models.Instance
{
Id = metadata.Id
};
var oldRevInfo = currentRevInfo;
currentRevInfo = new RevisionInformation
{
CommitSha = currentHead,
OriginCommitSha = onOrigin ? currentHead : oldRevInfo.OriginCommitSha,
Instance = attachedInstance
};
if (!onOrigin)
currentRevInfo.ActiveTestMerges = new List<RevInfoTestMerge>(
updatedTestMerges ?? oldRevInfo.ActiveTestMerges);
databaseContext.Instances.Attach(attachedInstance);
databaseContext.RevisionInformations.Add(currentRevInfo);
hasDbChanges = true;
}
// take appropriate auto update actions
bool shouldSyncTracked = false;
bool preserveTestMerges = repositorySettings.AutoUpdatesKeepTestMerges.Value;
if (preserveTestMerges)
{
logger.LogTrace("Preserving test merges...");
var currentRevInfoTask = LoadRevInfo();
var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
if (!result.HasValue)
throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict);
currentRevInfo = await currentRevInfoTask.ConfigureAwait(false);
var updatedTestMerges = await RemoveMergedPullRequests(
repo,
repositorySettings,
currentRevInfo,
cancellationToken)
.ConfigureAwait(false);
if (updatedTestMerges.Count == 0)
{
logger.LogTrace("All test merges have been merged on remote");
preserveTestMerges = false;
}
else
{
var lastRevInfoWasOriginCommit =
currentRevInfo == default
|| currentRevInfo.CommitSha == currentRevInfo.OriginCommitSha;
var stillOnOrigin = result.Value && lastRevInfoWasOriginCommit;
var currentHead = repo.Head;
if (currentHead != startSha)
{
await UpdateRevInfo(currentHead, stillOnOrigin, updatedTestMerges).ConfigureAwait(false);
shouldSyncTracked = stillOnOrigin;
}
else
shouldSyncTracked = false;
}
}
if (!preserveTestMerges)
{
logger.LogTrace("Resetting to origin...");
await repo.ResetToOrigin(NextProgressReporter(), jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
currentRevInfo = await databaseContext.RevisionInformations
.AsQueryable()
.Where(x => x.CommitSha == currentHead && x.Instance.Id == metadata.Id)
.FirstOrDefaultAsync(jobCancellationToken)
.ConfigureAwait(false);
if (currentHead != startSha && currentRevInfo == default)
await UpdateRevInfo(currentHead, true, null).ConfigureAwait(false);
shouldSyncTracked = true;
}
// synch if necessary
if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head)
{
var pushedOrigin = await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), shouldSyncTracked, jobCancellationToken).ConfigureAwait(false);
var currentHead = repo.Head;
if (currentHead != currentRevInfo.CommitSha)
await UpdateRevInfo(currentHead, pushedOrigin, null).ConfigureAwait(false);
}
repoHead = repo.Head;
if (hasDbChanges)
try
{
await databaseContext.Save(jobCancellationToken).ConfigureAwait(false);
}
catch
{
// DCT: Cancellation token is for job, operation must run regardless
await repo.ResetToSha(startSha, progressReporter, default).ConfigureAwait(false);
throw;
}
})
await jobManager.RegisterOperation(
repositoryUpdateJob,
RepositoryAutoUpdateJob,
cancellationToken)
.ConfigureAwait(false);
progressReporter(5 * ProgressStep);
deploySha = repoHead;
}, cancellationToken).ConfigureAwait(false);
// DCT: First token will cancel the job, second is for cancelling the cancellation, unwanted
await jobManager.WaitForJobCompletion(repositoryUpdateJob, null, cancellationToken, default).ConfigureAwait(false);
if (deploySha == null)
Job compileProcessJob;
using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false))
{
logger.LogTrace("Aborting auto update, repository error!");
continue;
}
if(deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
{
logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
continue;
}
// finally set up the job
var compileProcessJob = new Job
{
Instance = repositoryUpdateJob.Instance,
Description = "Scheduled code deployment",
CancelRightsType = RightsType.DreamMaker,
CancelRight = (ulong)DreamMakerRights.CancelCompile
};
await jobManager.RegisterOperation(
compileProcessJob,
(core, databaseContextFactory, job, progressReporter, jobCancellationToken) =>
var deploySha = repo.Head;
if (deploySha == null)
{
if (core != this)
throw new InvalidOperationException(DifferentCoreExceptionMessage);
return DreamMaker.DeploymentProcess(
job,
databaseContextFactory,
progressReporter,
jobCancellationToken);
},
cancellationToken).ConfigureAwait(false);
logger.LogTrace("Aborting auto update, repository error!");
continue;
}
if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
{
logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
continue;
}
// finally set up the job
compileProcessJob = new Job
{
Instance = repositoryUpdateJob.Instance,
Description = "Scheduled code deployment",
CancelRightsType = RightsType.DreamMaker,
CancelRight = (ulong)DreamMakerRights.CancelCompile
};
await jobManager.RegisterOperation(
compileProcessJob,
(core, databaseContextFactory, job, progressReporter, jobCancellationToken) =>
{
if (core != this)
throw new InvalidOperationException(DifferentCoreExceptionMessage);
return DreamMaker.DeploymentProcess(
job,
databaseContextFactory,
progressReporter,
jobCancellationToken);
},
cancellationToken)
.ConfigureAwait(false);
}
await jobManager.WaitForJobCompletion(compileProcessJob, null, default, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
logger.LogDebug("Cancelled auto update job!");
throw;
}
catch (Exception e)
catch (Exception e) when (!(e is OperationCanceledException))
{
logger.LogWarning(e, "Error in auto update loop!");
continue;
@@ -428,6 +448,7 @@ namespace Tgstation.Server.Host.Components
}
catch (OperationCanceledException)
{
logger.LogDebug("Cancelled auto update loop!");
break;
}
@@ -478,6 +499,7 @@ namespace Tgstation.Server.Host.Components
var newList = revisionInformation.ActiveTestMerges.ToList();
PullRequest lastMerged = null;
async Task CheckRemovePR(Task<PullRequest> task)
{
var pr = await task.ConfigureAwait(false);
@@ -486,9 +508,13 @@ namespace Tgstation.Server.Host.Components
// We don't just assume, actually check the repo contains the merge commit.
if (await repository.ShaIsParent(pr.MergeCommitSha, cancellationToken).ConfigureAwait(false))
{
if (lastMerged == null || lastMerged.MergedAt < pr.MergedAt)
lastMerged = pr;
newList.Remove(
newList.First(
potential => potential.TestMerge.Number == pr.Number));
}
}
foreach (var prTask in tasks)
@@ -141,5 +141,12 @@ namespace Tgstation.Server.Host.Components.Repository
/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if <paramref name="sha"/> is a parent of <see cref="Head"/>, <see langword="false"/> otherwise.</returns>
/// <remarks>This function is NOT reentrant.</remarks>
Task<bool> ShaIsParent(string sha, CancellationToken cancellationToken);
/// <summary>
/// Get the tracked reference's current SHA.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in the tracked origin reference's SHA.</returns>
Task<string> GetOriginSha(CancellationToken cancellationToken);
}
}
@@ -541,6 +541,17 @@ namespace Tgstation.Server.Host.Components.Repository
await ioMananger.CopyDirectory(ioMananger.ResolvePath(), path, new List<string> { ".git" }, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public Task<string> GetOriginSha(CancellationToken cancellationToken) => Task.Factory.StartNew(() =>
{
if (!Tracking)
throw new JobException(ErrorCode.RepoReferenceRequired);
cancellationToken.ThrowIfCancellationRequested();
return libGitRepo.Head.TrackedBranch.Tip.Sha;
}, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current);
/// <inheritdoc />
public async Task<bool?> MergeOrigin(string committerName, string committerEmail, Action<int> progressReporter, CancellationToken cancellationToken)
{
@@ -563,7 +574,11 @@ namespace Tgstation.Server.Host.Components.Repository
cancellationToken.ThrowIfCancellationRequested();
trackedBranch = libGitRepo.Head.TrackedBranch;
logger.LogDebug("Merge origin/{2}: <{0} ({1})>", committerName, committerEmail, trackedBranch.FriendlyName);
logger.LogDebug(
"Merge origin/{0}: <{1} ({2})>",
trackedBranch.FriendlyName,
committerName,
committerEmail);
result = libGitRepo.Merge(trackedBranch, new Signature(committerName, committerEmail, DateTimeOffset.Now), new MergeOptions
{
CommitOnSuccess = true,
@@ -601,10 +601,10 @@ namespace Tgstation.Server.Host.Controllers
var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false);
if (!fastForward.HasValue)
throw new JobException(ErrorCode.RepoMergeConflict);
lastRevisionInfo.OriginCommitSha = await repo.GetOriginSha(cancellationToken).ConfigureAwait(false);
await UpdateRevInfo().ConfigureAwait(false);
if (fastForward.Value)
{
lastRevisionInfo.OriginCommitSha = repo.Head;
await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false);
postUpdateSha = repo.Head;
}