Merge pull request #1923 from tgstation/SkipDMAPIValidation [APIDeploy][NugetDeploy]

Allow DMAPI Validation to be fully skipped
This commit is contained in:
Jordan Dominion
2024-09-08 17:12:00 -04:00
committed by GitHub
29 changed files with 4892 additions and 120 deletions
+3 -3
View File
@@ -5,10 +5,10 @@
<PropertyGroup>
<TgsCoreVersion>6.10.0</TgsCoreVersion>
<TgsConfigVersion>5.2.0</TgsConfigVersion>
<TgsApiVersion>10.8.0</TgsApiVersion>
<TgsApiVersion>10.9.0</TgsApiVersion>
<TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
<TgsApiLibraryVersion>14.0.0</TgsApiLibraryVersion>
<TgsClientVersion>17.0.0</TgsClientVersion>
<TgsApiLibraryVersion>14.1.0</TgsApiLibraryVersion>
<TgsClientVersion>17.1.0</TgsClientVersion>
<TgsDmapiVersion>7.3.0</TgsDmapiVersion>
<TgsInteropVersion>5.10.0</TgsInteropVersion>
<TgsHostWatchdogVersion>1.5.0</TgsHostWatchdogVersion>
@@ -0,0 +1,23 @@
namespace Tgstation.Server.Api.Models
{
/// <summary>
/// The DMAPI validation setting for deployments.
/// </summary>
public enum DMApiValidationMode
{
/// <summary>
/// DMAPI validation is performed but not required for the deployment to succeed.
/// </summary>
Optional,
/// <summary>
/// DMAPI validation must suceed for the deployment to succeed.
/// </summary>
Required,
/// <summary>
/// DMAPI validation will not be performed and no DMAPI features will be available in the deployment.
/// </summary>
Skipped,
}
}
@@ -1,5 +1,6 @@
using System;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace Tgstation.Server.Api.Models.Internal
{
@@ -29,11 +30,19 @@ namespace Tgstation.Server.Api.Models.Internal
public DreamDaemonSecurity? ApiValidationSecurityLevel { get; set; }
/// <summary>
/// If API validation should be required for a deployment to succeed.
/// If API validation should be required for a deployment to succeed. Must not be set on mutation if <see cref="DMApiValidationMode"/> is set.
/// </summary>
[Required]
[NotMapped]
[Obsolete($"Use {nameof(DMApiValidationMode)} instead.")]
public bool? RequireDMApiValidation { get; set; }
/// <summary>
/// The current <see cref="Models.DMApiValidationMode"/>. Must not be set on mutation if <see cref="RequireDMApiValidation"/> is set.
/// </summary>
[Required]
public DMApiValidationMode? DMApiValidationMode { get; set; }
/// <summary>
/// Amount of time before an in-progress deployment is cancelled.
/// </summary>
@@ -49,7 +49,7 @@ namespace Tgstation.Server.Api.Rights
SetSecurityLevel = 1 << 6,
/// <summary>
/// User may modify <see cref="Models.Internal.DreamMakerSettings.RequireDMApiValidation"/>.
/// User may modify <see cref="Models.Internal.DreamMakerSettings.DMApiValidationMode"/> and <see cref="Models.Internal.DreamMakerSettings.RequireDMApiValidation"/>.
/// </summary>
SetApiValidationRequirement = 1 << 7,
@@ -648,14 +648,14 @@ namespace Tgstation.Server.Host.Components.Deployment
ErrorCode.DeploymentExitCode,
new JobException($"Compilation failed:{Environment.NewLine}{Environment.NewLine}{job.Output}"));
progressReporter.StageName = "Validating DMAPI";
await VerifyApi(
launchParameters.StartupTimeout!.Value,
dreamMakerSettings.ApiValidationSecurityLevel!.Value,
job,
progressReporter,
engineLock,
dreamMakerSettings.ApiValidationPort!.Value,
dreamMakerSettings.RequireDMApiValidation!.Value,
dreamMakerSettings.DMApiValidationMode!.Value,
launchParameters.LogOutput!.Value,
cancellationToken);
}
@@ -767,9 +767,10 @@ namespace Tgstation.Server.Host.Components.Deployment
/// <param name="timeout">The timeout in seconds for validation.</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API.</param>
/// <param name="job">The <see cref="CompileJob"/> for the operation.</param>
/// <param name="progressReporter">The <see cref="JobProgressReporter"/>.</param>
/// <param name="engineLock">The current <see cref="IEngineExecutableLock"/>.</param>
/// <param name="portToUse">The port to use for API validation.</param>
/// <param name="requireValidate">If the API validation is required to complete the deployment.</param>
/// <param name="validationMode">The <see cref="DMApiValidationMode"/>.</param>
/// <param name="logOutput">If output should be logged to the DreamDaemon Diagnostics folder.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
@@ -777,12 +778,23 @@ namespace Tgstation.Server.Host.Components.Deployment
uint timeout,
DreamDaemonSecurity securityLevel,
Models.CompileJob job,
JobProgressReporter progressReporter,
IEngineExecutableLock engineLock,
ushort portToUse,
bool requireValidate,
DMApiValidationMode validationMode,
bool logOutput,
CancellationToken cancellationToken)
{
if (validationMode == DMApiValidationMode.Skipped)
{
logger.LogDebug("Skipping DMAPI validation");
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
return;
}
progressReporter.StageName = "Validating DMAPI";
var requireValidate = validationMode == DMApiValidationMode.Required;
logger.LogTrace("Verifying {possiblyRequired}DMAPI...", requireValidate ? "required " : String.Empty);
var launchParameters = new DreamDaemonLaunchParameters
{
@@ -236,12 +236,28 @@ namespace Tgstation.Server.Host.Controllers
hostModel.ApiValidationSecurityLevel = model.ApiValidationSecurityLevel;
}
if (model.RequireDMApiValidation.HasValue)
#pragma warning disable CS0618 // Type or member is obsolete
bool? legacyRequireDMApiValidation = model.RequireDMApiValidation;
#pragma warning restore CS0618 // Type or member is obsolete
if (legacyRequireDMApiValidation.HasValue)
{
if (!dreamMakerRights.HasFlag(DreamMakerRights.SetApiValidationRequirement))
return Forbid();
hostModel.RequireDMApiValidation = model.RequireDMApiValidation;
hostModel.DMApiValidationMode = legacyRequireDMApiValidation.Value
? DMApiValidationMode.Required
: DMApiValidationMode.Optional;
}
if (model.DMApiValidationMode.HasValue)
{
if (legacyRequireDMApiValidation.HasValue)
return BadRequest(new ErrorMessageResponse(ErrorCode.ModelValidationFailure));
if (!dreamMakerRights.HasFlag(DreamMakerRights.SetApiValidationRequirement))
return Forbid();
hostModel.DMApiValidationMode = model.DMApiValidationMode;
}
if (model.Timeout.HasValue)
@@ -760,7 +760,7 @@ namespace Tgstation.Server.Host.Controllers
{
ApiValidationPort = dmPort,
ApiValidationSecurityLevel = DreamDaemonSecurity.Safe,
RequireDMApiValidation = true,
DMApiValidationMode = DMApiValidationMode.Required,
Timeout = TimeSpan.FromHours(1),
CompilerAdditionalArguments = null,
},
@@ -318,6 +318,63 @@ namespace Tgstation.Server.Host.Database
return wasEmpty;
}
/// <inheritdoc />
public async ValueTask SchemaDowngradeForServerVersion(
ILogger<DatabaseContext> logger,
Version targetVersion,
DatabaseType currentDatabaseType,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(logger);
ArgumentNullException.ThrowIfNull(targetVersion);
if (targetVersion < new Version(4, 0))
throw new ArgumentOutOfRangeException(nameof(targetVersion), targetVersion, "Cannot migrate below version 4.0.0!");
if (currentDatabaseType == DatabaseType.PostgresSql && targetVersion < new Version(4, 3, 0))
throw new NotSupportedException("Cannot migrate below version 4.3.0 with PostgresSql!");
if (currentDatabaseType == DatabaseType.MariaDB)
currentDatabaseType = DatabaseType.MySql; // Keeping switch expressions while avoiding `or` syntax from C#9
if (targetVersion < new Version(4, 1, 0))
throw new NotSupportedException("Cannot migrate below version 4.1.0!");
var targetMigration = GetTargetMigration(targetVersion, currentDatabaseType);
if (targetMigration == null)
{
logger.LogDebug("No down migration required.");
return;
}
// already setup
var migrationSubstitution = currentDatabaseType switch
{
DatabaseType.SqlServer => null, // already setup
DatabaseType.MySql => "MY{0}",
DatabaseType.Sqlite => "SL{0}",
DatabaseType.PostgresSql => "PG{0}",
_ => throw new InvalidOperationException($"Invalid DatabaseType: {currentDatabaseType}"),
};
if (migrationSubstitution != null)
targetMigration = String.Format(CultureInfo.InvariantCulture, migrationSubstitution, targetMigration[2..]);
// even though it clearly implements it in the DatabaseFacade definition this won't work without casting (╯ಠ益ಠ)╯︵ ┻━┻
var dbServiceProvider = ((IInfrastructure<IServiceProvider>)Database).Instance;
var migrator = dbServiceProvider.GetRequiredService<IMigrator>();
logger.LogInformation("Migrating down to version {targetVersion}. Target: {targetMigration}", targetVersion, targetMigration);
try
{
await migrator.MigrateAsync(targetMigration, cancellationToken);
}
catch (Exception e)
{
logger.LogCritical(e, "Failed to migrate!");
}
}
/// <inheritdoc />
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
@@ -393,45 +450,31 @@ namespace Tgstation.Server.Host.Database
/// <summary>
/// Used by unit tests to remind us to setup the correct MSSQL migration downgrades.
/// </summary>
internal static readonly Type MSLatestMigration = typeof(MSAddOpenDreamTopicPort);
internal static readonly Type MSLatestMigration = typeof(MSAddDMApiValidationMode);
/// <summary>
/// Used by unit tests to remind us to setup the correct MYSQL migration downgrades.
/// </summary>
internal static readonly Type MYLatestMigration = typeof(MYAddOpenDreamTopicPort);
internal static readonly Type MYLatestMigration = typeof(MYAddDMApiValidationMode);
/// <summary>
/// Used by unit tests to remind us to setup the correct PostgresSQL migration downgrades.
/// </summary>
internal static readonly Type PGLatestMigration = typeof(PGAddOpenDreamTopicPort);
internal static readonly Type PGLatestMigration = typeof(PGAddDMApiValidationMode);
/// <summary>
/// Used by unit tests to remind us to setup the correct SQLite migration downgrades.
/// </summary>
internal static readonly Type SLLatestMigration = typeof(SLAddOpenDreamTopicPort);
internal static readonly Type SLLatestMigration = typeof(SLAddDMApiValidationMode);
/// <inheritdoc />
#pragma warning disable CA1502 // Cyclomatic complexity
public async ValueTask SchemaDowngradeForServerVersion(
ILogger<DatabaseContext> logger,
Version targetVersion,
DatabaseType currentDatabaseType,
CancellationToken cancellationToken)
/// <summary>
/// Gets the name of the migration to run for migrating down to a given <paramref name="targetVersion"/> for the <paramref name="currentDatabaseType"/>.
/// </summary>
/// <param name="targetVersion">The <see cref="Version"/> TGS is being migratied down to.</param>
/// <param name="currentDatabaseType">The currently running <see cref="DatabaseType"/>.</param>
/// <returns>The name of the migration to run on success, <see langword="null"/> otherwise.</returns>
private string? GetTargetMigration(Version targetVersion, DatabaseType currentDatabaseType)
{
ArgumentNullException.ThrowIfNull(logger);
ArgumentNullException.ThrowIfNull(targetVersion);
if (targetVersion < new Version(4, 0))
throw new ArgumentOutOfRangeException(nameof(targetVersion), targetVersion, "Cannot migrate below version 4.0.0!");
if (currentDatabaseType == DatabaseType.PostgresSql && targetVersion < new Version(4, 3, 0))
throw new NotSupportedException("Cannot migrate below version 4.3.0 with PostgresSql!");
if (currentDatabaseType == DatabaseType.MariaDB)
currentDatabaseType = DatabaseType.MySql; // Keeping switch expressions while avoiding `or` syntax from C#9
if (targetVersion < new Version(4, 1, 0))
throw new NotSupportedException("Cannot migrate below version 4.1.0!");
// Update this with new migrations as they are made
string? targetMigration = null;
@@ -603,42 +646,11 @@ namespace Tgstation.Server.Host.Database
DatabaseType.Sqlite => nameof(SLRemoveSoftColumns),
_ => BadDatabaseType(),
};
if (targetVersion < new Version(4, 2, 0))
targetMigration = currentDatabaseType == DatabaseType.Sqlite ? nameof(SLRebuild) : nameof(MSFixCascadingDelete);
if (targetMigration == null)
{
logger.LogDebug("No down migration required.");
return;
}
// already setup
var migrationSubstitution = currentDatabaseType switch
{
DatabaseType.SqlServer => null, // already setup
DatabaseType.MySql => "MY{0}",
DatabaseType.Sqlite => "SL{0}",
DatabaseType.PostgresSql => "PG{0}",
_ => throw new InvalidOperationException($"Invalid DatabaseType: {currentDatabaseType}"),
};
if (migrationSubstitution != null)
targetMigration = String.Format(CultureInfo.InvariantCulture, migrationSubstitution, targetMigration[2..]);
// even though it clearly implements it in the DatabaseFacade definition this won't work without casting (╯ಠ益ಠ)╯︵ ┻━┻
var dbServiceProvider = ((IInfrastructure<IServiceProvider>)Database).Instance;
var migrator = dbServiceProvider.GetRequiredService<IMigrator>();
logger.LogInformation("Migrating down to version {targetVersion}. Target: {targetMigration}", targetVersion, targetMigration);
try
{
await migrator.MigrateAsync(targetMigration, cancellationToken);
}
catch (Exception e)
{
logger.LogCritical(e, "Failed to migrate!");
}
return targetMigration;
}
#pragma warning restore CA1502 // Cyclomatic complexity
}
}
@@ -0,0 +1,49 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace Tgstation.Server.Host.Database.Migrations
{
/// <inheritdoc />
public partial class MSAddDMApiValidationMode : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<int>(
name: "DMApiValidationMode",
table: "DreamMakerSettings",
type: "int",
nullable: false,
defaultValue: 0);
migrationBuilder.Sql("UPDATE DreamMakerSettings SET DMApiValidationMode = RequireDMApiValidation");
migrationBuilder.DropColumn(
name: "RequireDMApiValidation",
table: "DreamMakerSettings");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<bool>(
name: "RequireDMApiValidation",
table: "DreamMakerSettings",
type: "bit",
nullable: false,
defaultValue: false);
migrationBuilder.Sql("UPDATE DreamMakerSettings SET RequireDMApiValidation = 1 WHERE DMApiValidationMode != 0");
migrationBuilder.Sql("UPDATE DreamMakerSettings SET RequireDMApiValidation = 0 WHERE DMApiValidationMode = 0");
migrationBuilder.DropColumn(
name: "DMApiValidationMode",
table: "DreamMakerSettings");
}
}
}
@@ -0,0 +1,49 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace Tgstation.Server.Host.Database.Migrations
{
/// <inheritdoc />
public partial class MYAddDMApiValidationMode : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<int>(
name: "DMApiValidationMode",
table: "DreamMakerSettings",
type: "int",
nullable: false,
defaultValue: 0);
migrationBuilder.Sql("UPDATE DreamMakerSettings SET DMApiValidationMode = RequireDMApiValidation");
migrationBuilder.DropColumn(
name: "RequireDMApiValidation",
table: "DreamMakerSettings");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<bool>(
name: "RequireDMApiValidation",
table: "DreamMakerSettings",
type: "tinyint(1)",
nullable: false,
defaultValue: false);
migrationBuilder.Sql("UPDATE DreamMakerSettings SET RequireDMApiValidation = 1 WHERE DMApiValidationMode != 0");
migrationBuilder.Sql("UPDATE DreamMakerSettings SET RequireDMApiValidation = 0 WHERE DMApiValidationMode = 0");
migrationBuilder.DropColumn(
name: "DMApiValidationMode",
table: "DreamMakerSettings");
}
}
}
@@ -0,0 +1,49 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace Tgstation.Server.Host.Database.Migrations
{
/// <inheritdoc />
public partial class PGAddDMApiValidationMode : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<int>(
name: "DMApiValidationMode",
table: "DreamMakerSettings",
type: "integer",
nullable: false,
defaultValue: 0);
migrationBuilder.Sql("UPDATE \"DreamMakerSettings\" SET \"DMApiValidationMode\" = \"RequireDMApiValidation\"::int");
migrationBuilder.DropColumn(
name: "RequireDMApiValidation",
table: "DreamMakerSettings");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.AddColumn<bool>(
name: "RequireDMApiValidation",
table: "DreamMakerSettings",
type: "boolean",
nullable: false,
defaultValue: false);
migrationBuilder.Sql("UPDATE \"DreamMakerSettings\" SET \"RequireDMApiValidation\" = true WHERE \"DMApiValidationMode\" != 0");
migrationBuilder.Sql("UPDATE \"DreamMakerSettings\" SET \"RequireDMApiValidation\" = false WHERE \"DMApiValidationMode\" = 0");
migrationBuilder.DropColumn(
name: "DMApiValidationMode",
table: "DreamMakerSettings");
}
}
}
@@ -0,0 +1,34 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace Tgstation.Server.Host.Database.Migrations
{
/// <inheritdoc />
public partial class SLAddDMApiValidationMode : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.RenameColumn(
name: "RequireDMApiValidation",
table: "DreamMakerSettings",
newName: "DMApiValidationMode");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
ArgumentNullException.ThrowIfNull(migrationBuilder);
migrationBuilder.RenameColumn(
name: "DMApiValidationMode",
table: "DreamMakerSettings",
newName: "RequireDMApiValidation");
migrationBuilder.Sql("UPDATE DreamMakerSettings SET RequireDMApiValidation = 0 WHERE RequireDMApiValidation = 2");
}
}
}
@@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Database.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.7")
.HasAnnotation("ProductVersion", "8.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 64);
MySqlModelBuilderExtensions.AutoIncrementColumns(modelBuilder);
@@ -286,6 +286,9 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("varchar(10000)");
b.Property<int>("DMApiValidationMode")
.HasColumnType("int");
b.Property<long>("InstanceId")
.HasColumnType("bigint");
@@ -295,10 +298,6 @@ namespace Tgstation.Server.Host.Database.Migrations
MySqlPropertyBuilderExtensions.HasCharSet(b.Property<string>("ProjectName"), "utf8mb4");
b.Property<bool?>("RequireDMApiValidation")
.IsRequired()
.HasColumnType("tinyint(1)");
b.Property<TimeSpan?>("Timeout")
.IsRequired()
.HasColumnType("time(6)");
@@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Database.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.7")
.HasAnnotation("ProductVersion", "8.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@@ -261,6 +261,9 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("character varying(10000)");
b.Property<int>("DMApiValidationMode")
.HasColumnType("integer");
b.Property<long>("InstanceId")
.HasColumnType("bigint");
@@ -268,10 +271,6 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("character varying(10000)");
b.Property<bool?>("RequireDMApiValidation")
.IsRequired()
.HasColumnType("boolean");
b.Property<TimeSpan?>("Timeout")
.IsRequired()
.HasColumnType("interval");
@@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Database.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.7")
.HasAnnotation("ProductVersion", "8.0.8")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
@@ -263,6 +263,9 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("nvarchar(max)");
b.Property<int>("DMApiValidationMode")
.HasColumnType("int");
b.Property<long>("InstanceId")
.HasColumnType("bigint");
@@ -270,10 +273,6 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("nvarchar(max)");
b.Property<bool?>("RequireDMApiValidation")
.IsRequired()
.HasColumnType("bit");
b.Property<TimeSpan?>("Timeout")
.IsRequired()
.HasColumnType("time");
@@ -12,7 +12,7 @@ namespace Tgstation.Server.Host.Database.Migrations
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.7");
modelBuilder.HasAnnotation("ProductVersion", "8.0.8");
modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b =>
{
@@ -256,6 +256,9 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("TEXT");
b.Property<int>("DMApiValidationMode")
.HasColumnType("INTEGER");
b.Property<long>("InstanceId")
.HasColumnType("INTEGER");
@@ -263,10 +266,6 @@ namespace Tgstation.Server.Host.Database.Migrations
.HasMaxLength(10000)
.HasColumnType("TEXT");
b.Property<bool?>("RequireDMApiValidation")
.IsRequired()
.HasColumnType("INTEGER");
b.Property<TimeSpan?>("Timeout")
.IsRequired()
.HasColumnType("TEXT");
@@ -29,7 +29,10 @@ namespace Tgstation.Server.Host.Models
ProjectName = ProjectName,
ApiValidationPort = ApiValidationPort,
ApiValidationSecurityLevel = ApiValidationSecurityLevel,
RequireDMApiValidation = RequireDMApiValidation,
#pragma warning disable CS0618 // Type or member is obsolete
RequireDMApiValidation = DMApiValidationMode == Api.Models.DMApiValidationMode.Required,
#pragma warning restore CS0618 // Type or member is obsolete
DMApiValidationMode = DMApiValidationMode,
Timeout = Timeout,
CompilerAdditionalArguments = CompilerAdditionalArguments,
};
+1 -1
View File
@@ -1,3 +1,3 @@
#define TGS_INFO_LOG(message) world.log << "Info: [##message]"
#define TGS_ERROR_LOG(message) world.log << "Err: [##message]"
#define TGS_ERROR_LOG(message) TgsError("[##message]")
#define TGS_V3_API
+15
View File
@@ -10,6 +10,21 @@
set waitfor = FALSE
Run()
/proc/TgsError(message)
world.log << "TGS Error: [message]"
var/ibf = message == "Failed initial bridge request!"
if(ibf || message == "Failed bridge request, bad json: null" || findtext(message, "byond_world_export: Failed request: ") || message == "Failed to activate API!")
if(ibf)
world.log << "Writing test fail file..."
text2file("BRIDGE FAILED", "initial_bridge_failed.txt")
world.log << "File exists: [fexists("initial_bridge_failed.txt")]"
del(world)
sleep(1)
return
FailTest("DMAPI Error: [message]")
/proc/Run()
world.log << "sleep"
sleep(50)
@@ -0,0 +1,18 @@
// Hand crafted DME, will not work if saved with DreamMaker
// BEGIN_INTERNALS
// END_INTERNALS
// BEGIN_FILE_DIR
#define FILE_DIR .
// END_FILE_DIR
// BEGIN_PREFERENCES
// END_PREFERENCES
// BEGIN_INCLUDE
#include "Config.dm"
#undef TGS_V3_API
#include "../test_prelude.dm"
#include "Test.dm"
// END_INCLUDE
@@ -7,6 +7,7 @@ using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Api.Models.Request;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Api.Rights;
@@ -58,8 +59,69 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing);
var dmSettings = await dreamMakerClient.Read(cancellationToken);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(true, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
Assert.AreEqual(DMApiValidationMode.Required, dmSettings.DMApiValidationMode);
Assert.AreEqual(null, dmSettings.ProjectName);
// test legacy back and forth
dmSettings = await dreamMakerClient.Update(new DreamMakerRequest
{
DMApiValidationMode = DMApiValidationMode.Optional,
}, cancellationToken);
Assert.AreEqual(DMApiValidationMode.Optional, dmSettings.DMApiValidationMode);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(false, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
dmSettings = await dreamMakerClient.Update(new DreamMakerRequest
{
DMApiValidationMode = DMApiValidationMode.Skipped,
}, cancellationToken);
Assert.AreEqual(DMApiValidationMode.Skipped, dmSettings.DMApiValidationMode);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(false, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
dmSettings = await dreamMakerClient.Update(new DreamMakerRequest
{
DMApiValidationMode = DMApiValidationMode.Required,
}, cancellationToken);
Assert.AreEqual(DMApiValidationMode.Required, dmSettings.DMApiValidationMode);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(true, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
dmSettings = await dreamMakerClient.Update(new DreamMakerRequest
{
#pragma warning disable CS0618 // Type or member is obsolete
RequireDMApiValidation = false,
#pragma warning restore CS0618 // Type or member is obsolete
}, cancellationToken);
Assert.AreEqual(DMApiValidationMode.Optional, dmSettings.DMApiValidationMode);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(false, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
dmSettings = await dreamMakerClient.Update(new DreamMakerRequest
{
#pragma warning disable CS0618 // Type or member is obsolete
RequireDMApiValidation = true,
#pragma warning restore CS0618 // Type or member is obsolete
}, cancellationToken);
Assert.AreEqual(DMApiValidationMode.Required, dmSettings.DMApiValidationMode);
#pragma warning disable CS0618 // Type or member is obsolete
Assert.AreEqual(true, dmSettings.RequireDMApiValidation);
#pragma warning restore CS0618 // Type or member is obsolete
await ApiAssert.ThrowsException<ApiConflictException, DreamMakerResponse>(() => dreamMakerClient.Update(new DreamMakerRequest
{
#pragma warning disable CS0618 // Type or member is obsolete
RequireDMApiValidation = true,
DMApiValidationMode = DMApiValidationMode.Required,
#pragma warning restore CS0618 // Type or member is obsolete
}, cancellationToken), ErrorCode.ModelValidationFailure);
}
async ValueTask CheckDreamDaemonPriority(Task deploymentJobWaitTask, CancellationToken cancellationToken)
@@ -25,6 +25,7 @@ using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Api.Models.Request;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Client;
@@ -185,7 +186,14 @@ namespace Tgstation.Server.Tests.Live.Instance
ApiAssert.ThrowsException<ConflictException, JobResponse>(() => instanceClient.DreamDaemon.CreateDump(cancellationToken), ErrorCode.WatchdogNotRunning).AsTask(),
ApiAssert.ThrowsException<ConflictException, JobResponse>(() => instanceClient.DreamDaemon.Restart(cancellationToken), ErrorCode.WatchdogNotRunning).AsTask());
await RunBasicTest(cancellationToken);
await RunBasicTest(false, cancellationToken);
// hardlinks and DMAPI checks don't play well together
bool linuxAdvancedWatchdogWeirdness = testVersion.Engine.Value == EngineType.Byond
&& !new PlatformIdentifier().IsWindows
&& !watchdogRestartsProcess;
if (!linuxAdvancedWatchdogWeirdness)
await RunBasicTest(true, cancellationToken);
await TestDMApiFreeDeploy(cancellationToken);
@@ -220,7 +228,7 @@ namespace Tgstation.Server.Tests.Live.Instance
}, cancellationToken);
// for the restart staging tests
await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}");
}
@@ -234,7 +242,7 @@ namespace Tgstation.Server.Tests.Live.Instance
{
SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe,
}, cancellationToken);
var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken);
var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
await ddUpdateTask;
Assert.AreEqual(WatchdogStatus.Offline, currentStatus.Status);
@@ -378,7 +386,7 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(restartJob, 10, false, null, cancellationToken);
// we need to cycle deployments twice because TGS holds the initial deployment
var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
Assert.IsNotNull(currentStatus.StagedCompileJob);
@@ -399,7 +407,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsNotNull(topicRequestResult);
Assert.AreEqual("we love casting spells", topicRequestResult.StringData);
currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart);
ValidateSessionId(currentStatus, false);
@@ -515,7 +523,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsNotNull(deleteJob);
await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Safe, true, cancellationToken);
await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Safe, DMApiValidationMode.Required, cancellationToken);
return deleteJob;
}
@@ -600,7 +608,7 @@ namespace Tgstation.Server.Tests.Live.Instance
async Task TestDMApiFreeDeploy(CancellationToken cancellationToken)
{
System.Console.WriteLine("TEST: WATCHDOG API FREE TEST");
var daemonStatus = await DeployTestDme("ApiFree/api_free", DreamDaemonSecurity.Safe, false, cancellationToken);
var daemonStatus = await DeployTestDme("ApiFree/api_free", DreamDaemonSecurity.Safe, DMApiValidationMode.Optional, cancellationToken);
Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
Assert.IsNotNull(daemonStatus.ActiveCompileJob);
@@ -626,7 +634,7 @@ namespace Tgstation.Server.Tests.Live.Instance
var initialCompileJob = daemonStatus.ActiveCompileJob;
await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
daemonStatus = await DeployTestDme("BasicOperation/basic operation_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
daemonStatus = await DeployTestDme("BasicOperation/basic operation_test", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
Assert.AreEqual(false, daemonStatus.SoftRestart); // dme name change triggered, instant reboot
@@ -668,7 +676,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.AreEqual(expected, nonLiveDirs, $"Directories present: {String.Join(", ", lastDirectories.Select(Path.GetFileName))}");
}
async Task RunBasicTest(CancellationToken cancellationToken)
async Task RunBasicTest(bool skipApiValidation, CancellationToken cancellationToken)
{
System.Console.WriteLine("TEST: WATCHDOG BASIC TEST");
@@ -677,15 +685,26 @@ namespace Tgstation.Server.Tests.Live.Instance
AdditionalParameters = "test=bababooey"
}, cancellationToken);
Assert.AreEqual("test=bababooey", daemonStatus.AdditionalParameters);
daemonStatus = await DeployTestDme("BasicOperation/basic operation_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
daemonStatus = await DeployTestDme($"BasicOperation/basic operation_test{(skipApiValidation ? "_nov3" : String.Empty)}", DreamDaemonSecurity.Trusted, skipApiValidation ? DMApiValidationMode.Skipped : DMApiValidationMode.Required, cancellationToken);
Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
Assert.IsNotNull(daemonStatus.ActiveCompileJob);
if (skipApiValidation)
{
Assert.IsNull(daemonStatus.ActiveCompileJob.DMApiVersion);
Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, daemonStatus.ActiveCompileJob.MinimumSecurityLevel);
}
else
{
Assert.AreEqual(DMApiConstants.InteropVersion, daemonStatus.ActiveCompileJob.DMApiVersion);
Assert.AreEqual(DreamDaemonSecurity.Trusted, daemonStatus.ActiveCompileJob.MinimumSecurityLevel);
}
Assert.IsFalse(daemonStatus.SessionId.HasValue);
Assert.IsFalse(daemonStatus.LaunchTime.HasValue);
Assert.IsNull(daemonStatus.StagedCompileJob);
Assert.AreEqual(DMApiConstants.InteropVersion, daemonStatus.ActiveCompileJob.DMApiVersion);
Assert.AreEqual(DreamDaemonSecurity.Trusted, daemonStatus.ActiveCompileJob.MinimumSecurityLevel);
Assert.AreNotEqual(DreamDaemonSecurity.Ultrasafe, daemonStatus.SecurityLevel);
await ExpectGameDirectoryCount(1, cancellationToken);
@@ -726,6 +745,9 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue);
Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value);
if (skipApiValidation)
Assert.IsFalse(daemonStatus.ClientCount.HasValue);
await GracefulWatchdogShutdown(cancellationToken);
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
@@ -734,7 +756,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsFalse(daemonStatus.LaunchTime.HasValue);
await ExpectGameDirectoryCount(1, cancellationToken);
await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken, false);
await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken, false, skipApiValidation);
daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
{
@@ -1245,7 +1267,7 @@ namespace Tgstation.Server.Tests.Live.Instance
System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH UPDATE TEST");
const string DmeName = "LongRunning/long_running_test";
var daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Trusted, true, cancellationToken);
var daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
var initialCompileJob = daemonStatus.ActiveCompileJob;
Assert.IsNotNull(initialCompileJob);
@@ -1259,7 +1281,7 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken);
daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, DMApiValidationMode.Required, cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
ValidateSessionId(daemonStatus, true);
@@ -1292,7 +1314,7 @@ namespace Tgstation.Server.Tests.Live.Instance
{
System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH NEW DME TEST");
var daemonStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
var daemonStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, DMApiValidationMode.Required, cancellationToken);
var initialCompileJob = daemonStatus.ActiveCompileJob;
Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
@@ -1306,7 +1328,7 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken);
daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, DMApiValidationMode.Required, cancellationToken);
ValidateSessionId(daemonStatus, true);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
@@ -1371,7 +1393,7 @@ namespace Tgstation.Server.Tests.Live.Instance
const string DmeName = "LongRunning/long_running_test";
await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken);
await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, DMApiValidationMode.Required, cancellationToken);
var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
@@ -1512,13 +1534,13 @@ namespace Tgstation.Server.Tests.Live.Instance
return daemonStatus;
}
async Task<DreamDaemonResponse> DeployTestDme(string dmeName, DreamDaemonSecurity deploymentSecurity, bool requireApi, CancellationToken cancellationToken)
async Task<DreamDaemonResponse> DeployTestDme(string dmeName, DreamDaemonSecurity deploymentSecurity, DMApiValidationMode dmApiValidationMode, CancellationToken cancellationToken)
{
var refreshed = await instanceClient.DreamMaker.Update(new DreamMakerRequest
{
ApiValidationSecurityLevel = deploymentSecurity,
ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}",
RequireDMApiValidation = requireApi,
DMApiValidationMode = dmApiValidationMode,
Timeout = !ranTimeoutTest ? TimeSpan.FromMilliseconds(1) : TimeSpan.FromMinutes(5),
}, cancellationToken);
@@ -1526,7 +1548,7 @@ namespace Tgstation.Server.Tests.Live.Instance
if (!ranTimeoutTest)
{
Assert.AreEqual(deploymentSecurity, refreshed.ApiValidationSecurityLevel);
Assert.AreEqual(requireApi, refreshed.RequireDMApiValidation);
Assert.AreEqual(dmApiValidationMode, refreshed.DMApiValidationMode);
Assert.AreEqual(TimeSpan.FromMilliseconds(1), refreshed.Timeout);
compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken);
@@ -1550,7 +1572,7 @@ namespace Tgstation.Server.Tests.Live.Instance
var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob;
Assert.IsNotNull(targetJob);
if (requireApi)
if (dmApiValidationMode == DMApiValidationMode.Required)
Assert.IsNotNull(targetJob.DMApiVersion);
else
Assert.IsNull(targetJob.DMApiVersion);
@@ -1582,12 +1604,20 @@ namespace Tgstation.Server.Tests.Live.Instance
while (timeout > 0);
}
async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken, bool checkLogs = true)
async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken, bool checkLogs = true, bool expectInitialBridgeFailure = false)
{
var gameDir = Path.Combine(instanceClient.Metadata.Path, "Game", compileJob.DirectoryName.Value.ToString(), Path.GetDirectoryName(compileJob.DmeName));
var failFile = Path.Combine(gameDir, "test_fail_reason.txt");
if (!File.Exists(failFile))
{
var bridgeFailFile = Path.Combine(gameDir, "initial_bridge_failed.txt");
var initialBridgeFailed = File.Exists(bridgeFailFile);
System.Console.WriteLine($"Files in game dir:{Environment.NewLine}{String.Join(Environment.NewLine, Directory.GetFiles(gameDir))}");
Assert.AreEqual(expectInitialBridgeFailure, initialBridgeFailed, $"Initial bridge failure expectancy not met in {gameDir}");
var successFile = Path.Combine(gameDir, "test_success.txt");
Assert.IsTrue(File.Exists(successFile));
}
+2 -1
View File
@@ -13,6 +13,7 @@ using Tgstation.Server.Host.Database.Migrations;
using Tgstation.Server.Host.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Tgstation.Server.Api.Models.Internal;
namespace Tgstation.Server.Tests
{
@@ -130,7 +131,7 @@ namespace Tgstation.Server.Tests
{
ApiValidationPort = 1557,
ApiValidationSecurityLevel = DreamDaemonSecurity.Trusted,
RequireDMApiValidation = false,
DMApiValidationMode = DMApiValidationMode.Skipped,
Timeout = TimeSpan.FromSeconds(13),
},
InstancePermissionSets = new List<Host.Models.InstancePermissionSet>
+2
View File
@@ -77,6 +77,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tgs", "tgs", "{F7765A4B-021
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "core", "{DCCBA9DA-47BA-4C70-823B-E99A3ACA0377}"
ProjectSection(SolutionItems) = preProject
src\DMAPI\tgs\core\byond_world_export.dm = src\DMAPI\tgs\core\byond_world_export.dm
src\DMAPI\tgs\core\core.dm = src\DMAPI\tgs\core\core.dm
src\DMAPI\tgs\core\datum.dm = src\DMAPI\tgs\core\datum.dm
src\DMAPI\tgs\core\README.md = src\DMAPI\tgs\core\README.md
@@ -175,6 +176,7 @@ EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BasicOperation", "BasicOperation", "{F32B9514-AAD9-429D-841A-ED810FC2598C}"
ProjectSection(SolutionItems) = preProject
tests\DMAPI\BasicOperation\basic operation_test.dme = tests\DMAPI\BasicOperation\basic operation_test.dme
tests\DMAPI\BasicOperation\basic operation_test_nov3.dme = tests\DMAPI\BasicOperation\basic operation_test_nov3.dme
tests\DMAPI\BasicOperation\Config.dm = tests\DMAPI\BasicOperation\Config.dm
tests\DMAPI\BasicOperation\Test.dm = tests\DMAPI\BasicOperation\Test.dm
tests\DMAPI\BasicOperation\test_event-qwer.bat = tests\DMAPI\BasicOperation\test_event-qwer.bat
@@ -4,21 +4,17 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IdentityModel.Tokens.Jwt;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Sockets;
using System.Security;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.IdentityModel.Tokens;
using Newtonsoft.Json;
using Octokit;