v2.6.0 (#59)

* v2.6.0
- *Enhancement:* Added a `DbColumnSchema.SqlType2` that does _not_ include nullability.
- *Enhancement:* The `SqlServerSchemaScript.SupportsReplace` is enabled where a `CREATE OR ALTER` is specified.
- *Enhancement:* The SQL Server [Event Outbox](https://github.com/Avanade/CoreEx/blob/main/src/CoreEx.Database.SqlServer/Outbox/EventOutboxEnqueueBase.cs) (_CoreEx_ `v3.26.0`) capabilities now support events as JSON versus existing TVP removing database dependency on a UDT (user-defined type).
- *Fixed:* The logic for finding file-based SQL schema scripts has been corrected.

Author: chullybun

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 Represents the **NuGet** versions.
 
+## v2.6.0
+- *Enhancement:* Added a `DbColumnSchema.SqlType2` that does _not_ include nullability.
+- *Enhancement:* The `SqlServerSchemaScript.SupportsReplace` is enabled where a `CREATE OR ALTER` is specified.
+- *Enhancement:* The SQL Server [Event Outbox](https://github.com/Avanade/CoreEx/blob/main/src/CoreEx.Database.SqlServer/Outbox/EventOutboxEnqueueBase.cs) (_CoreEx_ `v3.26.0`) capabilities now support events as JSON versus existing TVP removing database dependency on a UDT (user-defined type).
+- *Fixed:* The logic for finding file-based SQL schema scripts has been corrected.
+
 ## v2.5.9
 - *Fixed:* All related package dependencies updated to latest.
 

Common.targets

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <Version>2.5.9</Version>
+    <Version>2.6.0</Version>
     <LangVersion>preview</LangVersion>
     <Authors>Avanade</Authors>
     <Company>Avanade</Company>

src/DbEx.MySql/DbEx.MySql.csproj

@@ -41,7 +41,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CoreEx.Database.MySql" Version="3.23.0" />
+    <PackageReference Include="CoreEx.Database.MySql" Version="3.25.6" />
     <PackageReference Include="dbup-mysql" Version="5.0.44" />
   </ItemGroup>
 

src/DbEx.MySql/Migration/MySqlMigration.cs

@@ -50,6 +50,10 @@ public MySqlMigration(MigrationArgsBase args) : base(args)
             if (SchemaObjectTypes.Length == 0)
                 SchemaObjectTypes = ["FUNCTION", "VIEW", "PROCEDURE"];
 
+            // MySql will require all schema objects to be dropped as replacements are not currently supported.
+            if (MustDropSchemaObjectTypes.Length == 0)
+                MustDropSchemaObjectTypes = ["FUNCTION", "VIEW", "PROCEDURE"];
+
             // Add/set standard parameters.
             Args.AddParameter(MigrationArgsBase.DatabaseNameParamName, _databaseName, true);
             Args.AddParameter(MigrationArgsBase.JournalSchemaParamName, null, true);

src/DbEx.Postgres/DbEx.Postgres.csproj

@@ -42,7 +42,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CoreEx.Database.Postgres" Version="3.23.0" />
+    <PackageReference Include="CoreEx.Database.Postgres" Version="3.25.6" />
     <PackageReference Include="dbup-postgresql" Version="5.0.40" />
   </ItemGroup>
 

src/DbEx.Postgres/Migration/PostgresSchemaScript.cs

@@ -36,7 +36,7 @@ public static PostgresSchemaScript Create(DatabaseMigrationScript migrationScrip
                 {
                     if (string.Compare(tokens[i + 1], "or", StringComparison.OrdinalIgnoreCase) == 0 && string.Compare(tokens[i + 2], "replace", StringComparison.OrdinalIgnoreCase) == 0)
                     {
-                        i = +2;
+                        i =+ 2;
                         script.SupportsReplace = true;
                     }
 

src/DbEx.SqlServer/DbEx.SqlServer.csproj

@@ -32,8 +32,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CoreEx.Database.SqlServer" Version="3.23.0" />
+    <PackageReference Include="CoreEx.Database.SqlServer" Version="3.25.6" />
     <PackageReference Include="dbup-sqlserver" Version="5.0.41" />
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/DbEx.SqlServer/Migration/SqlServerMigration.cs

@@ -54,6 +54,10 @@ public SqlServerMigration(MigrationArgsBase args) : base(args)
             if (SchemaObjectTypes.Length == 0)
                 SchemaObjectTypes = ["TYPE", "FUNCTION", "VIEW", "PROCEDURE", "PROC"];
 
+            // A schema object type that is a user-defined type will require all schema objects to be dropped (as it may be referenced).
+            if (MustDropSchemaObjectTypes.Length == 0)
+                MustDropSchemaObjectTypes = ["TYPE"];
+
             // Always add the dbo schema _first_ unless already specified.
             if (!Args.SchemaOrder.Contains(SchemaConfig.DefaultSchema))
                 Args.SchemaOrder.Insert(0, SchemaConfig.DefaultSchema);

src/DbEx.SqlServer/Migration/SqlServerSchemaScript.cs

@@ -32,8 +32,14 @@ public static SqlServerSchemaScript Create(DatabaseMigrationScript migrationScri
                 if (string.Compare(tokens[i], "create", StringComparison.OrdinalIgnoreCase) != 0)
                     continue;
 
-                if (i + 2 < tokens.Length)
+                if (i + 4 < tokens.Length)
                 {
+                    if (string.Compare(tokens[i + 1], "or", StringComparison.OrdinalIgnoreCase) == 0 && string.Compare(tokens[i + 2], "alter", StringComparison.OrdinalIgnoreCase) == 0)
+                    {
+                        i = +2;
+                        script.SupportsReplace = true;
+                    }
+
                     script.Type = tokens[i + 1];
                     script.FullyQualifiedName = tokens[i + 2];
 
@@ -67,7 +73,7 @@ private SqlServerSchemaScript(DatabaseMigrationScript migrationScript) : base(mi
         public override string SqlDropStatement => $"DROP {Type.ToUpperInvariant()} IF EXISTS [{Schema}].[{Name}]";
 
         /// <inheritdoc/>
-        public override string SqlCreateStatement => $"CREATE {Type.ToUpperInvariant()} [{Schema}].[{Name}]";
+        public override string SqlCreateStatement => $"CREATE {(SupportsReplace ? "OR ALTER " : "")}{Type.ToUpperInvariant()} [{Schema}].[{Name}]";
 
         private class SqlCommandTokenizer(string sqlText) : SqlCommandReader(sqlText)
         {
@@ -93,7 +99,7 @@ public string[] ReadAllTokens()
                                     sb.Clear();
                                     break;
                                 }
-                                else if (new char[] { '(', ')', ';', ',', '=' }.Contains(c))
+                                else if (delimiters.Contains(c))
                                 {
                                     if (sb.Length > 0)
                                         words.Add(sb.ToString());

src/DbEx.SqlServer/Templates/EventOutboxEnqueue_cs.hbs

@@ -17,9 +17,6 @@ namespace {{NamespaceOutbox}}.Data;
 /// <param name="logger">The <see cref="ILogger"/>.</param>
 public sealed class EventOutboxEnqueue(IDatabase database, ILogger<EventOutboxEnqueue> logger) : EventOutboxEnqueueBase(database, logger)
 {
-    /// <inheritdoc/>
-    protected override string DbTvpTypeName => "[{{OutboxSchema}}].[udt{{OutboxTable}}List]";
-
     /// <inheritdoc/>
     protected override string EnqueueStoredProcedure => "[{{OutboxSchema}}].[{{OutboxEnqueueStoredProcedure}}]";
 }{{#if Root.PreprocessorDirectives}}

src/DbEx.SqlServer/Templates/SpEventOutboxDequeue_sql.hbs

@@ -1,5 +1,5 @@
 {{! Copyright (c) Avanade. Licensed under the MIT License. See https://github.com/Avanade/NTangle }}
-CREATE PROCEDURE [{{OutboxSchema}}].[sp{{OutboxTable}}Dequeue]
+CREATE OR ALTER PROCEDURE [{{OutboxSchema}}].[sp{{OutboxTable}}Dequeue]
   @MaxDequeueSize INT = 10,  -- Maximum number of events to dequeue.
   @PartitionKey NVARCHAR(127) NULL = NULL,  -- Partition key; null indicates all.
   @Destination NVARCHAR(127) NULL = NULL  -- Destination (queue or topic); null indicates all.

src/DbEx.SqlServer/Templates/SpEventOutboxEnqueue_sql.hbs

@@ -1,7 +1,7 @@
 {{! Copyright (c) Avanade. Licensed under the MIT License. See https://github.com/Avanade/NTangle }}
-CREATE PROCEDURE [{{OutboxSchema}}].[sp{{OutboxTable}}Enqueue]
+CREATE OR ALTER PROCEDURE [{{OutboxSchema}}].[sp{{OutboxTable}}Enqueue]
   @SetEventsAsDequeued AS BIT = 0,
-  @EventList AS [{{OutboxSchema}}].[udt{{OutboxTable}}List] READONLY
+  @EventList AS NVARCHAR(MAX)
 AS
 BEGIN
   /*
@@ -25,6 +25,24 @@ BEGIN
     -- Enqueued outbox resultant identifier.
     DECLARE @enqueuedId TABLE([{{OutboxTable}}Id] BIGINT)
 
+    -- Convert the JSON to a temporary table.
+    SELECT * INTO #eventList FROM OPENJSON(@EventList) WITH (
+      [EventId] NVARCHAR(127) '$.EventId',
+      [EventDequeued] BIT '$.EventDequeued',
+      [Destination] NVARCHAR(127) '$.Destination',
+      [Subject] NVARCHAR(511) '$.Subject',
+      [Action] NVARCHAR(255) '$.Action',
+      [Type] NVARCHAR(1023) '$.Type',
+      [Source] NVARCHAR(1023) '$.Source',
+      [Timestamp] DATETIMEOFFSET '$.Timestamp',
+      [CorrelationId] NVARCHAR(127) '$.CorrelationId',
+      [Key] NVARCHAR(1023) '$.Key',
+      [TenantId] NVARCHAR(127) '$.TenantId',
+      [PartitionKey] NVARCHAR(127) '$.PartitionKey',
+      [ETag] NVARCHAR(127) '$.ETag',
+      [Attributes] VARBINARY(MAX) '$.Attributes',
+      [Data] VARBINARY(MAX) '$.Data')
+
     -- Cursor output variables.
     DECLARE @eventId NVARCHAR(127),
             @eventDequeued BIT,
@@ -44,7 +62,7 @@ BEGIN
 
     -- Declare, open, and fetch first event from cursor.
     DECLARE c CURSOR FORWARD_ONLY
-      FOR SELECT [EventId], [EventDequeued], [Destination], [Subject], [Action], [Type], [Source], [Timestamp], [CorrelationId], [Key], [TenantId], [PartitionKey], [ETag], [Attributes], [Data] FROM @EventList
+      FOR SELECT [EventId], [EventDequeued], [Destination], [Subject], [Action], [Type], [Source], [Timestamp], [CorrelationId], [Key], [TenantId], [PartitionKey], [ETag], [Attributes], [Data] FROM #eventList
 
     OPEN c
     FETCH NEXT FROM c INTO @eventId, @eventDequeued, @destination, @subject, @action, @type, @source, @timestamp, @correlationId, @key, @tenantId, @partitionKey, @etag, @attributes, @data

src/DbEx.SqlServer/Templates/UdtEventOutbox_sql.hbs

@@ -20,8 +20,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CoreEx.Database" Version="3.23.0" />
-    <PackageReference Include="OnRamp" Version="2.2.1" />
+    <PackageReference Include="CoreEx.Database" Version="3.25.6" />
+    <PackageReference Include="OnRamp" Version="2.2.2" />
   </ItemGroup>
 
   <Import Project="..\..\Common.targets" />

src/DbEx/DbEx.csproj

@@ -20,6 +20,7 @@ public class DbColumnSchema(DbTableSchema dbTable, string name, string type, str
         private string? _dotNetName = dotNetNameOverride;
         private string? _dotNetCleanedName;
         private string? _sqlType;
+        private string? _sqlType2;
 
         /// <summary>
         /// Gets the owning (parent) <see cref="DbTable"/>.
@@ -177,10 +178,15 @@ public class DbColumnSchema(DbTableSchema dbTable, string name, string type, str
         public string DotNetCleanedName { get => _dotNetCleanedName ?? DotNetName; set => _dotNetCleanedName = value; }
 
         /// <summary>
-        /// Gets the fully defined SQL type.
+        /// Gets the fully defined SQL type (includes nullability).
         /// </summary>
         public string SqlType => _sqlType ??= DbTable?.Migration.SchemaConfig.ToFormattedSqlType(this) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(SqlType)} property can be accessed.");
 
+        /// <summary>
+        /// Gets the fully defined SQL type (excludes nullability).
+        /// </summary>
+        public string SqlType2 => _sqlType2 ??= DbTable?.Migration.SchemaConfig.ToFormattedSqlType(this, false) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(SqlType)} property can be accessed.");
+
 #if NET7_0_OR_GREATER
         /// <summary>
         /// Indicates that the type can be expressed as a <see cref="DateOnly"/> .NET type.

src/DbEx/DbSchema/DbColumnSchema.cs

@@ -126,7 +126,12 @@ protected DatabaseMigrationBase(MigrationArgsBase args)
         /// </summary>
         /// <remarks>The objects will be added in the order specified, and removed in the reverse order. This is to allow for potential dependencies between the object types.
         /// <para>Where none are specified then the <see cref="MigrationCommand.Schema"/> phase will be skipped.</para></remarks>
-        public string[] SchemaObjectTypes { get; set; }
+        public string[] SchemaObjectTypes { get; set; } = [];
+
+        /// <summary>
+        /// Gets or sets the list of schema object types that where found must result in all schema objects being dropped and then recreated.
+        /// </summary>
+        public string[] MustDropSchemaObjectTypes { get; set; } = [];
 
         /// <summary>
         /// Gets the assemblies used for probing the requisite artefact resources (used for providing the underlying requisite database statements for the specified <see cref="Provider"/>).
@@ -513,16 +518,17 @@ private async Task<bool> DatabaseSchemaAsync(CancellationToken cancellationToken
             var scripts = new List<DatabaseMigrationScript>();
 
             // See if there are any files out there that should take precedence over embedded resources.
-            if (Args.OutputDirectory != null)
+            var dir = new DirectoryInfo(CodeGenConsole.GetBaseExeDirectory());
+            if (dir != null && dir.Exists)
             {
-                var di = new DirectoryInfo(Path.Combine(Args.OutputDirectory.FullName, SchemaNamespace));
+                var di = new DirectoryInfo(Path.Combine(dir.FullName, SchemaNamespace));
                 Logger.LogInformation("{Content}", $"  Probing for files (recursively): {Path.Combine(di.FullName, "*", "*.sql")}");
 
                 if (di.Exists)
                 {
                     foreach (var fi in di.GetFiles("*.sql", SearchOption.AllDirectories))
                     {
-                        var rn = $"{fi.FullName[((Args.OutputDirectory?.Parent?.FullName.Length + 1) ?? 0)..]}".Replace(' ', '_').Replace('-', '_').Replace('\\', '.').Replace('/', '.');
+                        var rn = $"{fi.FullName[((dir.Parent?.FullName.Length + 1) ?? 0)..]}".Replace(' ', '_').Replace('-', '_').Replace('\\', '.').Replace('/', '.');
                         scripts.Add(new DatabaseMigrationScript(this, fi, rn));
                     }
                 }
@@ -573,27 +579,35 @@ protected virtual async Task<bool> DatabaseSchemaAsync(List<DatabaseMigrationScr
                 var script = ValidateAndReadySchemaScript(CreateSchemaScript(migrationScript));
                 if (script.HasError)
                 {
-                    Logger.LogError("{Message}", $"SQL script '{migrationScript.Name}' is not valid: {script.ErrorMessage}");
+                    Logger.LogError("{Content}", $"SQL script '{migrationScript.Name}' is not valid: {script.ErrorMessage}");
                     return false;
                 }
 
                 list.Add(script);
             }
 
             // Drop all existing (in reverse order).
-            int i = 0;
-            var ss = new List<DatabaseMigrationScript>();
             Logger.LogInformation("{Content}", string.Empty);
             Logger.LogInformation("{Content}", "  Drop known schema objects...");
-            foreach (var sor in list.Where(x => !x.SupportsReplace).OrderByDescending(x => x.SchemaOrder).ThenByDescending(x => x.TypeOrder).ThenByDescending(x => x.Schema).ThenByDescending(x => x.Name))
+
+            var fullDrop = Args.DropSchemaObjects;
+            if (!fullDrop && MustDropSchemaObjectTypes.Length > 0)
+                fullDrop = list.Where(x => MustDropSchemaObjectTypes.Contains(x.Type, StringComparer.OrdinalIgnoreCase)).Any();
+
+            int i = 0;
+            var ss = new List<DatabaseMigrationScript>();
+            if (fullDrop || list.Where(x => !x.SupportsReplace).Any())
             {
-                ss.Add(new DatabaseMigrationScript(this, sor.SqlDropStatement, sor.SqlDropStatement) { GroupOrder = i++, RunAlways = true });
-            }
+                foreach (var sor in list.Where(x => fullDrop || !x.SupportsReplace).OrderByDescending(x => x.SchemaOrder).ThenByDescending(x => x.TypeOrder).ThenByDescending(x => x.Schema).ThenByDescending(x => x.Name))
+                {
+                    ss.Add(new DatabaseMigrationScript(this, sor.SqlDropStatement, sor.SqlDropStatement) { GroupOrder = i++, RunAlways = true });
+                }
 
-            if (i == 0)
-                Logger.LogInformation("{Content}", "    None.");
-            else if (!await ExecuteScriptsAsync(ss, true, cancellationToken).ConfigureAwait(false))
-                return false;
+                if (!await ExecuteScriptsAsync(ss, true, cancellationToken).ConfigureAwait(false))
+                    return false;
+            }
+            else
+                Logger.LogInformation("{Content}", "    ** Note: All schema objects implement replace functionality and therefore there is no need to drop existing. **");
 
             // Execute each migration script proper (i.e. create 'em as scripted).
             i = 0;

src/DbEx/Migration/DatabaseMigrationBase.cs

@@ -74,7 +74,7 @@ public abstract class DatabaseSchemaScriptBase(DatabaseMigrationScript migration
         public bool HasError => ErrorMessage != null;
 
         /// <summary>
-        /// Indicates whether the schema script supports a create or replace; i.e. does not require a drop and create as two separate operations.
+        /// Indicates whether the schema script supports a create or replace/alter; i.e. does not require a drop and create as two separate operations.
         /// </summary>
         public bool SupportsReplace { get; protected set; }
 

src/DbEx/Migration/DatabaseSchemaScriptBase.cs

@@ -167,6 +167,11 @@ public abstract class MigrationArgsBase : OnRamp.CodeGeneratorDbArgsBase
         /// </summary>
         public bool AcceptPrompts { get; set; }
 
+        /// <summary>
+        /// Indicates whether to drop all the known schema objects before creating them.
+        /// </summary>
+        public bool DropSchemaObjects { get; set; }
+
         /// <summary>
         /// Gets or sets the <see cref="MigrationCommand.Reset"/> table filtering predicate.
         /// </summary>

src/DbEx/Migration/MigrationArgsBase.cs

@@ -2,7 +2,7 @@
   "profiles": {
     "DbEx.Test.Console": {
       "commandName": "Project",
-      "commandLineArgs": "dropandall"
+      "commandLineArgs": "all"
     }
   }
 }

tests/DbEx.Test.Console/Properties/launchSettings.json

@@ -1,4 +1,4 @@
-CREATE PROCEDURE Test.spGetContact
+CREATE OR ALTER PROCEDURE Test.spGetContact
   @ContactId AS INT /* this is a comment */
 AS
 BEGIN

tests/DbEx.Test.Console/Schema/spGetContact.sql

@@ -1,4 +1,4 @@
-CREATE PROCEDURE [Test].[spGetContact2]
+CREATE OR ALTER PROCEDURE [Test].[spGetContact2]
   @ContactId AS INT /* this is a comment */
 AS
 BEGIN

tests/DbEx.Test.Console/Schema/spGetContact2.sql

@@ -29,7 +29,6 @@
   <ItemGroup>
     <Folder Include="Generated\" />
     <Folder Include="Migrations\" />
-    <Folder Include="Schema\" />
   </ItemGroup>
 
 </Project>

tests/DbEx.Test.OutboxConsole/DbEx.Test.OutboxConsole.csproj

@@ -11,9 +11,6 @@ namespace DbEx.Test.OutboxConsole.Data;
 /// <param name="logger">The <see cref="ILogger"/>.</param>
 public sealed class EventOutboxEnqueue(IDatabase database, ILogger<EventOutboxEnqueue> logger) : EventOutboxEnqueueBase(database, logger)
 {
-    /// <inheritdoc/>
-    protected override string DbTvpTypeName => "[Outbox].[udtEventOutboxList]";
-
     /// <inheritdoc/>
     protected override string EnqueueStoredProcedure => "[Outbox].[spEventOutboxEnqueue]";
 }

tests/DbEx.Test.OutboxConsole/Generated/EventOutboxEnqueue.cs

@@ -1,4 +1,4 @@
-CREATE PROCEDURE [Outbox].[spEventOutboxDequeue]
+CREATE OR ALTER PROCEDURE [Outbox].[spEventOutboxDequeue]
   @MaxDequeueSize INT = 10,  -- Maximum number of events to dequeue.
   @PartitionKey NVARCHAR(127) NULL = NULL,  -- Partition key; null indicates all.
   @Destination NVARCHAR(127) NULL = NULL  -- Destination (queue or topic); null indicates all.