Merge pull request #454 from atlanhq/DVX-196

Ensure failOnErrors applies to short-circuit exiting, too
atlanhq · Jan 23, 2024 · 3dde603 · 3dde603
2 parents b68ac31 + 1a6866d
commit 3dde603
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/serde/csv/CSVImporter.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/serde/csv/CSVImporter.kt
@@ -33,6 +33,7 @@ import kotlin.system.exitProcess
  * @param caseSensitive (only applies when updateOnly is true) attempt to match assets case-sensitively (true) or case-insensitively (false)
  * @param creationHandling if assets are to be created, how they should be created (as full assets or only partial assets)
  * @param tableViewAgnostic if true, tables and views will be treated interchangeably (an asset in the batch marked as a table will attempt to match a view if not found as a table, and vice versa)
+ * @param failOnErrors if true, fail if errors are encountered, otherwise continue processing
  */
 abstract class CSVImporter(
     private val filename: String,
@@ -45,6 +46,7 @@ abstract class CSVImporter(
     private val caseSensitive: Boolean = true,
     private val creationHandling: AssetCreationHandling = AssetCreationHandling.FULL,
     private val tableViewAgnostic: Boolean = false,
+    private val failOnErrors: Boolean = true,
 ) : AssetGenerator {
 
     /**
@@ -66,7 +68,7 @@ abstract class CSVImporter(
             val start = System.currentTimeMillis()
             val results = csv.streamRows(this, batchSize, logger, columnsToSkip)
             logger.info { "Total time taken: ${System.currentTimeMillis() - start} ms" }
-            if (results.anyFailures) {
+            if (results.anyFailures && failOnErrors) {
                 logger.error { "Some errors detected, failing the workflow." }
                 exitProcess(1)
             }

diff --git a/samples/packages/asset-import/src/main/kotlin/com/atlan/pkg/aim/AssetImporter.kt b/samples/packages/asset-import/src/main/kotlin/com/atlan/pkg/aim/AssetImporter.kt
@@ -25,6 +25,7 @@ import mu.KotlinLogging
  * @param caseSensitive (only applies when updateOnly is true) attempt to match assets case-sensitively (true) or case-insensitively (false)
  * @param creationHandling if assets are to be created, how they should be created (as full assets or only partial assets)
  * @param tableViewAgnostic if true, tables and views will be treated interchangeably (an asset in the batch marked as a table will attempt to match a view if not found as a table, and vice versa)
+ * @param failOnErrors if true, fail if errors are encountered, otherwise continue processing
  */
 class AssetImporter(
     private val filename: String,
@@ -34,6 +35,7 @@ class AssetImporter(
     private val caseSensitive: Boolean = true,
     private val creationHandling: AssetCreationHandling = AssetCreationHandling.NONE,
     private val tableViewAgnostic: Boolean = false,
+    private val failOnErrors: Boolean = true,
 ) : CSVImporter(
     filename,
     logger = KotlinLogging.logger {},
@@ -43,6 +45,7 @@ class AssetImporter(
     caseSensitive = caseSensitive,
     creationHandling = creationHandling,
     tableViewAgnostic = tableViewAgnostic,
+    failOnErrors = failOnErrors,
 ) {
     /** {@inheritDoc} */
     override fun getBuilder(deserializer: RowDeserializer): Asset.AssetBuilder<*, *> {

diff --git a/samples/packages/asset-import/src/main/kotlin/com/atlan/pkg/aim/Importer.kt b/samples/packages/asset-import/src/main/kotlin/com/atlan/pkg/aim/Importer.kt
@@ -112,6 +112,7 @@ object Importer {
                 assetsCaseSensitive,
                 creationHandling,
                 assetsTableViewAgnostic,
+                assetsFailOnErrors,
             )
             assetImporter.import()
         } else {