diff --git a/.github/workflows/std-libs-labels.yml b/.github/workflows/std-libs-labels.yml index f9bc21011f9a..361501ed63ab 100644 --- a/.github/workflows/std-libs-labels.yml +++ b/.github/workflows/std-libs-labels.yml @@ -89,6 +89,31 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} labels: -libs-API-change-Database + stdlib-api-check-DuckDB-linux-amd64: + name: DuckDB-change-labels + runs-on: + - ubuntu-latest + steps: + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + fetch-depth: 2 + - id: DuckDB-changed-files + name: DuckDB-changed-files + uses: step-security/changed-files@v45 + with: + files: distribution/lib/Standard/DuckDB/**/docs/api/**/**.md + - name: List all changed files in DuckDB + run: "\n if [[ \"${{ steps.DuckDB-changed-files.outputs.any_changed }}\" == \"true\" ]]; then\n echo \"Files changed:\"\n fi\n for file in ${ALL_CHANGED_FILES}; do\n echo \"$file\"\n done\n " + env: + ALL_CHANGED_FILES: ${{ steps.DuckDB-changed-files.outputs.all_changed_files }} + - if: steps.DuckDB-changed-files.outputs.any_changed == 'true' + name: Append -libs-API-change-DuckDB label + uses: actions-ecosystem/action-add-labels@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: -libs-API-change-DuckDB stdlib-api-check-Generic_JDBC-linux-amd64: name: Generic_JDBC-change-labels runs-on: diff --git a/.gitignore b/.gitignore index fcc19b428e1e..885343c199ea 100644 --- a/.gitignore +++ b/.gitignore @@ -115,6 +115,9 @@ bench-report*.xml *.pdb *.so *.jar +*.so_linux_amd64 +*.so_osx_universal +*.so_windows_amd64 ######### diff --git a/bazel_scripts/stdlibs.bzl b/bazel_scripts/stdlibs.bzl index eca7a19ebd59..ade5a275fa11 100644 --- a/bazel_scripts/stdlibs.bzl +++ b/bazel_scripts/stdlibs.bzl @@ -6,6 +6,7 @@ STDLIB_NAMES = [ "AWS", "Base", "Database", + "DuckDB", "Examples", "Generic_JDBC", "Geo", diff --git a/build.sbt b/build.sbt index 3a1af7bace9d..6f319d1700c0 100644 --- a/build.sbt +++ b/build.sbt @@ -146,6 +146,10 @@ GatherLicenses.distributions := Seq( makeStdLibDistribution( "Saas", Distribution.sbtProjects(`std-saas`) + ), + makeStdLibDistribution( + "DuckDB", + Distribution.sbtProjects(`std-duckdb`) ) ) @@ -400,6 +404,7 @@ lazy val enso = (project in file(".")) `std-table`, `std-tableau`, `std-saas`, + `std-duckdb`, `sqlite-wrapper`, `syntax-rust-definition`, `tableau-wrapper`, @@ -2852,6 +2857,7 @@ lazy val runtime = (project in file("engine/runtime")) .dependsOn(`std-microsoft` / Compile / packageBin) .dependsOn(`std-tableau` / Compile / packageBin) .dependsOn(`std-saas` / Compile / packageBin) + .dependsOn(`std-duckdb` / Compile / packageBin) .value ) .dependsOn(`common-polyglot-core-utils`) @@ -3931,6 +3937,9 @@ lazy val `engine-runner` = project .listFiles("*.jar") .map(_.getAbsolutePath()) ++ `std-tableau-polyglot-root` + .listFiles("*.jar") + .map(_.getAbsolutePath()) ++ + `std-duckdb-polyglot-root` .listFiles("*.jar") .map(_.getAbsolutePath()) ++ (if ( GraalVM.EnsoLauncher.disableMicrosoft @@ -4063,7 +4072,8 @@ lazy val `engine-runner` = project "com.tableau.hyperapi", // See https://github.com/HarrDevY/native-register-bouncy-castle "org.bouncycastle.jcajce.provider.drbg.DRBG$Default", - "org.bouncycastle.jcajce.provider.drbg.DRBG$NonceAndIV" + "org.bouncycastle.jcajce.provider.drbg.DRBG$NonceAndIV", + "org.duckdb" ), initializeAtBuildtime = NativeImage.defaultBuildTimeInitClasses ++ Seq( @@ -5025,6 +5035,10 @@ val `std-tableau-native-libs` = stdLibComponentRoot("Tableau") / "polyglot" / "lib" val `std-saas-polyglot-root` = stdLibComponentRoot("Saas") / "polyglot" / "java" +val `std-duckdb-polyglot-root` = + stdLibComponentRoot("DuckDB") / "polyglot" / "java" +val `std-duckdb-native-libs` = + stdLibComponentRoot("DuckDB") / "polyglot" / "lib" lazy val `std-base` = project .in(file("std-bits") / "base") @@ -5518,6 +5532,31 @@ lazy val `sqlite-wrapper` = project ) ) +lazy val `duckdb-wrapper` = project + .in(file("lib/java/duckdb-wrapper")) + .enablePlugins(JarExtractPlugin) + .settings( + frgaalJavaCompilerSetting, + autoScalaLibrary := false, + libraryDependencies ++= Seq( + "org.duckdb" % "duckdb_jdbc" % duckdbVersion + ), + inputJar := "org.duckdb" % "duckdb_jdbc" % duckdbVersion, + version := "0.1", + jarExtractor := JarExtractor( + "libduckdb_java.so_linux_amd64" -> PolyglotLib(LinuxAMD64), + "libduckdb_java.so_osx_universal" -> PolyglotLib(MacOSArm64), + "libduckdb_java.so_osx_universal" -> PolyglotLib(MacOSAMD64), + "libduckdb_java.so_windows_amd64" -> PolyglotLib(WindowsAMD64), + "META-INF/**" -> CopyToOutputJar, + "org/**/*.class" -> CopyToOutputJar + ), + inputJarResolved := assembly.value, + assemblyMergeStrategy := { case _ => + MergeStrategy.preferProject + } + ) + lazy val `std-image` = project .in(file("std-bits") / "image") .settings( @@ -6045,6 +6084,52 @@ lazy val `std-saas` = project .dependsOn(`std-base` % "provided") .dependsOn(`std-table` % "provided") +lazy val `std-duckdb` = project + .in(file("std-bits") / "duckdb") + .settings( + frgaalJavaCompilerSetting, + autoScalaLibrary := false, + Compile / compile / compileInputs := (Compile / compile / compileInputs) + .dependsOn(SPIHelpers.ensureSPIConsistency) + .value, + Compile / packageBin / artifactPath := + `std-duckdb-polyglot-root` / "std-duckdb.jar", + libraryDependencies ++= Seq( + "org.duckdb" % "duckdb_jdbc" % duckdbVersion % "provided" + ), + Compile / packageBin := { + val stdDuckDBJar = (Compile / packageBin).value + val cacheStoreFactory = streams.value.cacheStoreFactory + StdBits + .copyDependencies( + `std-duckdb-polyglot-root`, + Seq("std-duckdb.jar"), + ignoreScalaLibrary = true, + libraryUpdates = (Compile / update).value, + unmanagedClasspath = (Compile / unmanagedClasspath).value, + polyglotLibDir = Some(`std-duckdb-native-libs`), + ignoreDependencies = None, + extractedNativeLibsDirs = Seq( + (`duckdb-wrapper` / extractedFilesDir).value + ), + extraJars = Seq( + (`duckdb-wrapper` / thinJarOutput).value + ), + logger = streams.value.log, + cacheStoreFactory = cacheStoreFactory + ) + stdDuckDBJar + }, + clean := Def.task { + val _ = clean.value + IO.delete(`std-duckdb-polyglot-root`) + IO.delete(`std-duckdb-native-libs`) + }.value + ) + .dependsOn(`std-base` % "provided") + .dependsOn(`std-table` % "provided") + .dependsOn(`std-database` % "provided") + lazy val fetchZipToUnmanaged = taskKey[Seq[Attributed[File]]]( "Download zip file from an `unmanagedExternalZip` url and unpack jars to unmanaged libs directory" @@ -6353,7 +6438,8 @@ val stdBitsProjects = "Microsoft", "Snowflake", "Table", - "Saas" + "Saas", + "DuckDB" ) ++ allStdBitsSuffix val allStdBits: Parser[String] = stdBitsProjects.map(v => v: Parser[String]).reduce(_ | _) @@ -6434,6 +6520,8 @@ pkgStdLibInternal := Def.inputTask { (`std-tableau` / Compile / packageBin).value case "Saas" => (`std-saas` / Compile / packageBin).value + case "DuckDB" => + (`std-duckdb` / Compile / packageBin).value case _ if buildAllCmd => (`std-base` / Compile / packageBin).value (`enso-test-java-helpers` / Compile / packageBin).value @@ -6451,6 +6539,7 @@ pkgStdLibInternal := Def.inputTask { (`std-microsoft` / Compile / packageBin).value (`std-tableau` / Compile / packageBin).value (`std-saas` / Compile / packageBin).value + (`std-duckdb` / Compile / packageBin).value case _ => } val libs = diff --git a/build_tools/build/src/ci_gen.rs b/build_tools/build/src/ci_gen.rs index f75b6c909339..fb945e8d4721 100644 --- a/build_tools/build/src/ci_gen.rs +++ b/build_tools/build/src/ci_gen.rs @@ -955,6 +955,7 @@ fn stdlib_api_change_labels_workflow() -> Result { "AWS", "Base", "Database", + "DuckDB", "Generic_JDBC", "Google", "Image", diff --git a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md index 4dbafcfc1fab..a7bb509073d3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md +++ b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md @@ -37,4 +37,5 @@ - make_schema_selector connection:Standard.Base.Any.Any include_any:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any - make_structure_creator -> Standard.Base.Any.Any - make_table_name_selector connection:Standard.Base.Any.Any -> Standard.Base.Any.Any +- make_table_name_with_schema_selector connection:Standard.Base.Any.Any schema_black_list:Standard.Base.Any.Any= -> Standard.Base.Any.Any - make_table_types_selector connection:Standard.Base.Any.Any -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md index ffdb0d1d90a1..3bfcba137324 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md +++ b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md @@ -16,7 +16,7 @@ - with_metadata self ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any - with_prepared_statement self query:Standard.Base.Any.Any statement_setter:Standard.Base.Any.Any action:Standard.Base.Any.Any skip_log:Standard.Base.Any.Any= -> Standard.Base.Any.Any - close_connection connection:Standard.Base.Any.Any -> Standard.Base.Any.Any -- create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any -> Standard.Base.Any.Any +- create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= -> Standard.Base.Any.Any - from_java java_jdbc_connection:Standard.Base.Any.Any sql_exception:Standard.Base.Any.Any= -> Standard.Base.Any.Any - get_pragma_value jdbc_connection:Standard.Base.Any.Any sql:Standard.Base.Any.Any -> Standard.Base.Any.Any - handle_sql_errors ~action:Standard.Base.Any.Any related_query:Standard.Base.Any.Any= sql_exception:Standard.Base.Any.Any= -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL_Query.md b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL_Query.md index d73af55bf45a..be162adc9114 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL_Query.md +++ b/distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL_Query.md @@ -4,6 +4,12 @@ - Raw_SQL sql:Standard.Base.Data.Text.Text= - Table_Name name:Standard.Base.Data.Text.Text= - to_db_table self connection:Standard.Database.Connection.Connection.Connection alias:Standard.Base.Data.Text.Text -> Standard.Base.Any.Any +- type SQL_Query_With_Schema + - Raw_SQL sql:Standard.Base.Data.Text.Text= + - Table_Name name:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= + - From that:Standard.Database.SQL_Query.SQL_Query -> Standard.Base.Any.Any + - to_db_table self connection:Standard.Database.Connection.Connection.Connection alias:Standard.Base.Data.Text.Text -> Standard.Base.Any.Any - make_table_for_name connection:Standard.Base.Any.Any name:Standard.Base.Any.Any schema:Standard.Base.Any.Any alias:Standard.Base.Any.Any internal_temporary_keep_alive_reference:Standard.Base.Any.Any= -> Standard.Base.Any.Any - make_table_from_query connection:Standard.Base.Any.Any query:(Standard.Base.Data.Text.Text|Standard.Database.SQL.SQL_Statement) alias:Standard.Base.Data.Text.Text -> (Standard.Table.Table.Table&Standard.Database.DB_Table.DB_Table) - Standard.Database.SQL_Query.SQL_Query.from that:Standard.Base.Data.Text.Text -> Standard.Database.SQL_Query.SQL_Query +- Standard.Database.SQL_Query.SQL_Query_With_Schema.from that:Standard.Base.Data.Text.Text -> Standard.Database.SQL_Query.SQL_Query_With_Schema diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso index 4b1ea54f9cb2..9354f8b9b9ac 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso @@ -593,6 +593,20 @@ make_table_name_selector connection = values = table_name_values + [Option '' '..Table_Name ""', Option '' '..Raw_SQL ""'] Single_Choice display=Display.Always values=values +## --- + private: true + --- +make_table_name_with_schema_selector connection schema_black_list=[] = + schema = connection.schema + schema_text sch = if sch == schema then "" else sch+"." + tables = connection.tables schema="*" . filter "Schema" (..Is_In schema_black_list ..Remove) + table_name_values = tables.rows.map r-> + name = r.at "Name" + sch = r.at "Schema" + Option ((schema_text sch) + name) "(..Table_Name "+name.pretty+(if sch == schema then "" else " "+sch.pretty)+")" + values = table_name_values + [Option '' '..Table_Name ""', Option '' '..Raw_SQL ""'] + Single_Choice display=..Always values=values + ## --- private: true --- diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso index beb0e6610853..5ac3d87a3b97 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso @@ -280,7 +280,6 @@ type Postgres_Connection execute_query self query (limit : Rows_To_Read = ..First_With_Warning 1000) write_operation:Boolean=True = self.connection.execute_query query limit write_operation - ## --- advanced: true aliases: [sql] diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso index 5fca4185b347..6fd265d768cf 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso @@ -292,9 +292,11 @@ type JDBC_Connection ## Arguments - `url`: The URL to connect to. - `properties`: A vector of properties for the connection. -create : Text -> Vector -> JDBC_Connection -create url properties = handle_sql_errors <| - java_connection = JDBCProxy.getConnection url (properties_as_java_props properties) + - `database`: The database name to connect to. + - `schema`: The schema name to connect to. +create : Text -> Vector -> Text -> Text -> JDBC_Connection +create url properties database:Text="" schema:Text="" = handle_sql_errors <| + java_connection = JDBCProxy.getConnectionWithCatalogSchema url (properties_as_java_props properties) database schema from_java java_connection ## --- diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Main.enso index d43b5162bbe6..3842de9876e1 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Main.enso @@ -20,6 +20,7 @@ export project.Extensions.Upload_Table.select_into_database_table export project.Extensions.Upload_Table.update_rows export project.SQL_Query.SQL_Query +export project.SQL_Query.SQL_Query_With_Schema export project.Update_Action.Update_Action diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/SQL_Query.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/SQL_Query.enso index 11a64308fa78..cac332642afc 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/SQL_Query.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/SQL_Query.enso @@ -34,11 +34,43 @@ type SQL_Query connection.dialect.ensure_query_has_no_holes connection.jdbc_connection raw_sql . if_not_error <| make_table_from_query connection raw_sql alias +## Enhanced SQL_Query that also carries schema information. +type SQL_Query_With_Schema + ## Query a whole table or view. + Table_Name (name : Text = Missing_Argument.throw "name") (schema : Text = "") + + ## Raw SQL query statement. + Raw_SQL (sql : Text = Missing_Argument.throw "sql") + + ## --- + private: true + --- + to_db_table self connection:Connection alias:Text = alias.if_not_error <| case self of + SQL_Query_With_Schema.Table_Name name schema -> + table_naming_helper = connection.base_connection.table_naming_helper + table_naming_helper.verify_table_name name <| + make_table_for_name connection name schema alias + SQL_Query_With_Schema.Raw_SQL raw_sql -> handle_sql_errors <| + connection.dialect.ensure_query_has_no_holes connection.jdbc_connection raw_sql . if_not_error <| + make_table_from_query connection raw_sql alias + ## --- private: true --- SQL_Query.from (that:Text) = SQL_Query.Table_Name that +## --- + private: true + --- +SQL_Query_With_Schema.from (that:Text) = SQL_Query_With_Schema.Table_Name that + +## --- + private: true + --- +SQL_Query_With_Schema.From (that:SQL_Query) = case that of + SQL_Query.Table_Name name -> SQL_Query_With_Schema.Table_Name name "" + SQL_Query.Raw_SQL sql -> SQL_Query_With_Schema.Raw_SQL sql + ## --- private: true --- diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/DuckDB/0.0.0-dev/THIRD-PARTY/NOTICE new file mode 100644 index 000000000000..8bde48c4df03 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/THIRD-PARTY/NOTICE @@ -0,0 +1,2 @@ +Enso +Copyright 2020 - 2025 New Byte Order sp. z o. o. diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB.md new file mode 100644 index 000000000000..f60f254a2469 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB.md @@ -0,0 +1,9 @@ +## Enso Signatures 1.0 +## module Standard.DuckDB.DuckDB +- type DuckDB + - From_File location:Standard.Base.System.File.File= schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean= + - In_Memory schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean= + - connect self options:Standard.Base.Any.Any -> Standard.Base.Any.Any + - jdbc_properties self -> Standard.Base.Any.Any + - jdbc_url self -> Standard.Base.Any.Any + - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md new file mode 100644 index 000000000000..2265206347d6 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md @@ -0,0 +1,28 @@ +## Enso Signatures 1.0 +## module Standard.DuckDB.DuckDB_Connection +- type DuckDB_Connection + - base_connection self -> Standard.Base.Any.Any + - close self -> Standard.Base.Any.Any + - create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any make_new:Standard.Base.Any.Any schema:Standard.Base.Any.Any -> Standard.Base.Any.Any + - create_table self table_name:Standard.Base.Data.Text.Text structure:(Standard.Base.Any.Any|Standard.Table.Table.Table) primary_key:(Standard.Base.Any.Any|Standard.Base.Nothing.Nothing)= temporary:Standard.Base.Data.Boolean.Boolean= allow_existing:Standard.Base.Data.Boolean.Boolean= on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior= -> Standard.Base.Any.Any + - database self -> Standard.Base.Any.Any + - databases self -> Standard.Base.Any.Any + - dialect self -> Standard.Base.Any.Any + - drop_table self table_name:Standard.Base.Any.Any if_exists:Standard.Base.Any.Any= -> Standard.Base.Any.Any + - execute self query:Standard.Base.Any.Any -> Standard.Base.Any.Any + - execute_query self query:Standard.Base.Any.Any limit:Standard.Table.Rows_To_Read.Rows_To_Read= write_operation:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any + - execute_update self query:Standard.Base.Any.Any -> Standard.Base.Any.Any + - jdbc_connection self -> Standard.Base.Any.Any + - query self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) alias:Standard.Base.Any.Any= -> Standard.Base.Any.Any + - read self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) limit:Standard.Table.Rows_To_Read.Rows_To_Read= -> Standard.Base.Any.Any + - schema self -> Standard.Base.Any.Any + - schemas self -> Standard.Base.Any.Any + - set_database self database:Standard.Base.Any.Any -> Standard.Base.Any.Any + - set_schema self schema:Standard.Base.Any.Any -> Standard.Base.Any.Any + - table_types self -> Standard.Base.Any.Any + - tables self name_like:Standard.Base.Data.Text.Text= database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= types:Standard.Base.Any.Any= all_fields:Standard.Base.Any.Any= -> Standard.Base.Any.Any + - to_js_object self -> Standard.Base.Any.Any + - truncate_table self table_name:Standard.Base.Any.Any -> Standard.Base.Any.Any + - version self -> Standard.Base.Data.Text.Text +- schema_black_list -> Standard.Base.Any.Any +- Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.DuckDB.DuckDB_Connection.DuckDB_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Internal/DuckDB_Entity_Naming_Properties.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Internal/DuckDB_Entity_Naming_Properties.md new file mode 100644 index 000000000000..c513677d47f7 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Internal/DuckDB_Entity_Naming_Properties.md @@ -0,0 +1,3 @@ +## Enso Signatures 1.0 +## module Standard.DuckDB.Internal.DuckDB_Entity_Naming_Properties +- new -> Standard.Base.Any.Any diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Main.md b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Main.md new file mode 100644 index 000000000000..ddc66838a1c7 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Main.md @@ -0,0 +1,2 @@ +## Enso Signatures 1.0 +## module Standard.DuckDB.Main diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/package.yaml b/distribution/lib/Standard/DuckDB/0.0.0-dev/package.yaml new file mode 100644 index 000000000000..085b3cd0304a --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/package.yaml @@ -0,0 +1,14 @@ +name: DuckDB +namespace: Standard +version: 0.0.0-dev +license: APLv2 +authors: + - name: Enso Team + email: contact@enso.org +maintainers: + - name: Enso Team + email: contact@enso.org + +services: + - provides: Standard.Database.Connection.Database.Database_Connection_Details_SPI + with: Standard.DuckDB.Internal.DuckDB_Services.Impl diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB.enso new file mode 100644 index 000000000000..f763156b1d9e --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB.enso @@ -0,0 +1,72 @@ +from Standard.Base import all +import Standard.Base.Errors.Common.Type_Error + +import Standard.Database.Connection.Connection_Options.Connection_Options + +import project.DuckDB_Connection.DuckDB_Connection + +polyglot java import org.duckdb.DuckDBDriver + +type DuckDB + ## Connect to a DuckDB database file. + + ## Arguments: + - 'location': The file path to the DuckDB database file. + - 'schema': The schema to use within the database. Default is 'main'. + - 'read_only': Whether to open the database in read-only mode. Default is false. + From_File (location : File = (Missing_Argument.throw "location")) (schema : Text = "") (read_only : Boolean = False) + + ## Connect to a DuckDB In Memory. + + ## Arguments: + - 'schema': The schema to use within the database. Default is 'main'. + - 'read_only': Whether to open the database in read-only mode. Default is false. + In_Memory (schema : Text = "") (read_only : Boolean = False) + + ## --- + private: true + --- + Attempt to resolve the constructor. + resolve : Function -> DuckDB | Nothing + resolve constructor = + Panic.catch Type_Error (constructor:DuckDB) _->Nothing + + ## --- + private: true + --- + Build the Connection resource. + + ## Arguments + - `options`: Overrides for the connection properties. + connect : Connection_Options -> DuckDB_Connection + connect self options = + properties = options.merge self.jdbc_properties + + make_new schema = + used_schema = (if schema.is_nothing then "" else schema).trim + + new_settings = case self of + DuckDB.From_File f _ r -> DuckDB.From_File f used_schema r + DuckDB.In_Memory _ r -> DuckDB.In_Memory used_schema r + new_settings.connect options + + DuckDB_Connection.create self.jdbc_url properties make_new self.schema + + ## --- + private: true + --- + Provides the jdbc url for the connection. + jdbc_url : Text + jdbc_url self = case self of + DuckDB.From_File _ _ _ -> "jdbc:duckdb:" + (self.location.absolute.path.replace '\\' '/') + DuckDB.In_Memory _ _ -> "jdbc:duckdb::memory:" + + ## --- + private: true + --- + Provides the properties for the connection. + jdbc_properties : Vector + jdbc_properties self = case self of + DuckDB.From_File _ _ True -> [Pair.new "duckdb.read_only" "true"] + DuckDB.In_Memory _ True -> [Pair.new "duckdb.read_only" "true"] + _ -> [] diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso new file mode 100644 index 000000000000..1801b955f963 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/DuckDB_Connection.enso @@ -0,0 +1,406 @@ +from Standard.Base import all +from Standard.Base.Metadata.Choice import Option +from Standard.Base.Metadata.Widget import Single_Choice, Text_Input +import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data +import Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Header + +import Standard.Table.Rows_To_Read.Rows_To_Read +from Standard.Table import Table + +from Standard.Database import Column_Description, DB_Table, SQL_Query, SQL_Query_With_Schema +import Standard.Database.Connection.Connection.Connection +import Standard.Database.Dialects.Dialect +import Standard.Database.Internal.JDBC_Connection +import Standard.Database.SQL.SQL_Statement +from Standard.Database.Connection.Connection import make_database_selector, make_schema_selector, make_structure_creator, make_table_name_with_schema_selector, make_table_types_selector +from Standard.Database.Errors import SQL_Error, Table_Already_Exists, Table_Not_Found +from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_column_name_in_structure + +import project.Internal.DuckDB_Entity_Naming_Properties + +polyglot java import java.sql.SQLException +polyglot java import org.enso.database.JDBCProxy +polyglot java import org.enso.duckdb.DuckDBUtils +polyglot java import org.graalvm.collections.Pair as Java_Pair + +from Standard.Base.Enso_Cloud.Enso_Secret import as_hideable_value +polyglot java import org.enso.base.enso_cloud.HideableValue + +type DuckDB_Connection + ## --- + private: true + --- + Creates a DuckDB connection based on a URL and properties. + + ## Arguments + - `url`: The URL to connect to. + - `properties`: A vector of properties for the connection. + - `make_new`: A function that returns a new connection (Schema->DuckDB). + - `schema`: The initial schema to connect to. + create : Text -> Vector -> (Text -> Text -> DuckDB_Connection) -> Text -> DuckDB_Connection + create url properties make_new schema = + properties_as_java_props properties = + properties.map pair-> + # Some parameters may be passed by the dialect as a `HideableValue` directly, so they do not need to be converted. + value = pair.second + java_value = if value.is_a HideableValue then value else + as_hideable_value value factory=HideableValue + Java_Pair.create pair.first java_value + + java_connection = JDBC_Connection.handle_sql_errors <| + props = properties_as_java_props properties + JDBCProxy.getConnectionWithCatalogSchema url props Nothing schema + jdbc_connection = JDBC_Connection.from_java sql_exception=SQLException java_connection + + ## For initial work using DuckDB, we will use the Postgres dialect. + DuckDB_Connection.Value (Connection.new jdbc_connection Dialect.postgres DuckDB_Entity_Naming_Properties.new) make_new + + ## --- + private: true + --- + A DuckDB database connection. + + ## Arguments + - `connection`: the underlying connection. + - `make_new`: a function that returns a new connection. + private Value (connection:Connection) (make_new : Text -> Text -> DuckDB_Connection) + + ## Gets the DuckDB Version string. + version self -> Text = + self.jdbc_connection.with_connection connection-> + DuckDBUtils.getVersion connection + + ## --- + icon: close + --- + Closes the connection releasing the underlying database resources + immediately instead of waiting for them to be automatically released. + The connection is not usable afterwards. + close : Nothing + close self = self.connection.close + + ## --- + icon: metadata + --- + Returns the list of databases (or catalogs) for the connection. + databases : Vector Text + databases self = self.connection.databases + + ## --- + icon: metadata + --- + Returns the name of the current database (or catalog). + database : Text + database self = self.connection.database + + ## --- + icon: data_input + --- + Returns a new Connection with the specified database set as default. + + ## Arguments + - database: The name of the database to connect to. + @database (self-> Single_Choice display=..Always values=(self.databases . map d-> Option d d.pretty)) + set_database : Text -> Connection ! SQL_Error + set_database self database = + if database . equals_ignore_case self.database then self else + SQL_Error.throw_sql_error "Changing the database is not supported for DuckDB." + + ## --- + icon: metadata + --- + Returns the list of schemas for the connection within the current database + (or catalog). + schemas : Vector Text + schemas self = self.connection.schemas.distinct + + ## --- + icon: metadata + --- + Returns the name of the current schema. + schema : Text + schema self = self.connection.schema + + ## --- + icon: data_input + --- + Returns a new Connection with the specified schema set as default. + + ## Arguments + - schema: The name of the schema to connect to. + @schema make_schema_selector + set_schema : Text -> Connection ! SQL_Error + set_schema self schema = + if schema . equals_ignore_case self.schema then self else + self.make_new schema + + ## --- + group: Standard.Base.Metadata + icon: metadata + --- + Gets a list of the table types. + table_types : Vector Text + table_types self = self.connection.table_types + + ## --- + group: Standard.Base.Metadata + icon: metadata + --- + Returns a materialized Table of all the matching views and tables. + + ## Arguments + - `name_like`: The table name pattern to search for. Supports SQL + wildcards (`%`, `_`). Defaults to `""` which means all tables are + selected. + - `database`: The database name to search in (default is current). + - `schema`: The schema name to search in (defaults to current). If "*" is + provided, all schemas are searched. + - `types`: The table types to search for. The list of possible values can + be obtained using the `table_types` method. Defaults to a set of most + commonly used table types, ignoring internal system tables or indices. + - `all_fields`: Return all the fields in the metadata table. + + ## Remarks + + ### Temporary Tables + Note that the temporary tables may be created in a different schema than + the current one, so take this into account when filtering by schema. + @types make_table_types_selector + @database (make_database_selector include_any=True) + @schema (make_schema_selector include_any=True) + tables : Text -> Text -> Text -> Vector -> Boolean -> Table + tables self name_like:Text="" database:Text=self.database schema:Text=self.schema types=["BASE TABLE", "VIEW"] all_fields=False = + parsed_database = if database == "*" then Nothing else (if database == "" then self.database else database) + parsed_schema = if schema == "*" then Nothing else (if schema == "" then self.schema else schema) + self.connection.tables (if name_like == "" then Nothing else name_like) parsed_database parsed_schema types all_fields + + ## --- + aliases: [import, load, open, read, sql] + group: Standard.Base.Input + icon: data_input + --- + Set up a query returning a Table object, which can be used to work with + data within the database or load it into memory. + + ## Arguments + - `query`: name of the table or sql statement to query. + If supplied as `Text` it is treated as a table name. + - `alias`: optionally specify a friendly alias for the query. + + ## Errors + - If provided with a `Raw_SQL` query or `Text` that looks like a query, + if any SQL error occurs when executing the query, a `SQL_Error` error + is raised. + - If provided with a `Table_Name` or a text short-hand and the table is + not found, a `Table_Not_Found` error is raised. + @query make_table_name_with_schema_selector schema_black_list=schema_black_list + query : (SQL_Query_With_Schema | SQL_Query) -> Text -> Table & DB_Table ! Table_Not_Found + query self query:(SQL_Query_With_Schema | SQL_Query) alias="" = + used_query = case query of + SQL_Query.Table_Name name -> SQL_Query_With_Schema.Table_Name name "" + SQL_Query.Raw_SQL sql -> SQL_Query_With_Schema.Raw_SQL sql + _ : SQL_Query_With_Schema -> query + self.connection.query used_query alias + + ## --- + aliases: [import, load, open, query, sql] + group: Standard.Base.Input + icon: data_input + --- + Execute the query and load the results into memory as a Table. + + ## Arguments + - `query`: name of the table or sql statement to query. If supplied as + `Text` it is treated as a table name. + - `limit`: the maximum number of rows to read. + + ## Remarks + + ### Side Effects + Note that the `read` method is running without restrictions when the + output context is disabled, but it can technically cause side effects, if + it is provided with a DML query. Usually it is preferred to use + `execute_update` for DML queries, or if they are supposed to return + results, the `read` should be wrapped in an execution context check. + @query make_table_name_with_schema_selector schema_black_list=schema_black_list + @limit Rows_To_Read.default_widget + read : (SQL_Query_With_Schema | SQL_Query) -> Rows_To_Read -> Table ! Table_Not_Found + read self query:(SQL_Query_With_Schema | SQL_Query) (limit : Rows_To_Read = ..First_With_Warning 1000) = + self.query query . read limit + + ## --- + group: Standard.Base.Output + icon: data_output + --- + Creates a new empty table in the database and returns a query referencing + the new table. + + ## Arguments + - `table_name`: the name of the table to create. + - `structure`: the structure of the table, provided as either an existing + `Table` (no data will be copied) or a `Vector` of `Column_Description`. + - `primary_key`: the names of the columns to use as the primary key. The + first column from the table is used by default. If it is set to + `Nothing` or an empty vector, no primary key will be created. + - `temporary`: if set to `True`, the table will be temporary, meaning + that it will be dropped once the `connection` is closed. Defaults to + `False`. + - `allow_existing`: Defaults to `False`, meaning that if the table with + the provided name already exists, an error will be raised. If set to + `True`, the existing table will be returned instead. Note that the + existing table is not guaranteed to have the same structure as the one + provided. + - `on_problems`: the behavior to use when encountering non-fatal + problems. Defaults to reporting them as warning. + + ## Errors + - If a table with the given name already exists, then a + `Table_Already_Exists` error is raised. + - If a column type is not supported and is coerced to a similar supported + type, an `Inexact_Type_Coercion` problem is reported according to the + `on_problems` setting. + - If a column type is not supported and there is no replacement (e.g. + native Enso types), an `Unsupported_Type` error is raised. + - If the provided primary key columns are not present in table structure + provided, `Missing_Input_Columns` error is raised. + - An `SQL_Error` may be reported if there is a failure on the database + side. + + ## Remarks + + ### Dry Run if Output disabled + If performing output actions is disabled, only a dry run is performed and + no permanent changes occur. The operation will test for errors (like + missing columns) and if successful, return a temporary table with a + `Dry_Run_Operation` warning attached. + @structure make_structure_creator + create_table : Text -> Vector Column_Description | Table -> Vector Text | Nothing -> Boolean -> Boolean -> Problem_Behavior -> Table & DB_Table ! Table_Already_Exists + create_table self (table_name : Text) (structure : Vector Column_Description | Table) (primary_key : (Vector Text | Nothing) = [first_column_name_in_structure structure]) (temporary : Boolean = False) (allow_existing : Boolean = False) (on_problems:Problem_Behavior = ..Report_Warning) = + self.connection.create_table table_name structure primary_key temporary allow_existing on_problems + + ## --- + advanced: true + aliases: [sql] + icon: data_input + --- + Executes a raw query returning a Table object, which can be used to get + the results of the query. This is meant for advanced users who need to + call direct SQL queries that are not supported by the `query` method. + + ## Arguments + - `query`: either raw SQL code as Text or an instance of SQL_Statement + representing the query to execute. + - `write_operation`: if set to `True`, the query is expected to be a + write operation and will only run if the output context is enabled. If + set to `False`, the query is expected to be a read operation and will + run regardless of the output context. Defaults to `True`. + @query Text_Input display=..Always + @limit Rows_To_Read.default_widget + execute_query : Text | SQL_Statement -> Rows_To_Read -> Boolean -> Table + execute_query self query (limit : Rows_To_Read = ..First_With_Warning 1000) write_operation:Boolean=True = + self.connection.execute_query query limit write_operation + + ## --- + advanced: true + aliases: [sql] + group: Standard.Base.Output + icon: data_output + --- + Executes a raw update query. If the query was inserting, updating or + deleting rows, the number of affected rows is returned; otherwise it + returns 0 for other types of queries (like creating or altering tables). + + ## Arguments + - `query`: either raw SQL code as Text or an instance of SQL_Statement + representing the query to execute. + execute_update : Text | SQL_Statement -> Integer + execute_update self query = + self.connection.execute_update query + + ## --- + private: true + advanced: true + group: Standard.Base.Output + icon: data_output + --- + Executes a raw query. If the query was inserting, updating or deleting + rows, the number of affected rows is returned; otherwise it returns 0 for + other types of queries (like creating or altering tables). + + ## Arguments + - `query`: either raw SQL code as Text or an instance of SQL_Statement + representing the query to execute. + execute : Text | SQL_Statement -> Integer + execute self query = + self.connection.execute query + + ## --- + private: true + --- + Access the dialect. + dialect self = self.connection.dialect + + ## --- + private: true + --- + Access the underlying JDBC connection. + jdbc_connection self = self.connection.jdbc_connection + + ## --- + private: true + --- + Drops a table. + + ## Arguments + - `table_name`: the name of the table to drop. + - `if_exists`: if set to `True`, the operation will not fail if the table + does not exist. Defaults to `False`. + drop_table : Text -> Boolean -> Nothing + drop_table self table_name if_exists=False = + self.connection.drop_table table_name if_exists + + ## --- + private: true + --- + Removes all rows from a table. + + ## Arguments + - `table_name`: the name of the table to truncate. + truncate_table : Text -> Nothing ! Table_Not_Found + truncate_table self table_name = + self.connection.truncate_table table_name + + ## --- + private: true + --- + Returns the base `Connection` instance. + Used, so that all internal helper functions do not need to be replicated + on the 'subclasses'. + base_connection : Connection + base_connection self = self.connection + + ## --- + private: true + --- + Converts this value to a JSON serializable object. + to_js_object : JS_Object + to_js_object self = + JS_Object.from_pairs <| [["type", "DuckDB_Connection"], ["links", self.connection.tables.at "Name" . to_vector]] + +## --- + private: true + --- +schema_black_list = + ["temp", "system"] + +## --- + private: true + --- +Table_Viz_Data.from (that:DuckDB_Connection) = + tables = that.tables schema="*" . filter "Schema" (..Is_In schema_black_list ..Remove) + multi_schema = tables.at "Schema" . to_vector . any s-> s != that.schema + case multi_schema of + True -> + Table_Viz_Data.GenericGrid [Table_Viz_Header.Link "Table" "table" "query (..Table_Name {{@Table}} {{@Schema}})", Table_Viz_Header.Label "Schema"] [tables.at "Name" . to_vector, tables.at "Schema" . to_vector] + False -> + Table_Viz_Data.GenericGrid [Table_Viz_Header.Link "Table" "table" "query (..Table_Name {{@Table}})"] [tables.at "Name" . to_vector] diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Entity_Naming_Properties.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Entity_Naming_Properties.enso new file mode 100644 index 000000000000..0fef121fe05d --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Entity_Naming_Properties.enso @@ -0,0 +1,17 @@ +private + +from Standard.Base import all + +import Standard.Table.Internal.Naming_Properties.Enso_Length_Limited_Naming_Properties +import Standard.Table.Internal.Naming_Properties.Unlimited_Naming_Properties + +import Standard.Database.Internal.Connection.Entity_Naming_Properties.Entity_Naming_Properties + +## --- + private: true + --- +new : Entity_Naming_Properties +new = + default = Unlimited_Naming_Properties.Instance is_case_sensitive=False + limited = Enso_Length_Limited_Naming_Properties.Instance limit=255 is_case_sensitive=False + Entity_Naming_Properties.Value for_table_names=default for_column_names=default for_generated_column_names=limited diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Services.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Services.enso new file mode 100644 index 000000000000..085c2e41f3a6 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Internal/DuckDB_Services.enso @@ -0,0 +1,11 @@ +private + +import Standard.Database.Connection.Database.Database_Connection_Details_SPI +import project.DuckDB.DuckDB + +type Impl + +Database_Connection_Details_SPI.from (_:Impl) = + file = Database_Connection_Details_SPI.new DuckDB "DuckDB" "DuckDB.From_File" + memory = Database_Connection_Details_SPI.new DuckDB "DuckDB (In-Memory)" "DuckDB.In_Memory" + file+memory diff --git a/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Main.enso new file mode 100644 index 000000000000..f96335271603 --- /dev/null +++ b/distribution/lib/Standard/DuckDB/0.0.0-dev/src/Main.enso @@ -0,0 +1 @@ +export project.DuckDB.DuckDB \ No newline at end of file diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/SQLServer_Connection.md b/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/SQLServer_Connection.md index 66da31cae484..7dbbe8e34ffd 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/SQLServer_Connection.md +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/docs/api/SQLServer_Connection.md @@ -24,6 +24,5 @@ - tables self name_like:Standard.Base.Data.Text.Text= database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= types:Standard.Base.Any.Any= all_fields:Standard.Base.Any.Any= -> Standard.Base.Any.Any - to_js_object self -> Standard.Base.Any.Any - truncate_table self table_name:Standard.Base.Any.Any -> Standard.Base.Any.Any -- make_table_name_selector connection:Standard.Base.Any.Any -> Standard.Base.Any.Any - schema_black_list -> Standard.Base.Any.Any - Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.Microsoft.SQLServer_Connection.SQLServer_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/SQLServer_Connection.enso b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/SQLServer_Connection.enso index c8a4fe1466ea..d4f9afbe845c 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/src/SQLServer_Connection.enso +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/src/SQLServer_Connection.enso @@ -18,7 +18,7 @@ import Standard.Database.Internal.Data_Link_Setup.Data_Link_Setup import Standard.Database.Internal.JDBC_Connection import Standard.Database.SQL.SQL_Statement import Standard.Database.SQL_Query.SQL_Query -from Standard.Database.Connection.Connection import make_database_selector, make_schema_selector, make_structure_creator, make_table_types_selector +from Standard.Database.Connection.Connection import make_database_selector, make_schema_selector, make_structure_creator, make_table_name_with_schema_selector, make_table_types_selector from Standard.Database.Errors import SQL_Error, Table_Already_Exists, Table_Not_Found from Standard.Database.Internal.Upload.Helpers.Default_Arguments import first_column_name_in_structure @@ -199,7 +199,7 @@ type SQLServer_Connection is raised. - If provided with a `Table_Name` or a text short-hand and the table is not found, a `Table_Not_Found` error is raised. - @query make_table_name_selector + @query make_table_name_with_schema_selector schema_black_list=schema_black_list query : (SQLServer_Query | SQL_Query) -> Text -> Table & DB_Table ! Table_Not_Found query self query:(SQLServer_Query | SQL_Query) alias="" = used_query = case query of @@ -228,15 +228,11 @@ type SQLServer_Connection it is provided with a DML query. Usually it is preferred to use `execute_update` for DML queries, or if they are supposed to return results, the `read` should be wrapped in an execution context check. - @query make_table_name_selector + @query make_table_name_with_schema_selector schema_black_list=schema_black_list @limit Rows_To_Read.default_widget read : (SQLServer_Query | SQL_Query) -> Rows_To_Read -> Table ! Table_Not_Found read self query:(SQLServer_Query | SQL_Query) (limit : Rows_To_Read = ..First_With_Warning 1000) = - used_query = case query of - SQL_Query.Table_Name name -> SQLServer_Query.Table_Name name "" - SQL_Query.Raw_SQL sql -> SQLServer_Query.Raw_SQL sql - _ : SQLServer_Query -> query - self.connection.read used_query limit + self.query query . read limit ## --- group: Standard.Base.Output @@ -418,17 +414,3 @@ Table_Viz_Data.from (that:SQLServer_Connection) = --- schema_black_list = ["INFORMATION_SCHEMA", "sys", "db_accessadmin", "db_backupoperator", "db_datareader", "db_datawriter", "db_ddladmin", "db_denydatareader", "db_denydatawriter", "db_owner", "db_securityadmin"] - -## --- - private: true - --- -make_table_name_selector connection = - schema = connection.schema - schema_text sch = if sch == schema then "" else sch+"." - tables = connection.tables schema="*" . filter "Schema" (..Is_In schema_black_list ..Remove) - table_name_values = tables.rows.map r-> - name = r.at "Name" - sch = r.at "Schema" - Option ((schema_text sch) + name) "(..Table_Name "+name.pretty+(if sch == schema then "" else " "+sch.pretty)+")" - values = table_name_values + [Option '' '..Table_Name ""', Option '' '..Raw_SQL ""'] - Single_Choice display=..Always values=values diff --git a/lib/java/duckdb-wrapper/README.md b/lib/java/duckdb-wrapper/README.md new file mode 100644 index 000000000000..6e3a446772ef --- /dev/null +++ b/lib/java/duckdb-wrapper/README.md @@ -0,0 +1 @@ +Thin wrapper project for DuckDB jar extraction. diff --git a/lib/java/duckdb-wrapper/src/main/java/org/duckdb/DuckDBNative.java b/lib/java/duckdb-wrapper/src/main/java/org/duckdb/DuckDBNative.java new file mode 100644 index 000000000000..f26fd6e2fba1 --- /dev/null +++ b/lib/java/duckdb-wrapper/src/main/java/org/duckdb/DuckDBNative.java @@ -0,0 +1,178 @@ +package org.duckdb; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.sql.SQLException; +import java.util.Properties; + +/** + * A version of org.duckdb.DuckDBNative (v1.4.0.0) that attempts to first load a shared library from a name. + * If the library is not present on the shared library path, or has a malformed name, a fallback is used to + * infer it via a constructed file path. + */ +final class DuckDBNative { + static { + var libName = "duckdb_java"; + try { + System.loadLibrary(libName); + } catch (Throwable ex) { + // Fallback to using a fixed path to load a native library + try { + String os_name = ""; + String os_arch; + String os_name_detect = System.getProperty("os.name").toLowerCase().trim(); + String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim(); + switch (os_arch_detect) { + case "x86_64": + case "amd64": + os_arch = "amd64"; + break; + case "aarch64": + case "arm64": + os_arch = "arm64"; + break; + case "i386": + os_arch = "i386"; + break; + default: + throw new IllegalStateException("Unsupported system architecture"); + } + if (os_name_detect.startsWith("windows")) { + os_name = "windows"; + } else if (os_name_detect.startsWith("mac")) { + os_name = "osx"; + os_arch = "universal"; + } else if (os_name_detect.startsWith("linux")) { + os_name = "linux"; + } + String lib_res_name = "/libduckdb_java.so" + + "_" + os_name + "_" + os_arch; + + Path lib_file = Files.createTempFile("libduckdb_java", ".so"); + URL lib_res = DuckDBNative.class.getResource(lib_res_name); + if (lib_res == null) { + System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString()); + } else { + try (final InputStream lib_res_input_stream = lib_res.openStream()) { + Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING); + } + new File(lib_file.toString()).deleteOnExit(); + System.load(lib_file.toAbsolutePath().toString()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + // We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers + // back and forth + + /* + * NB: if you change anything below, run `javah` on this class to re-generate + * the C header. CMake does this as well + */ + + // results ConnectionHolder reference object + static native ByteBuffer duckdb_jdbc_startup(byte[] path, boolean read_only, Properties props) throws SQLException; + + // returns conn_ref connection reference object + static native ByteBuffer duckdb_jdbc_connect(ByteBuffer conn_ref) throws SQLException; + + static native ByteBuffer duckdb_jdbc_create_db_ref(ByteBuffer conn_ref) throws SQLException; + + static native void duckdb_jdbc_destroy_db_ref(ByteBuffer db_ref) throws SQLException; + + static native void duckdb_jdbc_set_auto_commit(ByteBuffer conn_ref, boolean auto_commit) throws SQLException; + + static native boolean duckdb_jdbc_get_auto_commit(ByteBuffer conn_ref) throws SQLException; + + static native void duckdb_jdbc_disconnect(ByteBuffer conn_ref); + + static native void duckdb_jdbc_set_schema(ByteBuffer conn_ref, String schema); + + static native void duckdb_jdbc_set_catalog(ByteBuffer conn_ref, String catalog); + + static native String duckdb_jdbc_get_schema(ByteBuffer conn_ref); + + static native String duckdb_jdbc_get_catalog(ByteBuffer conn_ref); + + // returns stmt_ref result reference object + static native ByteBuffer duckdb_jdbc_prepare(ByteBuffer conn_ref, byte[] query) throws SQLException; + + static native void duckdb_jdbc_release(ByteBuffer stmt_ref); + + static native DuckDBResultSetMetaData duckdb_jdbc_query_result_meta(ByteBuffer result_ref) throws SQLException; + + static native DuckDBResultSetMetaData duckdb_jdbc_prepared_statement_meta(ByteBuffer stmt_ref) throws SQLException; + + // returns res_ref result reference object + static native ByteBuffer duckdb_jdbc_execute(ByteBuffer stmt_ref, Object[] params) throws SQLException; + + static native void duckdb_jdbc_free_result(ByteBuffer res_ref); + + static native DuckDBVector[] duckdb_jdbc_fetch(ByteBuffer res_ref, ByteBuffer conn_ref) throws SQLException; + + static native int duckdb_jdbc_fetch_size(); + + static native long duckdb_jdbc_arrow_stream(ByteBuffer res_ref, long batch_size); + + static native void duckdb_jdbc_arrow_register(ByteBuffer conn_ref, long arrow_array_stream_pointer, byte[] name); + + static native ByteBuffer duckdb_jdbc_create_appender(ByteBuffer conn_ref, byte[] schema_name, byte[] table_name) + throws SQLException; + + static native void duckdb_jdbc_appender_begin_row(ByteBuffer appender_ref) throws SQLException; + + static native void duckdb_jdbc_appender_end_row(ByteBuffer appender_ref) throws SQLException; + + static native void duckdb_jdbc_appender_flush(ByteBuffer appender_ref) throws SQLException; + + static native void duckdb_jdbc_interrupt(ByteBuffer conn_ref); + + static native QueryProgress duckdb_jdbc_query_progress(ByteBuffer conn_ref); + + static native void duckdb_jdbc_appender_close(ByteBuffer appender_ref) throws SQLException; + + static native void duckdb_jdbc_appender_append_boolean(ByteBuffer appender_ref, boolean value) throws SQLException; + + static native void duckdb_jdbc_appender_append_byte(ByteBuffer appender_ref, byte value) throws SQLException; + + static native void duckdb_jdbc_appender_append_short(ByteBuffer appender_ref, short value) throws SQLException; + + static native void duckdb_jdbc_appender_append_int(ByteBuffer appender_ref, int value) throws SQLException; + + static native void duckdb_jdbc_appender_append_long(ByteBuffer appender_ref, long value) throws SQLException; + + static native void duckdb_jdbc_appender_append_float(ByteBuffer appender_ref, float value) throws SQLException; + + static native void duckdb_jdbc_appender_append_double(ByteBuffer appender_ref, double value) throws SQLException; + + static native void duckdb_jdbc_appender_append_string(ByteBuffer appender_ref, byte[] value) throws SQLException; + + static native void duckdb_jdbc_appender_append_bytes(ByteBuffer appender_ref, byte[] value) throws SQLException; + + static native void duckdb_jdbc_appender_append_timestamp(ByteBuffer appender_ref, long value) throws SQLException; + + static native void duckdb_jdbc_appender_append_decimal(ByteBuffer appender_ref, BigDecimal value) + throws SQLException; + + static native void duckdb_jdbc_appender_append_null(ByteBuffer appender_ref) throws SQLException; + + static native void duckdb_jdbc_create_extension_type(ByteBuffer conn_ref) throws SQLException; + + protected static native String duckdb_jdbc_get_profiling_information(ByteBuffer conn_ref, + ProfilerPrintFormat format) + throws SQLException; + + public static void duckdb_jdbc_create_extension_type(DuckDBConnection conn) throws SQLException { + duckdb_jdbc_create_extension_type(conn.connRef); + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2b4e50a54f80..81d7893c09b4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -314,6 +314,7 @@ object Dependencies { val googleProtobufVersion = "3.25.1" val shapelessVersion = "2.3.10" val postgresVersion = "42.4.0" + val duckdbVersion = "1.4.0.0" val h2Version = "2.3.232" val jimFsVersion = "1.3.0" } diff --git a/project/Editions.scala b/project/Editions.scala index 059ad3d1ca6a..654c6c69b0bb 100644 --- a/project/Editions.scala +++ b/project/Editions.scala @@ -24,7 +24,8 @@ object Editions { "Standard.Snowflake", "Standard.Microsoft", "Standard.Tableau", - "Standard.Saas" + "Standard.Saas", + "Standard.DuckDB" ) case class ContribLibrary(name: String, version: String) diff --git a/project/JarExtractor.scala b/project/JarExtractor.scala index b207d8497815..e47a683b435d 100644 --- a/project/JarExtractor.scala +++ b/project/JarExtractor.scala @@ -24,20 +24,35 @@ object JarExtractor { sealed trait NativeLibArch { // Path of the library inside the extracted files directory // Inspired by `org.enso.pkg.NativeLibraryFinder` - val path: String + def path: String + // An extension expected when loading libraries from the given architecture + def extension: String + // A prefix of the library expected when loading libraries from the given architecture. + // If None, arch makes no assumptions about the name of the file. + // If Some(a: Left), then a library must not have a prefix `a` in the file name. + // If Some(b: Right), then a library must have a prefix `b` in the file name. + def prefix: Option[Either[String, String]] } case object LinuxAMD64 extends NativeLibArch { - override val path: String = "amd64/linux" + override val path: String = "amd64/linux" + override val extension: String = "so" + override val prefix: Option[Either[String, String]] = Some(Right("lib")) } case object WindowsAMD64 extends NativeLibArch { - override val path: String = "amd64/windows" + override val path: String = "amd64/windows" + override val extension: String = "dll" + override val prefix: Option[Either[String, String]] = Some(Left("lib")) } case object MacOSAMD64 extends NativeLibArch { - override val path: String = "amd64/macos" + override val path: String = "amd64/macos" + override val extension: String = "dylib" + override val prefix: Option[Either[String, String]] = Some(Right("lib")) } case object MacOSArm64 extends NativeLibArch { - override val path: String = "aarch64/macos" + override val path: String = "aarch64/macos" + override val extension: String = "dylib" + override val prefix: Option[Either[String, String]] = Some(Right("lib")) } // What to do with the matching jar entries. @@ -103,16 +118,33 @@ object JarExtractor { copyEntry(outputJar, inputJar, entry, logger) case PolyglotLib(arch) => // Silently rename the old `*.jnilib` files to `*.dylib`. - val destPath = polyglotLibDir - .resolve(arch.path) - .resolve( - entryPath.getFileName.toString.replace( - ".jnilib", - ".dylib" - ) - ) - if (archMatchesCurPlatform(arch)) { - copyEntry(destPath, inputJar, entry, logger) + val fullPath = entryPath.getFileName.toString + val idx = fullPath.lastIndexOf('.') + if (idx > 0) { + val oldExtension = fullPath.substring(idx + 1) + + val fullPath1 = + if (oldExtension != arch.extension) + fullPath.replace( + "." + oldExtension, + "." + arch.extension + ) + else fullPath + val fullPath2 = arch.prefix + .map { + case Left(prohibited) => + fullPath1.stripPrefix(prohibited) + case Right(required) => + if (fullPath1.startsWith(required)) fullPath1 + else required + fullPath1 + } + .getOrElse(fullPath1) + val destPath = polyglotLibDir + .resolve(arch.path) + .resolve(fullPath2) + if (archMatchesCurPlatform(arch)) { + copyEntry(destPath, inputJar, entry, logger) + } } } } diff --git a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java index 292156ac9fb9..f755a5a35ede 100644 --- a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java +++ b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java @@ -46,6 +46,24 @@ public static Object[] getDrivers() { */ public static Connection getConnection(String url, List> properties) throws SQLException { + return getConnectionWithCatalogSchema(url, properties, null, null); + } + + /** + * Tries to create a new connection using the JDBC DriverManager. + * + *

It delegates directly to {@code DriverManager.getConnection}. That is needed because if that + * method is called directly from Enso, the JDBC drivers are not detected correctly. + * + * @param url database url to connect to, starting with `jdbc:` + * @param properties configuration for the connection + * @param catalog the catalog to set on the connection, or null to not set it + * @param schema the schema to set on the connection, or null to not set it + * @return a connection + */ + public static Connection getConnectionWithCatalogSchema( + String url, List> properties, String catalog, String schema) + throws SQLException { // We need to manually register all the drivers because the DriverManager is not able // to correctly use our class loader, it only delegates to the platform class loader when // loading the java.sql.Driver service. @@ -57,6 +75,14 @@ public static Connection getConnection(String url, List new LocalAuditedConnection(rawConnection); case "cloud" -> diff --git a/std-bits/duckdb/src/main/java/org/enso/duckdb/DuckDBUtils.java b/std-bits/duckdb/src/main/java/org/enso/duckdb/DuckDBUtils.java new file mode 100644 index 000000000000..bc76d36d71ce --- /dev/null +++ b/std-bits/duckdb/src/main/java/org/enso/duckdb/DuckDBUtils.java @@ -0,0 +1,23 @@ +package org.enso.duckdb; + +import org.duckdb.DuckDBConnection; + +public class DuckDBUtils { + /** + * Retrieves the version of the connected DuckDB database. + * + * @param connection an active DuckDBConnection + * @return the version string of the DuckDB database + */ + public static String getVersion(DuckDBConnection connection) { + try (var stmt = connection.createStatement(); + var rs = stmt.executeQuery("SELECT version()")) { + if (rs.next()) { + return rs.getString(1); + } + return null; + } catch (Exception e) { + return null; + } + } +} diff --git a/std-bits/duckdb/src/main/resources/META-INF/native-image/org/enso/duckdb/reachability-metadata.json b/std-bits/duckdb/src/main/resources/META-INF/native-image/org/enso/duckdb/reachability-metadata.json new file mode 100644 index 000000000000..afe8464b47b6 --- /dev/null +++ b/std-bits/duckdb/src/main/resources/META-INF/native-image/org/enso/duckdb/reachability-metadata.json @@ -0,0 +1,445 @@ +{ + "reflection": [ + ], + "resources": [ + ], + "bundles": [], + "jni": [ + { + "type": "java.lang.Boolean", + "methods": [ + { + "name": "booleanValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Byte", + "methods": [ + { + "name": "byteValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Class", + "methods": [ + { + "name": "getClassLoader", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.ClassLoader", + "methods": [ + { + "name": "loadClass", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "java.lang.Double", + "methods": [ + { + "name": "doubleValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Float", + "methods": [ + { + "name": "floatValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Integer", + "methods": [ + { + "name": "intValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Long", + "methods": [ + { + "name": "longValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Object", + "methods": [ + { + "name": "toString", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Short", + "methods": [ + { + "name": "shortValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.String", + "methods": [ + { + "name": "getBytes", + "parameterTypes": [ + "java.nio.charset.Charset" + ] + } + ] + }, + { + "type": "java.lang.Throwable", + "methods": [ + { + "name": "getMessage", + "parameterTypes": [] + } + ] + }, + { + "type": "java.math.BigDecimal", + "methods": [ + { + "name": "longValue", + "parameterTypes": [] + }, + { + "name": "precision", + "parameterTypes": [] + }, + { + "name": "scale", + "parameterTypes": [] + }, + { + "name": "scaleByPowerOfTen", + "parameterTypes": [ + "int" + ] + }, + { + "name": "toPlainString", + "parameterTypes": [] + } + ] + }, + { + "type": "java.nio.ByteBuffer", + "methods": [ + { + "name": "order", + "parameterTypes": [ + "java.nio.ByteOrder" + ] + } + ] + }, + { + "type": "java.nio.ByteOrder", + "fields": [ + { + "name": "LITTLE_ENDIAN" + } + ] + }, + { + "type": "java.nio.CharBuffer", + "methods": [ + { + "name": "toString", + "parameterTypes": [] + } + ] + }, + { + "type": "java.nio.charset.Charset", + "methods": [ + { + "name": "decode", + "parameterTypes": [ + "java.nio.ByteBuffer" + ] + } + ] + }, + { + "type": "java.nio.charset.StandardCharsets", + "fields": [ + { + "name": "UTF_8" + } + ] + }, + { + "type": "java.sql.Array", + "methods": [ + { + "name": "getArray", + "parameterTypes": [] + }, + { + "name": "getBaseTypeName", + "parameterTypes": [] + } + ] + }, + { + "type": "java.sql.SQLException" + }, + { + "type": "java.sql.SQLTimeoutException" + }, + { + "type": "java.sql.Struct", + "methods": [ + { + "name": "getAttributes", + "parameterTypes": [] + }, + { + "name": "getSQLTypeName", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.Iterator", + "methods": [ + { + "name": "hasNext", + "parameterTypes": [] + }, + { + "name": "next", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.List", + "methods": [ + { + "name": "iterator", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.Map", + "methods": [ + { + "name": "entrySet", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.Map$Entry", + "methods": [ + { + "name": "getKey", + "parameterTypes": [] + }, + { + "name": "getValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.Set", + "methods": [ + { + "name": "iterator", + "parameterTypes": [] + } + ] + }, + { + "type": "java.util.UUID", + "methods": [ + { + "name": "getLeastSignificantBits", + "parameterTypes": [] + }, + { + "name": "getMostSignificantBits", + "parameterTypes": [] + } + ] + }, + { + "type": "org.duckdb.DuckDBArray", + "methods": [ + { + "name": "", + "parameterTypes": [ + "org.duckdb.DuckDBVector", + "int", + "int" + ] + } + ] + }, + { + "type": "org.duckdb.DuckDBDate", + "methods": [ + { + "name": "getDaysSinceEpoch", + "parameterTypes": [] + } + ] + }, + { + "type": "org.duckdb.DuckDBHugeInt", + "fields": [ + { + "name": "lower" + }, + { + "name": "upper" + } + ] + }, + { + "type": "org.duckdb.DuckDBResultSetMetaData", + "methods": [ + { + "name": "", + "parameterTypes": [ + "int", + "int", + "java.lang.String[]", + "java.lang.String[]", + "java.lang.String[]", + "java.lang.String", + "java.lang.String[]", + "java.lang.String[]" + ] + } + ] + }, + { + "type": "org.duckdb.DuckDBStruct", + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.String[]", + "org.duckdb.DuckDBVector[]", + "int", + "java.lang.String" + ] + } + ] + }, + { + "type": "org.duckdb.DuckDBTime" + }, + { + "type": "org.duckdb.DuckDBTimestamp", + "methods": [ + { + "name": "getMicrosEpoch", + "parameterTypes": [] + }, + { + "name": "valueOf", + "parameterTypes": [ + "java.lang.Object" + ] + } + ] + }, + { + "type": "org.duckdb.DuckDBTimestampTZ" + }, + { + "type": "org.duckdb.DuckDBVector", + "fields": [ + { + "name": "constlen_data" + }, + { + "name": "varlen_data" + } + ], + "methods": [ + { + "name": "", + "parameterTypes": [ + "java.lang.String", + "int", + "boolean[]" + ] + } + ] + }, + { + "type": "org.duckdb.ProfilerPrintFormat", + "fields": [ + { + "name": "GRAPHVIZ" + }, + { + "name": "HTML" + }, + { + "name": "JSON" + }, + { + "name": "NO_OUTPUT" + }, + { + "name": "QUERY_TREE" + }, + { + "name": "QUERY_TREE_OPTIMIZER" + } + ] + }, + { + "type": "org.duckdb.QueryProgress", + "methods": [ + { + "name": "", + "parameterTypes": [ + "double", + "long", + "long" + ] + } + ] + }, + { + "type": "org.duckdb.user.DuckDBMap", + "methods": [ + { + "name": "getSQLTypeName", + "parameterTypes": [] + } + ] + } + ] +} diff --git a/test/Base_Tests/src/System/Embedded_Native_Libs_Spec.enso b/test/Base_Tests/src/System/Embedded_Native_Libs_Spec.enso index 5f15189267c8..09b4ae950f13 100644 --- a/test/Base_Tests/src/System/Embedded_Native_Libs_Spec.enso +++ b/test/Base_Tests/src/System/Embedded_Native_Libs_Spec.enso @@ -62,7 +62,7 @@ collect_jars_with_native_libs root_dir:File jar_files_filter:Text -> Dictionary all_jars.is_empty . should_be_false should_include_entry entry_name = - entry_name.ends_with ".so" || entry_name.ends_with ".dll" || entry_name.ends_with ".dylib" + entry_name.match "^.+\.so.*$" || entry_name.ends_with ".dll" || entry_name.ends_with ".dylib" # Maps Jar paths to list of invalid entries dict = all_jars.fold Dictionary.empty \acc jar -> diff --git a/test/DuckDB_Tests/data/house-price.parquet b/test/DuckDB_Tests/data/house-price.parquet new file mode 100644 index 000000000000..58b5cf3af2e5 Binary files /dev/null and b/test/DuckDB_Tests/data/house-price.parquet differ diff --git a/test/DuckDB_Tests/package.yaml b/test/DuckDB_Tests/package.yaml new file mode 100644 index 000000000000..f123aa32058f --- /dev/null +++ b/test/DuckDB_Tests/package.yaml @@ -0,0 +1,7 @@ +name: DuckDB_Tests +namespace: enso_dev +version: 0.0.1 +license: MIT +author: enso-dev@enso.org +maintainer: enso-dev@enso.org +prefer-local-libraries: true diff --git a/test/DuckDB_Tests/src/DuckDB_In_Memory_Spec.enso b/test/DuckDB_Tests/src/DuckDB_In_Memory_Spec.enso new file mode 100644 index 000000000000..f1983918376a --- /dev/null +++ b/test/DuckDB_Tests/src/DuckDB_In_Memory_Spec.enso @@ -0,0 +1,23 @@ +from Standard.Base import all +from Standard.Table import all +from Standard.Database import all +from Standard.DuckDB import all + +from Standard.Test import all + +add_specs suite_builder = + suite_builder.group "DuckDB_Parquet" group_builder-> + group_builder.specify "Connect to an In-Memory Database" <| + connect = Database.connect DuckDB.In_Memory + connect.database.should_equal "memory" + + group_builder.specify "List schemas" <| + connect = Database.connect DuckDB.In_Memory + connect.schema.should_equal "main" + connect.schemas.should_contain "main" + connect.schemas.should_contain "information_schema" + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/test/DuckDB_Tests/src/DuckDB_Parquet_Spec.enso b/test/DuckDB_Tests/src/DuckDB_Parquet_Spec.enso new file mode 100644 index 000000000000..5de60890c0f8 --- /dev/null +++ b/test/DuckDB_Tests/src/DuckDB_Parquet_Spec.enso @@ -0,0 +1,27 @@ +from Standard.Base import all +from Standard.Table import all +from Standard.Database import all +from Standard.DuckDB import all + +from Standard.Test import all + +add_specs suite_builder = + suite_builder.group "DuckDB_Parquet" group_builder-> + data_file = enso_project.data / 'house-price.parquet' + + group_builder.specify "Read a parquet file into Enso as an In-Memory Table" <| + connect = Database.connect DuckDB.In_Memory + raw_query = "SELECT * FROM '" + (data_file.path.replace "'" "''") + "'" + table = connect.execute_query raw_query ..All_Rows + table.row_count.should_equal 545 + + group_builder.specify "Read a parquet file into Enso as an In-Database Table" <| + connect = Database.connect DuckDB.In_Memory + raw_query = "SELECT * FROM '" + (data_file.path.replace "'" "''") + "'" + table = connect.query (..Raw_SQL raw_query) + table.row_count.should_equal 545 + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/test/DuckDB_Tests/src/Main.enso b/test/DuckDB_Tests/src/Main.enso new file mode 100644 index 000000000000..e33ed433a1e8 --- /dev/null +++ b/test/DuckDB_Tests/src/Main.enso @@ -0,0 +1,13 @@ +from Standard.Base import all + +from Standard.Test import Test + +import project.DuckDB_In_Memory_Spec +import project.DuckDB_Parquet_Spec + +main filter=Nothing = + suite = Test.build suite_builder-> + DuckDB_In_Memory_Spec.add_specs suite_builder + DuckDB_Parquet_Spec.add_specs suite_builder + + suite.run_with_filter filter diff --git a/tools/legal-review/DuckDB/report-state b/tools/legal-review/DuckDB/report-state new file mode 100644 index 000000000000..7430a00d410d --- /dev/null +++ b/tools/legal-review/DuckDB/report-state @@ -0,0 +1,3 @@ +1F3966AB4B67545DA6CD61625301F926A0EB26DAABE0A45C88C68146E436C6D8 +A5A62D13D139C9AEE34004A1A494377BEC2927E72AD4F54268A26154BC9FEFFD +0 diff --git a/tools/legal-review/DuckDB/reviewed-licenses/MIT_License b/tools/legal-review/DuckDB/reviewed-licenses/MIT_License new file mode 100644 index 000000000000..3889131c8ace --- /dev/null +++ b/tools/legal-review/DuckDB/reviewed-licenses/MIT_License @@ -0,0 +1 @@ +tools/legal-review/license-texts/MIT