Draft -- PostgreSQL db replaces Oracle db

access-control-app/src/cmr/access_control/system.clj

@@ -138,7 +138,7 @@
     (try
       (access-control-index/create-index-or-update-mappings (:search-index started-system))
       ;; This is needed to bootstrap admin group for legacy services for integration tests
-      (bootstrap/bootstrap started-system)
+      ;; (bootstrap/bootstrap started-system) TODO -- uncomment me and debug? if not removed
       (catch Exception e
         (common-sys/stop started-system component-order)
         (throw e)))

aurora-postgres-lib/project.clj

@@ -0,0 +1,67 @@
+(defproject nasa-cmr/cmr-aurora-postgres-lib "0.1.0-SNAPSHOT"
+  :description "Contains utilities for connecting to and manipulating data in Aurora PostgreSQL"
+  :url "https://github.com/nasa/Common-Metadata-Repository/tree/master/aurora-postgres-lib"
+  ;; Dynamically include extra repositories in the project definition if configured.
+  :dependencies [[nasa-cmr/cmr-common-lib "0.1.1-SNAPSHOT"]
+                 [org.clojure/clojure "1.10.0"]
+                 [org.clojure/java.jdbc "0.4.2"]
+                 [org.postgresql/postgresql "42.6.0"]
+                 [software.amazon.jdbc/aws-advanced-jdbc-wrapper "2.2.2"]
+                 [com.zaxxer/HikariCP "5.0.1"]]
+  :plugins [[lein-shell "0.5.0"]
+            [test2junit "1.3.3"]]
+  :jvm-opts ^:replace ["-server"
+                       "-Dclojure.compiler.direct-linking=true"]
+  :profiles {:security {:plugins [[com.livingsocial/lein-dependency-check "1.1.1"]]
+                        :dependency-check {:output-format [:all]
+                                           :suppression-file "resources/security/suppression.xml"}}
+             :dev {:exclusions [[org.clojure/tools.nrepl]]
+                   :dependencies [[org.clojars.gjahad/debug-repl "0.3.3"]
+                                  [org.clojure/tools.namespace "0.2.11"]
+                                  [org.clojure/tools.nrepl "0.2.13"]]
+                   :jvm-opts ^:replace ["-server"]
+                   :source-paths ["src" "dev" "test"]}
+             :static {}
+             ;; This profile is used for linting and static analysis. To run for this
+             ;; project, use `lein lint` from inside the project directory. To run for
+             ;; all projects at the same time, use the same command but from the top-
+             ;; level directory.
+             :lint {:source-paths ^:replace ["src"]
+                    :test-paths ^:replace []
+                    :plugins [[jonase/eastwood "0.2.5"]
+                              [lein-ancient "0.6.15"]
+                              [lein-bikeshed "0.5.0"]
+                              [lein-kibit "0.1.6"]]}
+             ;; The following profile is overriden on the build server or in the user's
+             ;; ~/.lein/profiles.clj file.
+             :internal-repos {}
+             :kaocha {:dependencies [[lambdaisland/kaocha "1.0.732"]
+                                     [lambdaisland/kaocha-cloverage "1.0.75"]
+                                     [lambdaisland/kaocha-junit-xml "0.0.76"]]}}
+  :aliases {;; Alias to test2junit for consistency with lein-test-out
+            "test-out" ["test2junit"]
+
+            ;; Kaocha test aliases
+            ;; refer to tests.edn for test configuration
+            "kaocha" ["with-profile" "+kaocha" "run" "-m" "kaocha.runner"]
+            "itest" ["kaocha" "--focus" ":integration"]
+            "utest" ["kaocha" "--focus" ":unit"]
+            "ci-test" ["kaocha" "--profile" ":ci"]
+            "ci-itest" ["itest" "--profile" ":ci"]
+            "ci-utest" ["utest" "--profile" ":ci"]
+
+            ;; Linting aliases
+            "kibit" ["do"
+                     ["with-profile" "lint" "shell" "echo" "== Kibit =="]
+                     ["with-profile" "lint" "kibit"]]
+            ;; Eastwood needs special handling with libs that include oracle
+            ;; drivers in the deps, in particulear:
+            ;;   java.lang.ClassNotFoundException: oracle.dms.console.DMSConsole
+            "eastwood" ["with-profile" "lint" "eastwood"
+                        "{:namespaces [:source-paths] :exclude-namespaces [cmr.aurora.connection]}"]
+            "bikeshed" ["with-profile" "lint" "bikeshed" "--max-line-length=100"]
+            "check-deps" ["with-profile" "lint" "ancient" ":all"]
+            "check-sec" ["with-profile" "security" "dependency-check"]
+            "lint" ["do" ["check"] ["kibit"] ["eastwood"]]
+            ;; Placeholder for future docs and enabler of top-level alias
+            "generate-static" ["with-profile" "static" "shell" "echo"]})

aurora-postgres-lib/src/cmr/aurora/config.clj

@@ -0,0 +1,43 @@
+(ns cmr.aurora.config
+  "Contains functions for retrieving Aurora Postgres connection configuration from environment variables"
+  (:require [cmr.common.config :as cfg :refer [defconfig]]
+            [cmr.aurora.connection :as conn]))
+
+(defconfig aurora-cluster
+  "Aurora cluster name"
+  {:default "cmr-aurora-cluster"})
+
+(defconfig postgres-db-name
+  "Aurora database name"
+  {:default "cmrcdb"})
+
+(defconfig db-url-primary
+  "Primary db endpoint (for writes and reads)"
+  {:default "localhost"})
+
+(defconfig db-url-secondary
+  "Secondary db url (for reads only)
+   Note that for local development this is the same as db-url-primary since there is no local Aurora cluster mock-up."
+  {:default "localhost"})
+
+(defn db-connection-str
+  "Returns the connection string for the given postgres db endpoint"
+  [host]
+  (str "jdbc:aws-wrapper:postgresql://" host ":5432/" postgres-db-name))
+
+(defconfig master-db-user
+  "Postgres database master user"
+  {:default "postgres"})
+
+(defconfig master-db-password
+  "Postgres database master password"
+  {:default "admin"}) ;; TODO -- remove me
+
+(defn sys-dba-db-spec
+  []
+  (conn/db-spec
+    "pg-sys-dba-connection-pool"
+    (db-url-primary)
+    (master-db-user)
+    (master-db-password)
+    (postgres-db-name)))

aurora-postgres-lib/src/cmr/aurora/connection.clj

@@ -0,0 +1,154 @@
+(ns cmr.aurora.connection
+  "Contains functions for interacting with the Aurora DB cluster."
+  (:require
+   [clojure.java.jdbc :as j]
+   [clojure.string :as str]
+   [clj-time.coerce :as cr]
+   [cmr.common.date-time-parser :as p]
+   [cmr.common.lifecycle :as lifecycle]
+   [cmr.common.log :refer [debug error info trace warn]]
+   [cmr.common.util :as util]
+   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.services.errors :as errors]
+   [cmr.common.services.health-helper :as hh])
+  (:import
+   com.zaxxer.hikari.HikariDataSource
+   software.amazon.jdbc.ds.AwsWrapperDataSource
+   java.sql.DriverManager))
+
+(defn db-spec
+  [connection-pool-name db-url user password db-name]
+  {:classname "org.postgresql.ds.PGSimpleDataSource"
+   :subprotocol "jdbc:aws-wrapper:postgresql:"
+   :dbtype "postgresql"
+   :host db-url
+   :port "5432"
+   :dbname db-name
+   :user (str/lower-case user)
+   :password password
+   :connection-pool-name connection-pool-name})
+
+(defn pool
+  [spec]
+  (let [{:keys [classname
+                subprotocol
+                host
+                port
+                user
+                password
+                dbname
+                connection-pool-name]} spec]
+    (doto (HikariDataSource.)
+      (.setMaximumPoolSize 100)
+      (.setPoolName connection-pool-name)
+      (.setUsername user)
+      (.setPassword password)
+      (.setDataSourceClassName (.getName AwsWrapperDataSource))
+      (.addDataSourceProperty "jdbcProtocol" subprotocol)
+      (.addDataSourceProperty "serverName" host)
+      (.addDataSourceProperty "serverPort" port)
+      (.addDataSourceProperty "database" dbname)
+      (.addDataSourceProperty "targetDataSourceClassName" classname))))
+
+(defrecord PostgresStore
+           [;; The database spec.
+            spec
+
+            ;; The database pool of connections
+            datasource]
+
+  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+  ;; CMR Component Implementation
+  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+  lifecycle/Lifecycle
+
+  (start [this system]
+    (assoc this :datasource (pool spec)))
+
+  (stop [this system]
+
+    #_(try
+          ;; Cleanup the connection pool
+        (let [pool-name (get-in this [:spec :connection-pool-name])
+              hikari (HikariDataSource.)]
+          (.destroyConnectionPool hikari pool-name))
+        (catch Exception e
+          (warn (str "Unable to destroy connection pool. It may not have started. Error: "
+                     (.getMessage e)))))
+
+    (dissoc this :datasource)))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PostgresStore Constructor
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn create-db
+  "Creates and returns the database connection pool."
+  [spec]
+  (->PostgresStore spec nil))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Prototype Work
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; the following db connection pool method was from the prototype, it is an 
+;; alternative to the PostgresStore component/atom, not appropriate for production. 
+;; How to use it:
+;; (def pooled-pg-db (delay (aurora/make-prototype-pool (aurora/db-spec "connection pool name"))))
+;; (defn pg-db-connection [] @pooled-pg-db)        ;; this is what you pass to jdbc lib, e.g.:
+;; (jdbc/query (pg-db-connection) ["select * from some_table limit 10"])
+(defn make-prototype-pool
+  [spec]
+  (let [cpds (pool spec)]
+    {:datasource cpds}))
+
+(defn execute-query
+  [pool sql-query]
+  (with-open [conn (.getConnection pool)
+              stmt (.createStatement conn)
+              res (.executeQuery stmt sql-query)]
+    ;; return query result
+    (.next res)))
+
+(defn create-temp-table
+  "Creates temporary work area used in get-concepts and force-delete-concepts-by-params functions. 
+   Call from within a transaction, note that Postgres deletes temporary tables at the end of a session"
+  [conn]
+  (info "Creating get_concepts_work_area temporary table")
+  (j/db-do-prepared conn "CREATE TEMP TABLE IF NOT EXISTS get_concepts_work_area
+                  (concept_id VARCHAR(255),
+                  revision_id INTEGER)
+                  ON COMMIT DELETE ROWS"))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; PostgreSQL Timestamp Conversion Utils
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defmulti pg-timestamp->clj-time ;; TODO - add timezone adjusting logic if necessary
+  "Converts PostgreSQL timestamp types into clj-time. Must be called within
+  a with-db-transaction block with the connection"
+  (fn [db pt]
+    (type pt)))
+
+(defmethod pg-timestamp->clj-time java.sql.Timestamp ;; this is the one that executes but code needs work
+  [db ^java.sql.Timestamp pt]
+  (let [cal (java.util.Calendar/getInstance)
+        ^java.util.TimeZone gmt-tz (java.util.TimeZone/getTimeZone "GMT")]
+    (.setTimeZone cal gmt-tz)
+    (cr/from-sql-time (.timestampValue pt cal))))
+
+(defmethod pg-timestamp->clj-time org.postgresql.util.PGTimestamp ;; might execute ?
+  [db ^org.postgresql.util.PGTimestamp pt]
+  (println "GOT TIMESTAMP TYPE: org.postgresql.util.PGTimestamp")
+  (let [cal (java.util.Calendar/getInstance)
+        ^java.util.TimeZone gmt-tz (java.util.TimeZone/getTimeZone "GMT")]
+    (.setTimeZone cal gmt-tz)
+    (cr/from-sql-time (.timestampValue pt cal))))
+
+(defn db-timestamp->str-time ;; fall through; lacking manual timezone adjustment
+  "Converts database timestamp instance into a string representation of the time. 
+   Must be called within a with-db-transaction block with the connection -- TODO"
+  [db ot]
+  (p/clj-time->date-time-str (cr/from-sql-time ot)))

aurora-postgres-lib/src/cmr/aurora/user.clj

@@ -0,0 +1,77 @@
+(ns cmr.aurora.user
+  "Contains functions for creating and dropping users."
+  (:require [clojure.java.jdbc :as j]
+            [cmr.aurora.config :as config]
+            [cmr.aurora.connection :as conn]))
+
+(def create-user-sql-template
+  "CREATE USER %%CMR_USER%% WITH PASSWORD '%%CMR_PASSWORD%%'")
+
+(def create-schema-make-default-templates
+  ["CREATE SCHEMA %%CMR_USER%%"
+   "ALTER USER %%CMR_USER%% SET search_path = %%CMR_USER%%"])
+
+(def grant-sql-templates
+  ["GRANT CONNECT ON DATABASE %%DATABASE_NAME%% TO %%CMR_USER%%"
+   "GRANT CREATE ON SCHEMA %%SCHEMA%% TO %%CMR_USER%%"
+   "GRANT USAGE ON SCHEMA %%SCHEMA%% TO %%CMR_USER%%"])
+
+(def select-users-tables-template
+  "SELECT table_name
+   FROM information_schema.tables
+   WHERE table_schema = '%%USERNAME%%'")
+
+(def grant-select-template
+  "grant select on %%FROM_USER%%.%%TABLE%% to %%CMR_USER%%")
+
+(def grant-any-table-template
+  "Grants ability to create and drop tables at will to the user. It also adds create any index which
+  is needed if creating a table with an index.
+  Note that in PostgreSQL, users can only be granted privileges on all tables within a specific schema,
+  and there is no direct equivalent of ANY privileges as in Oracle."
+  "GRANT ALL PRIVILEGES ON SCHEMA %%SCHEMA%% TO %%CMR_USER%%")
+
+(defn replace-values
+  "Replaces values in a sql template using the key values given"
+  [key-values template]
+  (reduce (fn [temp [key,val]]
+            (clojure.string/replace temp (str "%%" key "%%") val))
+          template
+          key-values))
+
+(defn create-user
+  "Creates the given user in the database."
+  [db user password]
+  (let [replacer (partial replace-values {"CMR_USER" user
+                                          "CMR_PASSWORD" password
+                                          "DATABASE_NAME" (config/postgres-db-name)
+                                          "SCHEMA" user})
+        create-user-sql (replacer create-user-sql-template)
+        default-schema-sqls (map replacer create-schema-make-default-templates)
+        grant-sqls (map replacer grant-sql-templates)]
+    (j/db-do-commands db create-user-sql)
+
+    (doseq [sql (vec (concat default-schema-sqls grant-sqls))]
+      (j/db-do-commands db sql))))
+
+(defn grant-select-privileges
+  "Grant select privileges from one user's tables to another user."
+  [db from-user to-user]
+  (println "Granting select privileges to" to-user "on tables in" from-user)
+  (let [select-tables-sql (replace-values {"USERNAME" from-user} select-users-tables-template)
+        tables (j/query db select-tables-sql)]
+    (when (empty? tables)
+      (println "WARNING: Found 0 tables owned by" from-user))
+    (doseq [{:keys [table_name]} tables]
+      (let [grant-sql (replace-values {"FROM_USER" from-user
+                                       "TABLE" table_name
+                                       "CMR_USER" to-user}
+                                      grant-select-template)]
+        (j/db-do-commands db grant-sql)))))
+
+(defn grant-create-drop-any-table-privileges
+  "Grant privileges to create and drop any table or modify any table in a schema. This is useful in testing
+  situations where we want to create test tables and data in another schema."
+  [db to-user schema]
+  (println "Granting create and drop any table privileges to" to-user)
+  (j/db-do-commands db (replace-values {"CMR_USER" to-user "SCHEMA" schema} grant-any-table-template)))

bootstrap-app/project.clj

@@ -20,6 +20,7 @@
                  [commons-codec/commons-codec "1.11"]
                  [commons-io "2.6"]
                  [compojure "1.6.1"]
+                 [nasa-cmr/cmr-aurora-postgres-lib "0.1.0-SNAPSHOT"]
                  [nasa-cmr/cmr-access-control-app "0.1.0-SNAPSHOT"]
                  [nasa-cmr/cmr-common-app-lib "0.1.0-SNAPSHOT"]
                  [nasa-cmr/cmr-indexer-app "0.1.0-SNAPSHOT"]

bootstrap-app/src/cmr/bootstrap/config.clj

@@ -4,7 +4,9 @@
    [cmr.common.config :as cfg :refer [defconfig]]
    [cmr.message-queue.config :as queue-config]
    [cmr.oracle.config :as oracle-config]
-   [cmr.oracle.connection :as conn]))
+   [cmr.oracle.connection :as conn]
+   [cmr.aurora.config :as aurora-config]
+   [cmr.aurora.connection :as pg-conn]))
 
 (defconfig bootstrap-username
   "Defines the bootstrap database username."
@@ -15,15 +17,14 @@
   {})
 
 (defn db-spec
-  "Returns a db spec populated with config information that can be used to connect to oracle"
+  "Returns a db spec populated with config information that can be used to connect to postgres"
   [connection-pool-name]
-  (conn/db-spec
-    connection-pool-name
-    (oracle-config/db-url)
-    (oracle-config/db-fcf-enabled)
-    (oracle-config/db-ons-config)
-    (bootstrap-username)
-    (bootstrap-password)))
+  (pg-conn/db-spec
+   connection-pool-name
+   (aurora-config/db-url-primary)
+   (bootstrap-username)
+   (bootstrap-password)
+   (aurora-config/postgres-db-name)))
 
 (defconfig bootstrap-nrepl-port
   "Port to listen for nREPL connections"