From d3f87d9c646847957031a40212ba2e59278f3471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= Date: Thu, 2 Dec 2021 13:15:27 +0100 Subject: [PATCH 01/81] working i2b2 docker image --- build/package/i2b2/Dockerfile | 59 +++++++ build/package/i2b2/I2b2PasswordHash.java | 21 +++ build/package/i2b2/docker-entrypoint.sh | 33 ++++ build/package/i2b2/download-i2b2-sources.sh | 14 ++ build/package/i2b2/install-i2b2.sh | 41 +++++ build/package/i2b2/patches/crc_pg_db_fix.diff | 24 +++ .../pre-init-scripts/05-wildfly-config.sh | 4 + .../i2b2/pre-init-scripts/10-i2b2-config.sh | 7 + .../10-write-i2b2-datasources.sh | 149 +++++++++++++++++ build/package/i2b2/sql/05-i2b2-orig-data.sh | 157 ++++++++++++++++++ .../package/i2b2/sql/10-i2b2-modifications.sh | 16 ++ .../package/i2b2/sql/20-i2b2-configuration.sh | 46 +++++ .../package/i2b2/sql/50-stored-procedures.sh | 7 + .../get_concept_codes.plpgsql | 21 +++ .../get_modifier_codes.plpgsql | 21 +++ .../get_ontology_elements.plpgsql | 35 ++++ .../50-stored-procedures/table_name.plpgsql | 16 ++ build/package/i2b2/sql/common.sh | 15 ++ 18 files changed, 686 insertions(+) create mode 100644 build/package/i2b2/Dockerfile create mode 100644 build/package/i2b2/I2b2PasswordHash.java create mode 100644 build/package/i2b2/docker-entrypoint.sh create mode 100644 build/package/i2b2/download-i2b2-sources.sh create mode 100644 build/package/i2b2/install-i2b2.sh create mode 100644 build/package/i2b2/patches/crc_pg_db_fix.diff create mode 100644 build/package/i2b2/pre-init-scripts/05-wildfly-config.sh create mode 100644 build/package/i2b2/pre-init-scripts/10-i2b2-config.sh create mode 100644 build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh create mode 100644 build/package/i2b2/sql/05-i2b2-orig-data.sh create mode 100644 build/package/i2b2/sql/10-i2b2-modifications.sh create mode 100644 build/package/i2b2/sql/20-i2b2-configuration.sh create mode 100644 build/package/i2b2/sql/50-stored-procedures.sh create mode 100644 build/package/i2b2/sql/50-stored-procedures/get_concept_codes.plpgsql create mode 100644 build/package/i2b2/sql/50-stored-procedures/get_modifier_codes.plpgsql create mode 100644 build/package/i2b2/sql/50-stored-procedures/get_ontology_elements.plpgsql create mode 100644 build/package/i2b2/sql/50-stored-procedures/table_name.plpgsql create mode 100644 build/package/i2b2/sql/common.sh diff --git a/build/package/i2b2/Dockerfile b/build/package/i2b2/Dockerfile new file mode 100644 index 0000000..346715b --- /dev/null +++ b/build/package/i2b2/Dockerfile @@ -0,0 +1,59 @@ +FROM jboss/wildfly:17.0.1.Final + +# build-time variables +ENV I2B2_DATA_ARCHIVE="$JBOSS_HOME/standalone/data/i2b2-data.tar.gz" \ + I2B2_FR_FILES_DIR="$JBOSS_HOME/standalone/data/i2b2_FR_files" \ + I2B2_PASSWORD_HASH_TOOL="$JBOSS_HOME/standalone/data/I2b2PasswordHash" \ + SRC_DIR=/src + +# needed packages and pre-requisites +USER root +RUN yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \ + yum update -y && yum -y install wget git ant postgresql13 && yum clean all && \ + mkdir -p "$SRC_DIR" "$I2B2_FR_FILES_DIR" "$I2B2_PASSWORD_HASH_TOOL" && chown -R jboss:jboss "$SRC_DIR" "$JBOSS_HOME" +USER jboss + +# wildfly custom configuration +RUN sed -i 's/Xmx512m/Xmx2048m/g' "$JBOSS_HOME"/bin/standalone.conf && \ + sed -i 's/MaxMetaspaceSize=256m/MaxMetaspaceSize=1024m/g' "$JBOSS_HOME"/bin/standalone.conf + +# download i2b2 sources +ARG I2B2_VERSION=tags/v1.7.12a.0002 +ARG I2B2_DATA_VERSION=01488ff1367717fda2799809bc531384382d6a94 +COPY --chown=jboss:jboss download-i2b2-sources.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/download-i2b2-sources.sh +RUN download-i2b2-sources.sh + +# install i2b2 +COPY --chown=jboss:jboss patches "$SRC_DIR/patches" +COPY --chown=jboss:jboss install-i2b2.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/install-i2b2.sh +RUN install-i2b2.sh + +# install i2b2 password hasher +COPY --chown=jboss:jboss I2b2PasswordHash.java "$SRC_DIR/" +RUN javac -d "$I2B2_PASSWORD_HASH_TOOL" "$SRC_DIR/I2b2PasswordHash.java" + +# run-time variables +ENV I2B2_DB_HOST="postgresql" \ + I2B2_DB_PORT="5432" \ + I2B2_DB_USER="i2b2" \ + I2B2_DB_PW="i2b2" \ + I2B2_DB_NAME="i2b2" \ + WILDFLY_ADMIN_PASSWORD="admin" \ + I2B2_DOMAIN_NAME="i2b2demo" \ + I2B2_SERVICE_PASSWORD="changeme" \ + DEFAULT_USER_PASSWORD="demouser" \ + AXIS2_LOGLEVEL="INFO" + +COPY --chown=jboss:jboss pre-init-scripts "$SRC_DIR/pre-init-scripts" +COPY --chown=jboss:jboss sql "$SRC_DIR/sql" +COPY --chown=jboss:jboss docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh +EXPOSE 8080 9990 +ARG PG_JDBC_JAR=postgresql-42.2.8.jar +ENV PG_JDBC_JAR="$PG_JDBC_JAR" +ENTRYPOINT \ + I2B2_SQL_DIR="$SRC_DIR/sql" \ + PRE_INIT_SCRIPT_DIR="$SRC_DIR/pre-init-scripts" \ + docker-entrypoint.sh diff --git a/build/package/i2b2/I2b2PasswordHash.java b/build/package/i2b2/I2b2PasswordHash.java new file mode 100644 index 0000000..0a00a4d --- /dev/null +++ b/build/package/i2b2/I2b2PasswordHash.java @@ -0,0 +1,21 @@ +import java.security.MessageDigest; + +public final class I2b2PasswordHash { + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.err.println("Usage: java I2b2PasswordHash "); + System.exit(-1); + } + + final MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(args[0].getBytes()); + + byte[] digest = md5.digest(); + final StringBuffer buf = new StringBuffer(); + for (int i = 0; i < digest.length; i++) { + buf.append(Integer.toHexString((int) digest[i] & 0x00FF)); + } + + System.out.println(buf.toString()); + } +} diff --git a/build/package/i2b2/docker-entrypoint.sh b/build/package/i2b2/docker-entrypoint.sh new file mode 100644 index 0000000..fb214b4 --- /dev/null +++ b/build/package/i2b2/docker-entrypoint.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# wait for postgres to be available +export PGPASSWORD="$I2B2_DB_PW" +export PSQL_PARAMS="-v ON_ERROR_STOP=1 -h ${I2B2_DB_HOST} -p ${I2B2_DB_PORT} -U ${I2B2_DB_USER}" +until psql $PSQL_PARAMS -d postgres -c '\q'; do + >&2 echo "Waiting for postgresql..." + sleep 1 +done + +# load initial data if database does not exist (credentials must be valid and have create database right) +DB_CHECK=$(psql ${PSQL_PARAMS} -d postgres -X -A -t -c "select count(*) from pg_database where datname = '${I2B2_DB_NAME}';") +if [[ "$DB_CHECK" -ne "1" ]]; then + echo "Initialising i2b2 database" + psql $PSQL_PARAMS -d postgres <<-EOSQL + CREATE DATABASE ${I2B2_DB_NAME}; +EOSQL + + export I2B2_DATA_DIR=/tmp/i2b2-data + mkdir -p "$I2B2_DATA_DIR" + tar xvzf "$I2B2_DATA_ARCHIVE" -C "$I2B2_DATA_DIR" + for f in "$I2B2_SQL_DIR"/*.sh; do + bash "$f" + done + rm -rf "$I2B2_DATA_DIR" +fi + +# execute pre-init scripts & run wildfly +for f in "$PRE_INIT_SCRIPT_DIR"/*.sh; do + bash "$f" +done +exec /opt/jboss/wildfly/bin/standalone.sh -b 0.0.0.0 -bmanagement 0.0.0.0 diff --git a/build/package/i2b2/download-i2b2-sources.sh b/build/package/i2b2/download-i2b2-sources.sh new file mode 100644 index 0000000..3578461 --- /dev/null +++ b/build/package/i2b2/download-i2b2-sources.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +git init "$SRC_DIR/i2b2-core-server" +pushd "$SRC_DIR/i2b2-core-server" +git remote add origin https://github.com/i2b2/i2b2-core-server.git +git pull --depth=1 origin "$I2B2_VERSION" +popd + +git init "$SRC_DIR/i2b2-data" +pushd "$SRC_DIR/i2b2-data" +git remote add origin https://github.com/i2b2/i2b2-data.git +git pull --depth=1 origin "$I2B2_DATA_VERSION" +popd diff --git a/build/package/i2b2/install-i2b2.sh b/build/package/i2b2/install-i2b2.sh new file mode 100644 index 0000000..5fca5ed --- /dev/null +++ b/build/package/i2b2/install-i2b2.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +JBOSS_HOME_DEPLOYMENTS="$JBOSS_HOME/standalone/deployments" + +pushd "$SRC_DIR" + pushd i2b2-core-server + + # patch i2b2 sources + git apply "../patches/"*.diff + + pushd edu.harvard.i2b2.server-common + + # build i2b2 WAR from sources and copy war file + sed -i "/jboss.home/c\jboss.home=$JBOSS_HOME" build.properties + ant clean dist war + cp dist/i2b2.war "$JBOSS_HOME_DEPLOYMENTS/i2b2.war.zip" + + # unpack i2b2 WAR + pushd "$JBOSS_HOME_DEPLOYMENTS" + mkdir i2b2.war + unzip i2b2.war.zip -d i2b2.war + rm i2b2.war.zip + touch i2b2.war.dodeploy + popd + + # copy additional libraries + ant jboss_pre_deployment_setup + popd + popd + + # archive i2b2 data + pushd i2b2-data + rm -rf ./.git # remove git info to get a lighter image + GZIP=-9 tar cvzf "$I2B2_DATA_ARCHIVE" -C . . + popd + + # delete repositories to free up space + rm -rf i2b2-core-server i2b2-data + +popd diff --git a/build/package/i2b2/patches/crc_pg_db_fix.diff b/build/package/i2b2/patches/crc_pg_db_fix.diff new file mode 100644 index 0000000..6a50443 --- /dev/null +++ b/build/package/i2b2/patches/crc_pg_db_fix.diff @@ -0,0 +1,24 @@ +diff --git a/edu.harvard.i2b2.crc/src/server/edu/harvard/i2b2/crc/dao/pdo/input/EidListTypeHandler.java b/edu.harvard.i2b2.crc/src/server/edu/harvard/i2b2/crc/dao/pdo/input/EidListTypeHandler.java +index c68b82a..def4007 100755 +--- a/edu.harvard.i2b2.crc/src/server/edu/harvard/i2b2/crc/dao/pdo/input/EidListTypeHandler.java ++++ b/edu.harvard.i2b2.crc/src/server/edu/harvard/i2b2/crc/dao/pdo/input/EidListTypeHandler.java +@@ -242,13 +242,12 @@ public class EidListTypeHandler extends CRCDAO implements + + public String getTempTableName() { + String tempTableName = ""; +- if (dataSourceLookup.getServerType().equalsIgnoreCase( +- DAOFactoryHelper.ORACLE)) { +- tempTableName = this.getDbSchemaName() +- + FactRelatedQueryHandler.TEMP_PARAM_TABLE; +- } else { +- tempTableName = this.getDbSchemaName() +- + SQLServerFactRelatedQueryHandler.TEMP_PDO_INPUTLIST_TABLE; ++ if (dataSourceLookup.getServerType().equalsIgnoreCase(DAOFactoryHelper.ORACLE)) { ++ tempTableName = this.getDbSchemaName() + FactRelatedQueryHandler.TEMP_PARAM_TABLE; ++ } else if (dataSourceLookup.getServerType().equalsIgnoreCase(DAOFactoryHelper.SQLSERVER)) { ++ tempTableName = this.getDbSchemaName() + SQLServerFactRelatedQueryHandler.TEMP_PDO_INPUTLIST_TABLE; ++ } else if (dataSourceLookup.getServerType().equalsIgnoreCase(DAOFactoryHelper.POSTGRESQL)) { ++ tempTableName = FactRelatedQueryHandler.TEMP_PARAM_TABLE; + } + return tempTableName; + } diff --git a/build/package/i2b2/pre-init-scripts/05-wildfly-config.sh b/build/package/i2b2/pre-init-scripts/05-wildfly-config.sh new file mode 100644 index 0000000..350e1c9 --- /dev/null +++ b/build/package/i2b2/pre-init-scripts/05-wildfly-config.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -Eeuo pipefail + +"$JBOSS_HOME"/bin/add-user.sh admin "$WILDFLY_ADMIN_PASSWORD" --silent || true diff --git a/build/package/i2b2/pre-init-scripts/10-i2b2-config.sh b/build/package/i2b2/pre-init-scripts/10-i2b2-config.sh new file mode 100644 index 0000000..28f22f3 --- /dev/null +++ b/build/package/i2b2/pre-init-scripts/10-i2b2-config.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -Eeuo pipefail + +# set i2b2 log level +pushd "$JBOSS_HOME/standalone/deployments/i2b2.war/WEB-INF/classes" +sed -i "/^log4j.rootCategory=/c\log4j.rootCategory=$AXIS2_LOGLEVEL, CONSOLE" log4j.properties +popd diff --git a/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh b/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh new file mode 100644 index 0000000..132c9a5 --- /dev/null +++ b/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh @@ -0,0 +1,149 @@ +#!/bin/bash +set -Eeuo pipefail + +cat > "$JBOSS_HOME/standalone/deployments/pm-ds.xml" <<EOL +<?xml version="1.0" encoding="UTF-8"?> +<datasources xmlns="http://www.jboss.org/ironjacamar/schema"> + <datasource jta="false" jndi-name="java:/PMBootStrapDS" + pool-name="PMBootStrapDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2pm</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> +</datasources> +EOL + +cat > "$JBOSS_HOME/standalone/deployments/ont-ds.xml" <<EOL +<?xml version="1.0" encoding="UTF-8"?> +<datasources xmlns="http://www.jboss.org/ironjacamar/schema"> + <datasource jta="false" jndi-name="java:/OntologyBootStrapDS" + pool-name="OntologyBootStrapDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2hive</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> + + <datasource jta="false" jndi-name="java:/OntologyDemoDS" + pool-name="OntologyDemoDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2metadata</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> +</datasources> + +EOL + +cat > "$JBOSS_HOME/standalone/deployments/crc-ds.xml" <<EOL +<?xml version="1.0" encoding="UTF-8"?> +<datasources xmlns="http://www.jboss.org/ironjacamar/schema"> + <datasource jta="false" jndi-name="java:/CRCBootStrapDS" + pool-name="CRCBootStrapDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2hive</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> + + <datasource jta="false" jndi-name="java:/QueryToolDemoDS" + pool-name="QueryToolDemoDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2demodata</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> +</datasources> +EOL + +cat > "$JBOSS_HOME/standalone/deployments/work-ds.xml" <<EOL +<?xml version="1.0" encoding="UTF-8"?> +<datasources xmlns="http://www.jboss.org/ironjacamar/schema"> + <datasource jta="false" jndi-name="java:/WorkplaceBootStrapDS" + pool-name="WorkplaceBootStrapDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2hive</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> + + <datasource jta="false" jndi-name="java:/WorkplaceDemoDS" + pool-name="WorkplaceDemoDS" enabled="true" use-ccm="false"> + <connection-url>jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$I2B2_DB_NAME?currentSchema=i2b2workdata</connection-url> + <driver-class>org.postgresql.Driver</driver-class> + <driver>$PG_JDBC_JAR</driver> + <security> + <user-name>$I2B2_DB_USER</user-name> + <password>$I2B2_DB_PW</password> + </security> + <validation> + <validate-on-match>false</validate-on-match> + <background-validation>false</background-validation> + </validation> + <statement> + <share-prepared-statements>false</share-prepared-statements> + </statement> + </datasource> +</datasources> +EOL diff --git a/build/package/i2b2/sql/05-i2b2-orig-data.sh b/build/package/i2b2/sql/05-i2b2-orig-data.sh new file mode 100644 index 0000000..2c750c7 --- /dev/null +++ b/build/package/i2b2/sql/05-i2b2-orig-data.sh @@ -0,0 +1,157 @@ +#!/bin/bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +# load the structure and data of i2b2 database (including some bug fixes) + +function loadI2b2Data { + DB_NAME="$1" + IS_ADDITIONAL_DB="$2" + LOAD_DEMO_DATA="$3" + + # ---------- CRC data ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Crcdata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2demodata + db.project=demo +EOL + + ant -f data_build.xml create_crcdata_tables_release_1-7 + ant -f data_build.xml create_procedures_release_1-7 + + if ! ${IS_ADDITIONAL_DB} + then + if ${LOAD_DEMO_DATA} + then + ant -f data_build.xml db_demodata_load_data + fi + fi + + + # ---------- Hive data ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Hivedata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2hive +EOL + + if ! ${IS_ADDITIONAL_DB} + then + ant -f data_build.xml create_hivedata_tables_release_1-7 + ant -f data_build.xml db_hivedata_load_data + fi + + + # ---------- IM data ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Imdata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2imdata + db.project=demo +EOL + + ant -f data_build.xml create_imdata_tables_release_1-7 + + if ! ${IS_ADDITIONAL_DB} + then + if ${LOAD_DEMO_DATA} + then + ant -f data_build.xml db_imdata_load_data + fi + fi + + + # ---------- Metadata ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Metadata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2metadata + db.project=demo +EOL + + ant -f data_build.xml create_metadata_tables_release_1-7 + + if ! ${IS_ADDITIONAL_DB} + then + if ${LOAD_DEMO_DATA} + then + ant -f data_build.xml db_metadata_load_data + #ant -f data_build.xml db_metadata_load_identified_data + fi + fi + + + # ---------- PM data ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Pmdata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2pm + db.project=demo +EOL + + if ! ${IS_ADDITIONAL_DB} + then + ant -f data_build.xml create_pmdata_tables_release_1-7 + ant -f data_build.xml create_triggers_release_1-7 + ant -f data_build.xml db_pmdata_load_data + fi + + + # ---------- Work data ---------- + cd "$I2B2_DATA_DIR/edu.harvard.i2b2.data/Release_1-7/NewInstall/Workdata" + + cat > db.properties <<EOL + db.type=postgresql + db.username=$I2B2_DB_USER + db.password=$I2B2_DB_PW + db.driver=org.postgresql.Driver + db.url=jdbc:postgresql://$I2B2_DB_HOST:$I2B2_DB_PORT/$DB_NAME?currentSchema=i2b2workdata + db.project=demo +EOL + + ant -f data_build.xml create_workdata_tables_release_1-7 + ant -f data_build.xml db_workdata_load_data + + + # ---------- Data bug fixes ---------- + if ! ${IS_ADDITIONAL_DB} + then + psql $PSQL_PARAMS -d "$DB_NAME" <<-EOSQL + update i2b2hive.crc_db_lookup set c_db_fullschema = 'i2b2demodata' where c_domain_id = 'i2b2demo'; + update i2b2hive.im_db_lookup set c_db_fullschema = 'i2b2imdata' where c_domain_id = 'i2b2demo'; + update i2b2hive.ont_db_lookup set c_db_fullschema = 'i2b2metadata' where c_domain_id = 'i2b2demo'; + update i2b2hive.work_db_lookup set c_db_fullschema = 'i2b2workdata' where c_domain_id = 'i2b2demo'; +EOSQL + fi +} + +# create schemas +initI2b2Schema "$I2B2_DB_NAME" i2b2demodata +initI2b2Schema "$I2B2_DB_NAME" i2b2imdata +initI2b2Schema "$I2B2_DB_NAME" i2b2metadata +initI2b2Schema "$I2B2_DB_NAME" i2b2workdata +initI2b2Schema "$I2B2_DB_NAME" i2b2pm +initI2b2Schema "$I2B2_DB_NAME" i2b2hive + +# load data +loadI2b2Data "$I2B2_DB_NAME" false false diff --git a/build/package/i2b2/sql/10-i2b2-modifications.sh b/build/package/i2b2/sql/10-i2b2-modifications.sh new file mode 100644 index 0000000..d3f0c1c --- /dev/null +++ b/build/package/i2b2/sql/10-i2b2-modifications.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +# set up common medco ontology + +psql $PSQL_PARAMS -d "$I2B2_DB_NAME" <<-EOSQL + + -- increase size of encounter_num values (too large for type INT) + ALTER TABLE i2b2demodata.visit_dimension ALTER COLUMN encounter_num TYPE bigint; + ALTER TABLE i2b2demodata.observation_fact ALTER COLUMN encounter_num TYPE bigint; + ALTER TABLE i2b2demodata.encounter_mapping ALTER COLUMN encounter_num TYPE bigint; + + -- change tval_char to type text + ALTER TABLE i2b2demodata.observation_fact ALTER COLUMN tval_char TYPE text; + +EOSQL diff --git a/build/package/i2b2/sql/20-i2b2-configuration.sh b/build/package/i2b2/sql/20-i2b2-configuration.sh new file mode 100644 index 0000000..c087bc8 --- /dev/null +++ b/build/package/i2b2/sql/20-i2b2-configuration.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" + +# hive configuration +psql $PSQL_PARAMS -d "$I2B2_DB_NAME" <<-EOSQL + + -- database lookups + update i2b2hive.crc_db_lookup SET C_DOMAIN_ID = '$I2B2_DOMAIN_NAME' WHERE C_DOMAIN_ID = 'i2b2demo'; + UPDATE i2b2hive.im_db_lookup SET C_DOMAIN_ID = '$I2B2_DOMAIN_NAME' WHERE C_DOMAIN_ID = 'i2b2demo'; + UPDATE i2b2hive.ont_db_lookup SET C_DOMAIN_ID = '$I2B2_DOMAIN_NAME' WHERE C_DOMAIN_ID = 'i2b2demo'; + UPDATE i2b2hive.work_db_lookup SET C_DOMAIN_ID = '$I2B2_DOMAIN_NAME' WHERE C_DOMAIN_ID = 'i2b2demo'; + + -- CRC cell parameters + UPDATE i2b2hive.hive_cell_params + SET value='-1' + WHERE param_name_cd='edu.harvard.i2b2.crc.lockout.setfinderquery.count'; + UPDATE i2b2hive.hive_cell_params + SET value='$I2B2_SERVICE_PASSWORD' + WHERE param_name_cd='edu.harvard.i2b2.crc.pm.serviceaccount.password'; + +EOSQL + +# PM configuration +I2B2_SERVICE_PASSWORD_HASH=$(java -classpath "$I2B2_PASSWORD_HASH_TOOL" I2b2PasswordHash "$I2B2_SERVICE_PASSWORD") +DEFAULT_USER_PASSWORD_HASH=$(java -classpath "$I2B2_PASSWORD_HASH_TOOL" I2b2PasswordHash "$DEFAULT_USER_PASSWORD") + +psql $PSQL_PARAMS -d "$I2B2_DB_NAME" <<-EOSQL + + -- cell parameters + insert into i2b2pm.pm_cell_params (datatype_cd, cell_id, project_path, param_name_cd, value, changeby_char, status_cd) values + ('T', 'FRC', '/', 'DestDir', '$I2B2_FR_FILES_DIR', 'i2b2', 'A'); + + -- hive & users data + UPDATE i2b2pm.pm_hive_data SET DOMAIN_ID = '$I2B2_DOMAIN_NAME', DOMAIN_NAME = '$I2B2_DOMAIN_NAME' WHERE DOMAIN_ID = 'i2b2'; + + UPDATE i2b2pm.PM_CELL_DATA SET URL = '$I2B2_URL/QueryToolService/' WHERE CELL_ID = 'CRC'; + UPDATE i2b2pm.PM_CELL_DATA SET URL = '$I2B2_URL/FRService/' WHERE CELL_ID = 'FRC'; + UPDATE i2b2pm.PM_CELL_DATA SET URL = '$I2B2_URL/OntologyService/' WHERE CELL_ID = 'ONT'; + UPDATE i2b2pm.PM_CELL_DATA SET URL = '$I2B2_URL/WorkplaceService/' WHERE CELL_ID = 'WORK'; + UPDATE i2b2pm.PM_CELL_DATA SET URL = '$I2B2_URL/IMService/' WHERE CELL_ID = 'IM'; + + UPDATE i2b2pm.PM_USER_DATA SET PASSWORD = '$DEFAULT_USER_PASSWORD_HASH' WHERE USER_ID = 'i2b2'; + UPDATE i2b2pm.PM_USER_DATA SET PASSWORD = '$DEFAULT_USER_PASSWORD_HASH' WHERE USER_ID = 'demo'; + UPDATE i2b2pm.PM_USER_DATA SET PASSWORD = '$I2B2_SERVICE_PASSWORD_HASH' WHERE USER_ID = 'AGG_SERVICE_ACCOUNT'; +EOSQL diff --git a/build/package/i2b2/sql/50-stored-procedures.sh b/build/package/i2b2/sql/50-stored-procedures.sh new file mode 100644 index 0000000..d989970 --- /dev/null +++ b/build/package/i2b2/sql/50-stored-procedures.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -Eeuo pipefail +# add the functions to manipulate the i2b2 data + +for f in "${I2B2_SQL_DIR}/50-stored-procedures/"/*; do + psql ${PSQL_PARAMS} -d "${I2B2_DB_NAME}" -f "$f" +done diff --git a/build/package/i2b2/sql/50-stored-procedures/get_concept_codes.plpgsql b/build/package/i2b2/sql/50-stored-procedures/get_concept_codes.plpgsql new file mode 100644 index 0000000..80a1b69 --- /dev/null +++ b/build/package/i2b2/sql/50-stored-procedures/get_concept_codes.plpgsql @@ -0,0 +1,21 @@ +-- pl/pgsql function to return concept codes for a given concept path and its descendants + +CREATE OR REPLACE FUNCTION i2b2metadata.get_concept_codes(ontology character varying,path character varying) + RETURNS SETOF character varying + LANGUAGE 'plpgsql' + STABLE + PARALLEL SAFE +AS $BODY$ +BEGIN + + RETURN QUERY EXECUTE format( + 'SELECT c_basecode + FROM i2b2metadata.%I + WHERE (c_basecode IS NOT NULL AND c_basecode != %L + AND upper(c_facttablecolumn) = %L + AND c_fullname LIKE $1);', + ontology, '', 'CONCEPT_CD' + ) + USING path; + END; + $BODY$; diff --git a/build/package/i2b2/sql/50-stored-procedures/get_modifier_codes.plpgsql b/build/package/i2b2/sql/50-stored-procedures/get_modifier_codes.plpgsql new file mode 100644 index 0000000..76bd17f --- /dev/null +++ b/build/package/i2b2/sql/50-stored-procedures/get_modifier_codes.plpgsql @@ -0,0 +1,21 @@ +-- pl/pgsql function to return modifier codes for a given modifier path and its descendants, for a given applied path + +CREATE OR REPLACE FUNCTION i2b2metadata.get_modifier_codes(ontology character varying,path character varying,applied_path character varying) + RETURNS SETOF character varying + LANGUAGE 'plpgsql' + STABLE + PARALLEL SAFE +AS $BODY$ +BEGIN + + RETURN QUERY EXECUTE format( + 'SELECT c_basecode + FROM i2b2metadata.%I + WHERE (c_basecode IS NOT NULL AND c_basecode != %L + AND upper(c_facttablecolumn) = %L + AND c_fullname LIKE $1 AND m_applied_path = $2);', + ontology, '', 'MODIFIER_CD' + ) + USING path, applied_path; + END; + $BODY$; diff --git a/build/package/i2b2/sql/50-stored-procedures/get_ontology_elements.plpgsql b/build/package/i2b2/sql/50-stored-procedures/get_ontology_elements.plpgsql new file mode 100644 index 0000000..5763fea --- /dev/null +++ b/build/package/i2b2/sql/50-stored-procedures/get_ontology_elements.plpgsql @@ -0,0 +1,35 @@ +-- pl/pgsql function that returns (maximum lim) ontology elements whose paths contain the given search_string. + +CREATE OR REPLACE FUNCTION i2b2metadata.get_ontology_elements(search_string varchar, lim integer DEFAULT 10) + RETURNS TABLE ( + c_fullname varchar, + c_name varchar, + c_visualattributes char(3), + c_basecode varchar, + c_metadataxml text, + c_comment text, + m_applied_path varchar) + LANGUAGE 'plpgsql' + STABLE + PARALLEL SAFE +AS $BODY$ +DECLARE + rec record; + strSQL text; +BEGIN + + strSQL := ''; + + FOR rec IN SELECT DISTINCT c_table_name FROM i2b2metadata.table_access + LOOP + strSQL := strSQL || 'SELECT c_fullname, c_name, c_visualattributes, c_basecode, c_metadataxml, c_comment, m_applied_path + FROM i2b2metadata.' || lower(rec.c_table_name) || ' + WHERE lower(c_name) LIKE $1 + UNION ALL '; + END LOOP; + strSQL := trim(trailing ' UNION ALL ' from strSQL) || ' ORDER BY c_fullname LIMIT $2;'; + + RETURN QUERY EXECUTE strSQL USING '%' || lower(search_string) || '%', lim; + +END; +$BODY$; diff --git a/build/package/i2b2/sql/50-stored-procedures/table_name.plpgsql b/build/package/i2b2/sql/50-stored-procedures/table_name.plpgsql new file mode 100644 index 0000000..95c953d --- /dev/null +++ b/build/package/i2b2/sql/50-stored-procedures/table_name.plpgsql @@ -0,0 +1,16 @@ +-- pl/pgsql function that returns the table name for a given table code + +CREATE OR REPLACE FUNCTION i2b2metadata.table_name(table_cd varchar) + RETURNS varchar + STABLE + PARALLEL SAFE + AS $$ +DECLARE + table_name_ varchar; +BEGIN + EXECUTE 'SELECT c_table_name from i2b2metadata.table_access WHERE c_table_cd = $1;' + USING table_cd INTO table_name_; + RETURN table_name_; +END; +$$ +LANGUAGE plpgsql diff --git a/build/package/i2b2/sql/common.sh b/build/package/i2b2/sql/common.sh new file mode 100644 index 0000000..afac432 --- /dev/null +++ b/build/package/i2b2/sql/common.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -Eeuo pipefail + +# i2b2 web service URL used in configuration +export I2B2_URL="http://i2b2:8080/i2b2/services" + +function initI2b2Schema { + DB_NAME="$1" + SCHEMA_NAME="$2" + psql $PSQL_PARAMS -d "$DB_NAME" <<-EOSQL + create schema $SCHEMA_NAME; + grant all on schema $SCHEMA_NAME to $I2B2_DB_USER; + grant all privileges on all tables in schema $SCHEMA_NAME to $I2B2_DB_USER; +EOSQL +} From 9abcaf557f5c5ec15b264af24bef1ba21f9017df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 2 Dec 2021 13:18:57 +0100 Subject: [PATCH 02/81] add geco repo as a submodule --- .gitmodules | 3 +++ test/geco | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 test/geco diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4df8ab3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/geco"] + path = test/geco + url = git@github.com:ldsec/geco.git diff --git a/test/geco b/test/geco new file mode 160000 index 0000000..d3f5e8b --- /dev/null +++ b/test/geco @@ -0,0 +1 @@ +Subproject commit d3f5e8b7c84c87ae81bcf1053bc5f9f7428c9784 From 7ecdc41dfcd34a6ae20edf8322ece058dcf4539f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 2 Dec 2021 15:13:36 +0100 Subject: [PATCH 03/81] add test scripts for i2b2 docker --- test/i2b2/.gitignore | 2 + test/i2b2/docker-compose.yml | 24 ++++++++++ test/i2b2/ont_get_categories_req.xml | 53 +++++++++++++++++++++ test/i2b2/pm_get_user_configuration_req.xml | 53 +++++++++++++++++++++ test/i2b2/test_i2b2_docker.sh | 24 ++++++++++ 5 files changed, 156 insertions(+) create mode 100644 test/i2b2/.gitignore create mode 100644 test/i2b2/docker-compose.yml create mode 100644 test/i2b2/ont_get_categories_req.xml create mode 100644 test/i2b2/pm_get_user_configuration_req.xml create mode 100755 test/i2b2/test_i2b2_docker.sh diff --git a/test/i2b2/.gitignore b/test/i2b2/.gitignore new file mode 100644 index 0000000..dfee0c5 --- /dev/null +++ b/test/i2b2/.gitignore @@ -0,0 +1,2 @@ +*.xml.log +*_resp.xml diff --git a/test/i2b2/docker-compose.yml b/test/i2b2/docker-compose.yml new file mode 100644 index 0000000..746ff60 --- /dev/null +++ b/test/i2b2/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.9' +services: + i2b2: + image: ghcr.io/ldsec/i2b2-geco + build: + context: ../../build/package/i2b2 + ports: + - "8080:8080" + environment: + - I2B2_DB_HOST=postgresql + - I2B2_DB_PORT=5432 + - I2B2_DB_USER=postgres + - I2B2_DB_PW=postgres + - I2B2_DB_NAME=i2b2 + - WILDFLY_ADMIN_PASSWORD=admin + - I2B2_DOMAIN_NAME=i2b2demo + - I2B2_SERVICE_PASSWORD=changeme + - DEFAULT_USER_PASSWORD=changeme + - AXIS2_LOGLEVEL=INFO + +networks: + default: + external: true + name: dev-local-3nodes_inter-nodes # join all containers to the GeCo dev-local-3nodes inter-nodes network diff --git a/test/i2b2/ont_get_categories_req.xml b/test/i2b2/ont_get_categories_req.xml new file mode 100644 index 0000000..b7f1fe4 --- /dev/null +++ b/test/i2b2/ont_get_categories_req.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<msgns:request + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns:msgns="http://www.i2b2.org/xsd/hive/msg/1.1/" + xmlns:ontns="http://www.i2b2.org/xsd/cell/ont/1.1/"> + <message_header> + <i2b2_version_compatible>0.3</i2b2_version_compatible> + <hl7_version_compatible>2.4</hl7_version_compatible> + <sending_application> + <application_name>Test</application_name> + <application_version>0.2</application_version> + </sending_application> + <sending_facility> + <facility_name>Test</facility_name> + </sending_facility> + <receiving_application> + <application_name>i2b2 cell</application_name> + <application_version>1.7</application_version> + </receiving_application> + <receiving_facility> + <facility_name>i2b2 hive</facility_name> + </receiving_facility> + <datetime_of_message>2000-01 31T20:59:59.222</datetime_of_message> + <security> + <domain>i2b2demo</domain> + <username>demo</username> + <password>changeme</password> + </security> + <message_type> + <message_code>Q04</message_code> + <event_type>EQQ</event_type> + </message_type> + <message_control_id> + <session_id>2000-01 31T20:59:59.222</session_id> + <message_num>123</message_num> + <instance_num>0</instance_num> + </message_control_id> + <processing_id> + <processing_id>P</processing_id> + <processing_mode>I</processing_mode> + </processing_id> + <accept_acknowledgement_type>messageId</accept_acknowledgement_type> + <application_acknowledgement_type/> + <country_code>CH</country_code> + <project_id>Demo</project_id> + </message_header> + <request_header> + <result_waittime_ms>180000</result_waittime_ms> + </request_header> + <message_body> + <get_categories type="default" blob="false" hiddens="true" synonyms="false"/> + </message_body> +</msgns:request> diff --git a/test/i2b2/pm_get_user_configuration_req.xml b/test/i2b2/pm_get_user_configuration_req.xml new file mode 100644 index 0000000..8ab9d05 --- /dev/null +++ b/test/i2b2/pm_get_user_configuration_req.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<msgns:request + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns:msgns="http://www.i2b2.org/xsd/hive/msg/1.1/" + xmlns:pmns="http://www.i2b2.org/xsd/cell/pm/1.1/"> + <message_header> + <i2b2_version_compatible>0.3</i2b2_version_compatible> + <hl7_version_compatible>2.4</hl7_version_compatible> + <sending_application> + <application_name>Test</application_name> + <application_version>0.2</application_version> + </sending_application> + <sending_facility> + <facility_name>Test</facility_name> + </sending_facility> + <receiving_application> + <application_name>i2b2 cell</application_name> + <application_version>1.7</application_version> + </receiving_application> + <receiving_facility> + <facility_name>i2b2 hive</facility_name> + </receiving_facility> + <datetime_of_message>2000-01 31T20:59:59.222</datetime_of_message> + <security> + <domain>i2b2demo</domain> + <username>demo</username> + <password>changeme</password> + </security> + <message_type> + <message_code>Q04</message_code> + <event_type>EQQ</event_type> + </message_type> + <message_control_id> + <session_id>2000-01 31T20:59:59.222</session_id> + <message_num>123</message_num> + <instance_num>0</instance_num> + </message_control_id> + <processing_id> + <processing_id>P</processing_id> + <processing_mode>I</processing_mode> + </processing_id> + <accept_acknowledgement_type>messageId</accept_acknowledgement_type> + <application_acknowledgement_type/> + <country_code>CH</country_code> + <project_id>Demo</project_id> + </message_header> + <request_header> + <result_waittime_ms>180000</result_waittime_ms> + </request_header> + <message_body> + <pmns:get_user_configuration/> + </message_body> +</msgns:request> diff --git a/test/i2b2/test_i2b2_docker.sh b/test/i2b2/test_i2b2_docker.sh new file mode 100755 index 0000000..79cb2b6 --- /dev/null +++ b/test/i2b2/test_i2b2_docker.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +function postXmlI2b2 { + I2B2_ENDPOINT="$1" + REQ_XML="$2" + RESP_XML="$3" + + echo "HTTP POST $I2B2_ENDPOINT of $REQ_XML..." + curl -v --header "Content-Type:application/xml" -d "@$REQ_XML" -o "$RESP_XML" "http://localhost:8080/i2b2/services/$I2B2_ENDPOINT" > "$RESP_XML.log" 2>&1 + + echo "Checking HTTP code..." + grep "HTTP/1.1 200 OK" "$RESP_XML.log" +} + +MSG=pm_get_user_configuration +postXmlI2b2 PMService/getServices "${MSG}_req.xml" "${MSG}_resp.xml" +xmllint --xpath "string(//status)" "${MSG}_resp.xml" | grep "PM processing completed" +xmllint --xpath "string(//password)" "${MSG}_resp.xml" | grep "SessionKey" + +# todo: with test data, fails with empty DB +#MSG=ont_get_categories +#postXmlI2b2 TBD "${MSG}_req.xml" "${MSG}_resp.xml" +#xmllint --xpath "string(//TBD)" "${MSG}_resp.xml" | grep "TBD" From fd266fcba300599714a07207a699a454498765e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 19:28:49 +0100 Subject: [PATCH 04/81] move geco submodule to vendor dir --- .gitmodules | 4 ++-- {test => vendor/github.com/ldsec}/geco | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename {test => vendor/github.com/ldsec}/geco (100%) diff --git a/.gitmodules b/.gitmodules index 4df8ab3..e4905dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "test/geco"] - path = test/geco +[submodule "vendor/github.com/ldsec/geco"] + path = vendor/github.com/ldsec/geco url = git@github.com:ldsec/geco.git diff --git a/test/geco b/vendor/github.com/ldsec/geco similarity index 100% rename from test/geco rename to vendor/github.com/ldsec/geco From 41c93580e21670cfe1e9a1941b890dd1b66a2e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 20:39:45 +0100 Subject: [PATCH 05/81] reset submodule --- .gitmodules | 3 --- vendor/github.com/ldsec/geco | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 vendor/github.com/ldsec/geco diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e4905dc..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "vendor/github.com/ldsec/geco"] - path = vendor/github.com/ldsec/geco - url = git@github.com:ldsec/geco.git diff --git a/vendor/github.com/ldsec/geco b/vendor/github.com/ldsec/geco deleted file mode 160000 index d3f5e8b..0000000 --- a/vendor/github.com/ldsec/geco +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d3f5e8b7c84c87ae81bcf1053bc5f9f7428c9784 From 740e1a6147e3dd0d9ead6acd20e430afcd0e757a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 20:40:42 +0100 Subject: [PATCH 06/81] properly add back geco as submodule --- .gitmodules | 3 +++ third_party/geco | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 third_party/geco diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ef67b5a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/geco"] + path = third_party/geco + url = git@github.com:ldsec/geco.git diff --git a/third_party/geco b/third_party/geco new file mode 160000 index 0000000..d3f5e8b --- /dev/null +++ b/third_party/geco @@ -0,0 +1 @@ +Subproject commit d3f5e8b7c84c87ae81bcf1053bc5f9f7428c9784 From 89208b98f1ec1499cbcfb2209bcd6ecb54352a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 21:04:52 +0100 Subject: [PATCH 07/81] first attempt at GA CI --- .github/workflows/i2b2.yml | 87 ++++++++++++++++++++++++++++++++++++++ Makefile | 67 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 .github/workflows/i2b2.yml create mode 100644 Makefile diff --git a/.github/workflows/i2b2.yml b/.github/workflows/i2b2.yml new file mode 100644 index 0000000..d52e1b0 --- /dev/null +++ b/.github/workflows/i2b2.yml @@ -0,0 +1,87 @@ +name: Build i2b2 docker image and run some tests +on: + push: #todo: determine when to trigger + workflow_dispatch: +jobs: + i2b2: + name: i2b2 docker image build and test + runs-on: ubuntu-latest +# env: +# PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} + + steps: + - name: Environment + run: | + go version + env + + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + +# - name: Setup SSH deploy key for private repositories +# uses: webfactory/ssh-agent@v0.4.1 +# with: +# ssh-private-key: ${{ secrets.GECO_SSH_DEPLOY_KEY }} + +# - name: Setup Git and Golang environment for private repositories +# run: | +# git config --global url."git@github.com:".insteadOf "https://github.com/" +# go env -w GOPRIVATE=github.com/ldsec/geco,github.com/ldsec/spindle,github.com/ldsec/geco-i2b2-data-source + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + +# - name: Get versions +# id: get_versions +# run: | +# VERSION=$(scripts/version.sh) +# GECO_VERSION=$(cd test/geco && scripts/version.sh) +# echo ::set-output name=version::${VERSION} +# echo ::set-output name=geco_version::${GECO_VERSION} +# echo ::set-output name=i2b2_docker_tag::ghcr.io/ldsec/i2b2-geco:${VERSION} + +# # todo: GET VERSION OF i2b2 SOMEWHERE AND SET THIS AS VERSION ON DOCKER +# use a variant of PUSH? or have it only manual? in terms of being triggered + + - name: Build i2b2 Docker image + uses: docker/build-push-action@v2 + with: + context: ./build/package/i2b2 + build-args: BUILD=docker + load: true + ssh: default + # tags: ${{ steps.get_version.outputs.docker_tag }} + tags: ghcr.io/ldsec/i2b2-geco + cache-from: type=gha,scope=buildkit + cache-to: type=gha,scope=buildkit,mode=max + + - name: Start GeCo deployment + run: | + make start-geco-dev-local-3nodes + sleep 10 # todo + + - name: Start i2b2 + run: | + make i2b2-docker-compose ARGS="up -d" + sleep 10 # todo + + - name: Run i2b2 test + run: make test-i2b2 + + - name: Show deployment logs + if: always() + run: make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="logs" + +# - name: Login to GitHub Container Registry +# if: ${{ env.PUSH }} +# uses: docker/login-action@v1 +# with: +# registry: ghcr.io +# username: ${{ secrets.LDS_PKG_PAT_USERNAME }} +# password: ${{ secrets.LDS_PKG_PAT }} + +# - name: Push GeCo Docker image +# if: ${{ env.PUSH }} +# run: docker push ${{ steps.get_version.outputs.i2b2_docker_tag }} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..32539dd --- /dev/null +++ b/Makefile @@ -0,0 +1,67 @@ +VERSION := $(shell scripts/version.sh) +USER_GROUP := $(shell id -u):$(shell id -g) +DOCKER_IMAGE ?= ghcr.io/ldsec/geco-i2b2-data-source:$(VERSION) + +export COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 + +# todo: set up CI with DB from geco for later , ensure exec times are OK +# todo: assumes geco submodule has been retrioeved: +# todo: gha caching https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching +# -> connect as postgres in geco DB like this +# docker exec -it i2b2postgresql psql -U postgres +# CREATE ROLE i2b2 LOGIN PASSWORD 'i2b2'; +# ALTER USER i2b2 CREATEDB; + +# todo: make clean swagger-gen +# todo pushd third_party/geco && make clean swagger-gen && popd + +start-geco-dev-local-3nodes: + make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="up -d postgresql" + # todo: only the DB at the moment + +# use ARGS to pass to docker-compose arguments, e.g. make docker-compose ARGS="up -d" +i2b2-docker-compose: + cd test/i2b2 && docker-compose -f docker-compose.yml $(ARGS) + # todo: version etc. to pass + +test-i2b2: + cd test/i2b2 && ./test_i2b2_docker.sh + +# --- go source code +.PHONY: build-bin test-go test clean +build-plugin: + go build -buildmode=plugin -v -o ./build/ ./cmd/... +test-go: go-imports go-lint go-unit-tests +clean: go-swagger-clean + rm -f ./build/geco-cli ./build/geco-server + +.PHONY: go-imports go-lint go-unit-tests +go-imports: + @echo Checking correct formatting of files + @{ \ + GO111MODULE=off go get -u golang.org/x/tools/cmd/goimports; \ + files=$$( goimports -w -l . ); \ + if [ -n "$$files" ]; then \ + echo "Files not properly formatted: $$files"; \ + exit 1; \ + fi; \ + if ! go vet ./...; then \ + exit 1; \ + fi \ + } + +go-lint: + @echo Checking linting of files + @{ \ + GO111MODULE=off go get -u golang.org/x/lint/golint; \ + el="_test.go"; \ + lintfiles=$$( golint ./... | egrep -v "$$el" ); \ + if [ -n "$$lintfiles" ]; then \ + echo "Lint errors:"; \ + echo "$$lintfiles"; \ + exit 1; \ + fi \ + } + +go-unit-tests: + go test -v -short -p=1 ./... From a3d42fb737f672255e7cfa4279dd376059b931f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 21:08:01 +0100 Subject: [PATCH 08/81] CI: set up SSH key --- .github/workflows/i2b2.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/i2b2.yml b/.github/workflows/i2b2.yml index d52e1b0..0e8fa45 100644 --- a/.github/workflows/i2b2.yml +++ b/.github/workflows/i2b2.yml @@ -15,16 +15,16 @@ jobs: go version env + - name: Setup SSH deploy key for private repositories + uses: webfactory/ssh-agent@v0.4.1 + with: + ssh-private-key: ${{ secrets.GECO_SSH_DEPLOY_KEY }} + - name: Checkout code uses: actions/checkout@v2 with: submodules: true -# - name: Setup SSH deploy key for private repositories -# uses: webfactory/ssh-agent@v0.4.1 -# with: -# ssh-private-key: ${{ secrets.GECO_SSH_DEPLOY_KEY }} - # - name: Setup Git and Golang environment for private repositories # run: | # git config --global url."git@github.com:".insteadOf "https://github.com/" From 1ea9be38a48d6e529f539e988aedc9655758c8ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 3 Dec 2021 21:11:04 +0100 Subject: [PATCH 09/81] CI: set up checkout token --- .github/workflows/i2b2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/i2b2.yml b/.github/workflows/i2b2.yml index 0e8fa45..87c9609 100644 --- a/.github/workflows/i2b2.yml +++ b/.github/workflows/i2b2.yml @@ -23,6 +23,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 with: + token: ${{ secrets.LDS_PKG_PAT }} submodules: true # - name: Setup Git and Golang environment for private repositories From e1f1e9ba224fc5577e51d9b3fd9a01b7cd6c7496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sat, 4 Dec 2021 13:52:14 +0100 Subject: [PATCH 10/81] CI: remove sleep between steps and add active check for i2b2 to be up --- .github/workflows/i2b2.yml | 3 +-- Makefile | 2 +- test/i2b2/test_i2b2_docker.sh | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/i2b2.yml b/.github/workflows/i2b2.yml index 87c9609..49ce652 100644 --- a/.github/workflows/i2b2.yml +++ b/.github/workflows/i2b2.yml @@ -1,4 +1,5 @@ name: Build i2b2 docker image and run some tests +# todo: i2b2 should be built only on request, or on change of something on: push: #todo: determine when to trigger workflow_dispatch: @@ -61,12 +62,10 @@ jobs: - name: Start GeCo deployment run: | make start-geco-dev-local-3nodes - sleep 10 # todo - name: Start i2b2 run: | make i2b2-docker-compose ARGS="up -d" - sleep 10 # todo - name: Run i2b2 test run: make test-i2b2 diff --git a/Makefile b/Makefile index 32539dd..92dcd95 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION := $(shell scripts/version.sh) +#VERSION := $(shell scripts/version.sh) USER_GROUP := $(shell id -u):$(shell id -g) DOCKER_IMAGE ?= ghcr.io/ldsec/geco-i2b2-data-source:$(VERSION) diff --git a/test/i2b2/test_i2b2_docker.sh b/test/i2b2/test_i2b2_docker.sh index 79cb2b6..7671387 100755 --- a/test/i2b2/test_i2b2_docker.sh +++ b/test/i2b2/test_i2b2_docker.sh @@ -13,6 +13,12 @@ function postXmlI2b2 { grep "HTTP/1.1 200 OK" "$RESP_XML.log" } +# wait for i2b2 to be up +until curl -v http://localhost:8080/i2b2/services/listServices; do + >&2 echo "Waiting for i2b2..." + sleep 1 +done + MSG=pm_get_user_configuration postXmlI2b2 PMService/getServices "${MSG}_req.xml" "${MSG}_resp.xml" xmllint --xpath "string(//status)" "${MSG}_resp.xml" | grep "PM processing completed" From 7b10b4e39a8b8218565e633aacdc12c386b38df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sat, 4 Dec 2021 13:59:50 +0100 Subject: [PATCH 11/81] CI: add xmllint tool --- .github/workflows/i2b2.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/i2b2.yml b/.github/workflows/i2b2.yml index 49ce652..b7d3334 100644 --- a/.github/workflows/i2b2.yml +++ b/.github/workflows/i2b2.yml @@ -68,7 +68,9 @@ jobs: make i2b2-docker-compose ARGS="up -d" - name: Run i2b2 test - run: make test-i2b2 + run: | + sudo apt-get install -y libxml2-utils + make test-i2b2 - name: Show deployment logs if: always() From 091d9d56b00858ad4f5d07f3ecd719e4dcd55cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:16:43 +0100 Subject: [PATCH 12/81] add data source interface --- pkg/geco-data-source.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 pkg/geco-data-source.go diff --git a/pkg/geco-data-source.go b/pkg/geco-data-source.go new file mode 100644 index 0000000..8e2cd2c --- /dev/null +++ b/pkg/geco-data-source.go @@ -0,0 +1,20 @@ +package pkg + +import ( + "github.com/ldsec/geco/pkg/datamanager" + "github.com/sirupsen/logrus" +) + +// todo: move this interface to geco once finalized + +// DataSource defines a GeCo data source plugin. The plugin must export a variable named DataSourcePlugin of the +// type DataSource to be compatible. +type DataSource interface { + + // Init the data source with the provided configuration. + Init(dm *datamanager.DataManager, logger *logrus.Logger, config map[string]string) error + + // Query data source with a specific operation. + // todo: parameters and results are json marshallable -> some other interface instead? check what swagger provides or not + Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) +} From d6d22b18474d27548edbea5eaac32dc58b899e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:17:07 +0100 Subject: [PATCH 13/81] add i2b2 XML API models --- pkg/i2b2/models/api_tmp.go | 149 +++++++++++++++++++ pkg/i2b2/models/common.go | 184 ++++++++++++++++++++++++ pkg/i2b2/models/crc_pdo.go | 103 ++++++++++++++ pkg/i2b2/models/crc_psm.go | 284 +++++++++++++++++++++++++++++++++++++ pkg/i2b2/models/ont.go | 208 +++++++++++++++++++++++++++ pkg/i2b2/models/util.go | 11 ++ 6 files changed, 939 insertions(+) create mode 100644 pkg/i2b2/models/api_tmp.go create mode 100644 pkg/i2b2/models/common.go create mode 100644 pkg/i2b2/models/crc_pdo.go create mode 100644 pkg/i2b2/models/crc_psm.go create mode 100644 pkg/i2b2/models/ont.go create mode 100644 pkg/i2b2/models/util.go diff --git a/pkg/i2b2/models/api_tmp.go b/pkg/i2b2/models/api_tmp.go new file mode 100644 index 0000000..1bb678e --- /dev/null +++ b/pkg/i2b2/models/api_tmp.go @@ -0,0 +1,149 @@ +package models + +// todo: those are c/c from swagger-generated medco models, they should be updated according to the definition of the API exposed through geco + +// APIPanel todo: should be changed according to the API exposed +type APIPanel struct { + + // items containing cohort names + CohortItems []string `json:"cohortItems"` + + // items containing i2b2 concepts (and optionally modifiers) + ConceptItems []*APIPanelConceptItemsItems0 `json:"conceptItems"` + + // exclude the i2b2 panel + // Required: true + Not *bool `json:"not"` + + // panel timing + PanelTiming APITiming `json:"panelTiming,omitempty"` +} + +// APIPanelConceptItemsItems0 todo: should be changed according to the API exposed +type APIPanelConceptItemsItems0 struct { + + // encrypted + // Required: true + Encrypted *bool `json:"encrypted"` + + // modifier + Modifier *APIPanelConceptItemsItems0Modifier `json:"modifier,omitempty"` + + // # NUMBER operators EQ: equal NE: not equal GT: greater than GE: greater than or equal LT: less than LE: less than or equal BETWEEN: between (value syntax: "x and y") # TEXT operators IN: in (value syntax: "'x','y','z'") LIKE[exact]: equal LIKE[begin]: begins with LIKE[end]: ends with LIKE[contains]: contains + // + // Enum: [EQ NE GT GE LT LE BETWEEN IN LIKE[exact] LIKE[begin] LIKE[end] LIKE[contains]] + Operator string `json:"operator,omitempty"` + + // query term + // Required: true + // Pattern: ^([\w=-]+)$|^((\/[^\/]+)+\/)$ + QueryTerm *string `json:"queryTerm"` + + // type + // Enum: [NUMBER TEXT] + Type string `json:"type,omitempty"` + + // value + Value string `json:"value,omitempty"` +} + +// APIPanelConceptItemsItems0Modifier todo: should be changed according to the API exposed +type APIPanelConceptItemsItems0Modifier struct { + + // applied path + // Required: true + // Pattern: ^((\/[^\/]+)+\/%?)$ + AppliedPath *string `json:"appliedPath"` + + // modifier key + // Required: true + // Pattern: ^((\/[^\/]+)+\/)$ + ModifierKey *string `json:"modifierKey"` +} + +// APITiming todo: should be changed according to the API exposed +type APITiming string +const ( + // APITimingAny captures enum value "any" + APITimingAny APITiming = "any" + + // APITimingSamevisit captures enum value "samevisit" + APITimingSamevisit APITiming = "samevisit" + + // APITimingSameinstancenum captures enum value "sameinstancenum" + APITimingSameinstancenum APITiming = "sameinstancenum" +) + +// Metadataxml todo: should be changed according to the API exposed +type Metadataxml struct { + + // value metadata + ValueMetadata *MetadataxmlValueMetadata `json:"ValueMetadata,omitempty"` +} + + +// MetadataxmlValueMetadata todo: should be changed according to the API exposed +type MetadataxmlValueMetadata struct { + + // children encrypt i ds + ChildrenEncryptIDs string `json:"ChildrenEncryptIDs,omitempty"` + + // creation date time + CreationDateTime string `json:"CreationDateTime,omitempty"` + + // data type + DataType string `json:"DataType,omitempty"` + + // encrypted type + EncryptedType string `json:"EncryptedType,omitempty"` + + // enum values + EnumValues string `json:"EnumValues,omitempty"` + + // flagstouse + Flagstouse string `json:"Flagstouse,omitempty"` + + // node encrypt ID + NodeEncryptID string `json:"NodeEncryptID,omitempty"` + + // oktousevalues + Oktousevalues string `json:"Oktousevalues,omitempty"` + + // test ID + TestID string `json:"TestID,omitempty"` + + // test name + TestName string `json:"TestName,omitempty"` + + // unit values + UnitValues *UnitValues `json:"UnitValues,omitempty"` + + // version + Version string `json:"Version,omitempty"` +} + +// UnitValues todo: should be changed according to the API exposed +type UnitValues struct { + + // converting units + ConvertingUnits []*UnitValuesConvertingUnitsItems0 `json:"ConvertingUnits"` + + // equal units + EqualUnits []string `json:"EqualUnits"` + + // excluding units + ExcludingUnits []string `json:"ExcludingUnits"` + + // normal units + NormalUnits string `json:"NormalUnits,omitempty"` +} + +// UnitValuesConvertingUnitsItems0 todo: should be changed according to the API exposed +type UnitValuesConvertingUnitsItems0 struct { + + // multiplying factor + MultiplyingFactor string `json:"MultiplyingFactor,omitempty"` + + // units + Units string `json:"Units,omitempty"` +} diff --git a/pkg/i2b2/models/common.go b/pkg/i2b2/models/common.go new file mode 100644 index 0000000..7267b6f --- /dev/null +++ b/pkg/i2b2/models/common.go @@ -0,0 +1,184 @@ +package models + +import ( + "encoding/xml" + "errors" + "strconv" + "time" +) + +// ConnectionInfo contains the data needed for connection to i2b2. +type ConnectionInfo struct { + + // HiveURL is the URL of the i2b2 hive + HiveURL string + + // Domain is the i2b2 login domain + Domain string + + // Username is the i2b2 login username + Username string + + // Password is the i2b2 login password + Password string + + // Project is the i2b2 project ID + Project string + + // WaitTime is the wait time to send with i2b2 requests + WaitTime time.Duration +} + +// NewRequest creates a new ready-to-use i2b2 request, with a nil message body. +func NewRequest(ci ConnectionInfo) Request { + now := time.Now() + return Request{ + XMLNSMSG: "http://www.i2b2.org/xsd/hive/msg/1.1/", + XMLNSONT: "http://www.i2b2.org/xsd/cell/ont/1.1/", + XMLNSPDO: "http://www.i2b2.org/xsd/hive/pdo/1.1/", + XMLNSCRCPDO: "http://www.i2b2.org/xsd/cell/crc/pdo/1.1/", + XMLNSCRCPSM: "http://www.i2b2.org/xsd/cell/crc/psm/1.1/", + + MessageHeader: MessageHeader{ + I2b2VersionCompatible: "0.3", + Hl7VersionCompatible: "2.4", + SendingApplicationApplicationName: "GeCo i2b2 Data Source", + SendingApplicationApplicationVersion: "0.2", + SendingFacilityFacilityName: "GeCo", + ReceivingApplicationApplicationName: "i2b2 cell", + ReceivingApplicationApplicationVersion: "1.7", + ReceivingFacilityFacilityName: "i2b2 hive", + DatetimeOfMessage: now.Format(time.RFC3339), + SecurityDomain: ci.Domain, + SecurityUsername: ci.Username, + SecurityPassword: ci.Password, + MessageTypeMessageCode: "EQQ", + MessageTypeEventType: "Q04", + MessageTypeMessageStructure: "EQQ_Q04", + MessageControlIDSessionID: now.Format(time.RFC3339), + MessageControlIDMessageNum: strconv.FormatInt(now.Unix(), 10), + MessageControlIDInstanceNum: "0", + ProcessingIDProcessingID: "P", + ProcessingIDProcessingMode: "I", + AcceptAcknowledgementType: "messageId", + ApplicationAcknowledgementType: "", + CountryCode: "CH", + ProjectID: ci.Project, + }, + RequestHeader: RequestHeader{ + ResultWaittimeMs: strconv.FormatInt(ci.WaitTime.Milliseconds(), 10), + }, + } +} + +// NewRequestWithBody creates a new ready-to-use i2b2 request, with a message body +func NewRequestWithBody(ci ConnectionInfo, body MessageBody) (req Request) { + req = NewRequest(ci) + req.MessageBody = body + return +} + +// Request is an i2b2 XML request +type Request struct { + XMLName xml.Name `xml:"msgns:request"` + XMLNSMSG string `xml:"xmlns:msgns,attr"` + XMLNSPDO string `xml:"xmlns:pdons,attr"` + XMLNSONT string `xml:"xmlns:ontns,attr"` + XMLNSCRCPDO string `xml:"xmlns:crcpdons,attr"` + XMLNSCRCPSM string `xml:"xmlns:crcpsmns,attr"` + + MessageHeader MessageHeader `xml:"message_header"` + RequestHeader RequestHeader `xml:"request_header"` + MessageBody MessageBody `xml:"message_body"` +} + +// Response is an i2b2 XML response +type Response struct { + XMLName xml.Name `xml:"response"` + MessageHeader MessageHeader `xml:"message_header"` + RequestHeader RequestHeader `xml:"request_header"` + ResponseHeader ResponseHeader `xml:"response_header"` + MessageBody MessageBody `xml:"message_body"` +} + +func (response *Response) CheckStatus() error { + if response.ResponseHeader.ResultStatus.Status.Type != "DONE" { + return errors.New(response.ResponseHeader.ResultStatus.Status.Text) + } + return nil +} + +// MessageHeader is an i2b2 XML header embedded in a request or response +type MessageHeader struct { + XMLName xml.Name `xml:"message_header"` + + I2b2VersionCompatible string `xml:"i2b2_version_compatible"` + Hl7VersionCompatible string `xml:"hl7_version_compatible"` + + SendingApplicationApplicationName string `xml:"sending_application>application_name"` + SendingApplicationApplicationVersion string `xml:"sending_application>application_version"` + + SendingFacilityFacilityName string `xml:"sending_facility>facility_name"` + + ReceivingApplicationApplicationName string `xml:"receiving_application>application_name"` + ReceivingApplicationApplicationVersion string `xml:"receiving_application>application_version"` + + ReceivingFacilityFacilityName string `xml:"receiving_facility>facility_name"` + + DatetimeOfMessage string `xml:"datetime_of_message"` + + SecurityDomain string `xml:"security>domain"` + SecurityUsername string `xml:"security>username"` + SecurityPassword string `xml:"security>password"` + + MessageTypeMessageCode string `xml:"message_type>message_code"` + MessageTypeEventType string `xml:"message_type>event_type"` + MessageTypeMessageStructure string `xml:"message_type>message_structure"` + + MessageControlIDSessionID string `xml:"message_control_id>session_id"` + MessageControlIDMessageNum string `xml:"message_control_id>message_num"` + MessageControlIDInstanceNum string `xml:"message_control_id>instance_num"` + + ProcessingIDProcessingID string `xml:"processing_id>processing_id"` + ProcessingIDProcessingMode string `xml:"processing_id>processing_mode"` + + AcceptAcknowledgementType string `xml:"accept_acknowledgement_type"` + ApplicationAcknowledgementType string `xml:"application_acknowledgement_type"` + CountryCode string `xml:"country_code"` + ProjectID string `xml:"project_id"` +} + +// RequestHeader is an i2b2 XML header embedded in a request +type RequestHeader struct { + XMLName xml.Name `xml:"request_header"` + ResultWaittimeMs string `xml:"result_waittime_ms"` +} + +// ResponseHeader is an i2b2 XML header embedded in a response +type ResponseHeader struct { + XMLName xml.Name `xml:"response_header"` + Info struct { + Text string `xml:",chardata"` + URL string `xml:"url,attr"` + } `xml:"info"` + ResultStatus struct { + Status struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"status"` + PollingURL struct { + Text string `xml:",chardata"` + IntervalMs string `xml:"interval_ms,attr"` + } `xml:"polling_url"` + Conditions struct { + Condition []struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + CodingSystem string `xml:"coding_system,attr"` + } `xml:"condition"` + } `xml:"conditions"` + } `xml:"result_status"` +} + +// MessageBody is an i2b2 XML generic body +type MessageBody interface{} diff --git a/pkg/i2b2/models/crc_pdo.go b/pkg/i2b2/models/crc_pdo.go new file mode 100644 index 0000000..762fe6e --- /dev/null +++ b/pkg/i2b2/models/crc_pdo.go @@ -0,0 +1,103 @@ +package models + +import ( + "encoding/xml" +) + +// NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request +func NewCrcPdoReqFromInputList(ci ConnectionInfo, patientSetID string) Request { + + // PDO header + pdoHeader := PdoHeader{ + PatientSetLimit: "0", + EstimatedTime: "0", + RequestType: "getPDO_fromInputList", + } + + // PDO request + pdoRequest := PdoRequestFromInputList{ + Type: "crcpdons:GetPDOFromInputList_requestType", + Xsi: "http://www.w3.org/2001/XMLSchema-instance", + } + + // set request for patient set ID + pdoRequest.InputList.PatientList.Max = "1000000" + pdoRequest.InputList.PatientList.Min = "0" + pdoRequest.InputList.PatientList.PatientSetCollID = patientSetID + pdoRequest.OutputOption.Name = "none" + pdoRequest.OutputOption.PatientSet.Blob = "false" + pdoRequest.OutputOption.PatientSet.TechData = "false" + pdoRequest.OutputOption.PatientSet.OnlyKeys = "false" + pdoRequest.OutputOption.PatientSet.Select = "using_input_list" + + return NewRequestWithBody(ci, CrcPdoReqFromInputListMessageBody{ + PdoHeader: pdoHeader, + PdoRequest: pdoRequest, + }) +} + +// --- request + +// CrcPdoReqFromInputListMessageBody is an i2b2 XML message body for CRC PDO request from input list +type CrcPdoReqFromInputListMessageBody struct { + XMLName xml.Name `xml:"message_body"` + + PdoHeader PdoHeader `xml:"crcpdons:pdoheader"` + PdoRequest PdoRequestFromInputList `xml:"crcpdons:request"` +} + +// PdoHeader is an i2b2 XML header for PDO requests +type PdoHeader struct { + PatientSetLimit string `xml:"patient_set_limit"` + EstimatedTime string `xml:"estimated_time"` + RequestType string `xml:"request_type"` +} + +// PdoRequestFromInputList is an i2b2 XML PDO request - from input list +type PdoRequestFromInputList struct { + Type string `xml:"xsi:type,attr"` + Xsi string `xml:"xmlns:xsi,attr"` + + InputList struct { + PatientList struct { + Max string `xml:"max,attr"` + Min string `xml:"min,attr"` + PatientSetCollID string `xml:"patient_set_coll_id"` + } `xml:"patient_list,omitempty"` + } `xml:"input_list"` + + OutputOption struct { + Name string `xml:"name,attr"` + PatientSet struct { + Select string `xml:"select,attr"` + OnlyKeys string `xml:"onlykeys,attr"` + Blob string `xml:"blob,attr"` + TechData string `xml:"techdata,attr"` + } `xml:"patient_set,omitempty"` + } `xml:"output_option"` +} + +// --- response + +// CrcPdoRespMessageBody is an i2b2 XML message body for CRC PDO response +type CrcPdoRespMessageBody struct { + XMLName xml.Name `xml:"message_body"` + + Response struct { + Xsi string `xml:"xsi,attr"` + Type string `xml:"type,attr"` + PatientData struct { + PatientSet struct { + Patient []struct { + PatientID string `xml:"patient_id"` + Param []struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + ColumnDescriptor string `xml:"column_descriptor,attr"` + Column string `xml:"column,attr"` + } `xml:"param"` + } `xml:"patient"` + } `xml:"patient_set"` + } `xml:"patient_data"` + } `xml:"response"` +} diff --git a/pkg/i2b2/models/crc_psm.go b/pkg/i2b2/models/crc_psm.go new file mode 100644 index 0000000..0df25e4 --- /dev/null +++ b/pkg/i2b2/models/crc_psm.go @@ -0,0 +1,284 @@ +package models + +import ( + "encoding/xml" + "errors" + "strconv" + "strings" +) + +// NewCrcPsmReqFromQueryDef returns a new request object for i2b2 psm request +func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels []*APIPanel, + resultOutputs []ResultOutputName, queryTiming APITiming) Request { + + // PSM header + psmHeader := PsmHeader{ + PatientSetLimit: "0", + EstimatedTime: "0", + QueryMode: "optimize_without_temp_table", + RequestType: "CRC_QRY_runQueryInstance_fromQueryDefinition", + } + psmHeader.User.Text = ci.Username + psmHeader.User.Group = ci.Domain + psmHeader.User.Login = ci.Username + + // PSM request + psmRequest := PsmRequestFromQueryDef{ + Type: "crcpsmns:query_definition_requestType", + Xsi: "http://www.w3.org/2001/XMLSchema-instance", + + QueryName: queryName, + QueryID: queryName, + QueryDescription: "Query from GeCo i2b2 data source (" + queryName + ")", + QueryTiming: strings.ToUpper(string(queryTiming)), + SpecificityScale: "0", + } + + // embed query in request + for p, queryPanel := range queryPanels { + + invert := "0" + if *queryPanel.Not { + invert = "1" + } + + i2b2Panel := Panel{ + PanelNumber: strconv.Itoa(p + 1), + PanelAccuracyScale: "100", + Invert: invert, + PanelTiming: strings.ToUpper(string(queryPanel.PanelTiming)), + TotalItemOccurrences: "1", + } + + for _, queryItem := range queryPanel.ConceptItems { + i2b2Item := Item{ + ItemKey: ConvertPathToI2b2Format(*queryItem.QueryTerm), + } + if queryItem.Operator != "" && queryItem.Modifier == nil { + i2b2Item.ConstrainByValue = &ConstrainByValue{ + ValueType: queryItem.Type, + ValueOperator: queryItem.Operator, + ValueConstraint: queryItem.Value, + } + } + if queryItem.Modifier != nil { + i2b2Item.ConstrainByModifier = &ConstrainByModifier{ + AppliedPath: strings.ReplaceAll(*queryItem.Modifier.AppliedPath, "/", `\`), + ModifierKey: ConvertPathToI2b2Format(*queryItem.Modifier.ModifierKey), + } + if queryItem.Operator != "" { + i2b2Item.ConstrainByModifier.ConstrainByValue = &ConstrainByValue{ + ValueType: queryItem.Type, + ValueOperator: queryItem.Operator, + ValueConstraint: queryItem.Value, + } + } + } + i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) + } + + for _, cohort := range queryPanel.CohortItems { + + i2b2Item := Item{ + ItemKey: cohort, + } + i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) + } + + psmRequest.Panels = append(psmRequest.Panels, i2b2Panel) + } + + // embed result outputs + for i, resultOutput := range resultOutputs { + psmRequest.ResultOutputs = append(psmRequest.ResultOutputs, ResultOutput{ + PriorityIndex: strconv.Itoa(i + 1), + Name: string(resultOutput), + }) + } + + return NewRequestWithBody(ci, CrcPsmReqFromQueryDefMessageBody{ + PsmHeader: psmHeader, + PsmRequest: psmRequest, + }) +} + +// --- request + +// CrcPsmReqFromQueryDefMessageBody is an i2b2 XML message body for CRC PSM request from query definition +type CrcPsmReqFromQueryDefMessageBody struct { + XMLName xml.Name `xml:"message_body"` + + PsmHeader PsmHeader `xml:"crcpsmns:psmheader"` + PsmRequest PsmRequestFromQueryDef `xml:"crcpsmns:request"` +} + +// PsmHeader is an i2b2 XML header for PSM request +type PsmHeader struct { + User struct { + Text string `xml:",chardata"` + Group string `xml:"group,attr"` + Login string `xml:"login,attr"` + } `xml:"user"` + + PatientSetLimit string `xml:"patient_set_limit"` + EstimatedTime string `xml:"estimated_time"` + QueryMode string `xml:"query_mode"` + RequestType string `xml:"request_type"` +} + +// PsmRequestFromQueryDef is an i2b2 XML PSM request from query definition +type PsmRequestFromQueryDef struct { + Type string `xml:"xsi:type,attr"` + Xsi string `xml:"xmlns:xsi,attr"` + + QueryName string `xml:"query_definition>query_name"` + QueryDescription string `xml:"query_definition>query_description"` + QueryID string `xml:"query_definition>query_id"` + QueryTiming string `xml:"query_definition>query_timing"` + SpecificityScale string `xml:"query_definition>specificity_scale"` + Panels []Panel `xml:"query_definition>panel"` + + ResultOutputs []ResultOutput `xml:"result_output_list>result_output"` +} + +// Panel is an i2b2 XML panel +type Panel struct { + PanelNumber string `xml:"panel_number"` + PanelAccuracyScale string `xml:"panel_accuracy_scale"` + Invert string `xml:"invert"` + PanelTiming string `xml:"panel_timing"` + TotalItemOccurrences string `xml:"total_item_occurrences"` + + Items []Item `xml:"item"` +} + +// Item is an i2b2 XML item +type Item struct { + Hlevel string `xml:"hlevel"` + ItemName string `xml:"item_name"` + ItemKey string `xml:"item_key"` + Tooltip string `xml:"tooltip"` + Class string `xml:"class"` + ConstrainByValue *ConstrainByValue `xml:"constrain_by_value,omitempty"` + ConstrainByModifier *ConstrainByModifier `xml:"constrain_by_modifier,omitempty"` + ItemIcon string `xml:"item_icon"` + ItemIsSynonym string `xml:"item_is_synonym"` +} + +// ConstrainByModifier is an i2b2 XML constrain_by_modifier element +type ConstrainByModifier struct { + AppliedPath string `xml:"applied_path"` + ModifierKey string `xml:"modifier_key"` + ConstrainByValue *ConstrainByValue `xml:"constrain_by_value"` +} + +// ConstrainByValue is an i2b2 XML constrain_by_value element +type ConstrainByValue struct { + ValueType string `xml:"value_type"` + ValueOperator string `xml:"value_operator"` + ValueConstraint string `xml:"value_constraint"` +} + +// ResultOutput is an i2b2 XML requested result type +type ResultOutput struct { + PriorityIndex string `xml:"priority_index,attr"` + Name string `xml:"name,attr"` +} + +// ResultOutputName is an i2b2 XML requested result type value +type ResultOutputName string + +// enumerated values of ResultOutputName +const ( + Patientset ResultOutputName = "PATIENTSET" + PatientEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" + PatientCountXML ResultOutputName = "PATIENT_COUNT_XML" + PatientGenderCountXML ResultOutputName = "PATIENT_GENDER_COUNT_XML" + PatientAgeCountXML ResultOutputName = "PATIENT_AGE_COUNT_XML" + PatientVitalstatusCountXML ResultOutputName = "PATIENT_VITALSTATUS_COUNT_XML" + PatientRaceCountXML ResultOutputName = "PATIENT_RACE_COUNT_XML" +) + +// --- response + +// CrcPsmRespMessageBody is an i2b2 XML message body for CRC PSM response +type CrcPsmRespMessageBody struct { + XMLName xml.Name `xml:"message_body"` + + Response struct { + Type string `xml:"type,attr"` + + Status []struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"status>condition"` + + QueryMasters []struct { + QueryMasterID string `xml:"query_master_id"` + Name string `xml:"name"` + UserID string `xml:"user_id"` + GroupID string `xml:"group_id"` + CreateDate string `xml:"create_date"` + DeleteDate string `xml:"delete_date"` + RequestXML string `xml:"request_xml"` + GeneratedSQL string `xml:"generated_sql"` + } `xml:"query_master"` + + QueryInstances []struct { + QueryInstanceID string `xml:"query_instance_id"` + QueryMasterID string `xml:"query_master_id"` + UserID string `xml:"user_id"` + GroupID string `xml:"group_id"` + BatchMode string `xml:"batch_mode"` + StartDate string `xml:"start_date"` + EndDate string `xml:"end_date"` + QueryStatusType struct { + StatusTypeID string `xml:"status_type_id"` + Name string `xml:"name"` + Description string `xml:"description"` + } `xml:"query_status_type"` + } `xml:"query_instance"` + + QueryResultInstances []QueryResultInstance `xml:"query_result_instance"` + } `xml:"response"` +} + +func (mb CrcPsmRespMessageBody) checkStatus() error { + var errorMessages []string + for _, status := range mb.Response.Status { + if status.Type == "ERROR" || status.Type == "FATAL_ERROR" { + errorMessages = append(errorMessages, status.Text) + } + } + + if len(errorMessages) != 0 { + return errors.New(strings.Join(errorMessages, "; ")) + } + return nil +} + +// QueryResultInstance is an i2b2 XML query result instance +type QueryResultInstance struct { + ResultInstanceID string `xml:"result_instance_id"` + QueryInstanceID string `xml:"query_instance_id"` + QueryResultType struct { + ResultTypeID string `xml:"result_type_id"` + Name string `xml:"name"` + Description string `xml:"description"` + } `xml:"query_result_type"` + SetSize string `xml:"set_size"` + StartDate string `xml:"start_date"` + EndDate string `xml:"end_date"` + QueryStatusType struct { + StatusTypeID string `xml:"status_type_id"` + Name string `xml:"name"` + Description string `xml:"description"` + } `xml:"query_status_type"` +} + +func (qri QueryResultInstance) checkStatus() error { + if qri.QueryStatusType.StatusTypeID != "3" { + return errors.New("i2b2 result instance does not have finished status") + } + return nil +} diff --git a/pkg/i2b2/models/ont.go b/pkg/i2b2/models/ont.go new file mode 100644 index 0000000..12863d5 --- /dev/null +++ b/pkg/i2b2/models/ont.go @@ -0,0 +1,208 @@ +package models + +import ( + "encoding/xml" +) + +// // NewOntReqGetTermInfoMessageBody returns a new request object for i2b2 get term info (information about node) +// func NewOntReqGetTermInfoMessageBody(path string) Request { +// body := OntReqGetTermInfoMessageBody{} +// body.GetTermInfo.Hiddens = "false" +// body.GetTermInfo.Blob = "true" +// body.GetTermInfo.Synonyms = "false" +// body.GetTermInfo.Max = utilserver.I2b2OntMaxElements +// body.GetTermInfo.Type = "core" +// body.GetTermInfo.Self = path +// +// return NewRequestWithBody(body) +// } +// +// // NewOntReqGetModifierInfoMessageBody returns a new request object for i2b2 get modifier info (information about node). +// // A modifier is identified by its own path (field self in XML API) and its applied path. +// func NewOntReqGetModifierInfoMessageBody(path string, appliedPath string) Request { +// body := OntReqGetModifierInfoMessageBody{} +// +// body.GetModifierInfo.Hiddens = "false" +// body.GetModifierInfo.Blob = "true" +// body.GetModifierInfo.Synonyms = "false" +// body.GetModifierInfo.Type = "core" +// body.GetModifierInfo.Self = path +// body.GetModifierInfo.AppliedPath = appliedPath +// +// return NewRequestWithBody(body) +// } + +// NewOntReqGetCategoriesMessageBody returns a new request object for i2b2 categories (ontology root nodes) +func NewOntReqGetCategoriesMessageBody(ci ConnectionInfo) Request { + body := OntReqGetCategoriesMessageBody{} + + body.GetCategories.Hiddens = "false" + body.GetCategories.Blob = "true" + body.GetCategories.Synonyms = "false" + body.GetCategories.Type = "core" + + return NewRequestWithBody(ci, body) +} + +// NewOntReqGetChildrenMessageBody returns a new request object for i2b2 children of a node +func NewOntReqGetChildrenMessageBody(ci ConnectionInfo, ontMaxElements, parent string) Request { + body := OntReqGetChildrenMessageBody{} + + body.GetChildren.Hiddens = "false" + body.GetChildren.Blob = "true" + body.GetChildren.Synonyms = "false" + body.GetChildren.Max = ontMaxElements + body.GetChildren.Type = "core" + + body.GetChildren.Parent = parent + + return NewRequestWithBody(ci, body) +} + +// NewOntReqGetModifiersMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path +func NewOntReqGetModifiersMessageBody(ci ConnectionInfo, self string) Request { + body := OntReqGetModifiersMessageBody{} + + body.GetModifiers.Blob = "true" + body.GetModifiers.Hiddens = "false" + body.GetModifiers.Synonyms = "false" + + body.GetModifiers.Self = self + + return NewRequestWithBody(ci, body) +} + +// NewOntReqGetModifierChildrenMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path +func NewOntReqGetModifierChildrenMessageBody(ci ConnectionInfo, ontMaxElements, parent, appliedPath, appliedConcept string) Request { + body := OntReqGetModifierChildrenMessageBody{} + + body.GetModifierChildren.Blob = "true" + body.GetModifierChildren.Type = "limited" + body.GetModifierChildren.Max = ontMaxElements + body.GetModifierChildren.Synonyms = "false" + body.GetModifierChildren.Hiddens = "false" + + body.GetModifierChildren.Parent = parent + body.GetModifierChildren.AppliedPath = appliedPath + body.GetModifierChildren.AppliedConcept = appliedConcept + + return NewRequestWithBody(ci, body) +} + +// --- request + +type baseMessageBody struct { + Hiddens string `xml:"hiddens,attr,omitempty"` + Synonyms string `xml:"synonyms,attr,omitempty"` + Type string `xml:"type,attr,omitempty"` + Blob string `xml:"blob,attr,omitempty"` + Max string `xml:"max,attr,omitempty"` +} + +// // OntReqGetTermInfoMessageBody is an i2b2 XML message body for ontology term info request +// type OntReqGetTermInfoMessageBody struct { +// XMLName xml.Name `xml:"message_body"` +// GetTermInfo struct { +// Max string `xml:"max,attr"` +// Hiddens string `xml:"hiddens,attr"` +// Synonyms string `xml:"synonyms,attr"` +// Type string `xml:"type,attr"` +// Blob string `xml:"blob,attr"` +// Self string `xml:"self"` +// } `xml:"ontns:get_term_info"` +// } +// +// // OntReqGetModifierInfoMessageBody is an i2b2 XML message body for ontology modifier info request +// type OntReqGetModifierInfoMessageBody struct { +// XMLName xml.Name `xml:"message_body"` +// GetModifierInfo struct { +// Hiddens string `xml:"hiddens,attr"` +// Synonyms string `xml:"synonyms,attr"` +// Type string `xml:"type,attr"` +// Blob string `xml:"blob,attr"` +// Self string `xml:"self"` +// AppliedPath string `xml:"applied_path"` +// } `xml:"ontns:get_modifier_info"` +// } + +// OntReqGetCategoriesMessageBody is an i2b2 XML message body for ontology categories request +type OntReqGetCategoriesMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetCategories struct { + baseMessageBody + } `xml:"ontns:get_categories"` +} + +// OntReqGetChildrenMessageBody is an i2b2 XML message for ontology children request +type OntReqGetChildrenMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetChildren struct { + baseMessageBody + Parent string `xml:"parent"` + } `xml:"ontns:get_children"` +} + +// OntReqGetModifiersMessageBody is an i2b2 XML message for ontology modifiers request +type OntReqGetModifiersMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetModifiers struct { + baseMessageBody + Self string `xml:"self"` + } `xml:"ontns:get_modifiers"` +} + +// OntReqGetModifierChildrenMessageBody is an i2b2 XML message for ontology modifier children request +type OntReqGetModifierChildrenMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetModifierChildren struct { + baseMessageBody + Parent string `xml:"parent"` + AppliedPath string `xml:"applied_path"` + AppliedConcept string `xml:"applied_concept"` + } `xml:"ontns:get_modifier_children"` +} + +// --- response + +// OntRespConceptsMessageBody is the message_body of the i2b2 get_children response message +type OntRespConceptsMessageBody struct { + XMLName xml.Name `xml:"message_body"` + Concepts []Concept `xml:"concepts>concept"` +} + +// Concept is an i2b2 XML concept +type Concept struct { + Level string `xml:"level"` + Key string `xml:"key"` + Name string `xml:"name"` + SynonymCd string `xml:"synonym_cd"` + Visualattributes string `xml:"visualattributes"` + Totalnum string `xml:"totalnum"` + Basecode string `xml:"basecode"` + Metadataxml *Metadataxml `xml:"metadataxml"` + Facttablecolumn string `xml:"facttablecolumn"` + Tablename string `xml:"tablename"` + Columnname string `xml:"columnname"` + Columndatatype string `xml:"columndatatype"` + Operator string `xml:"operator"` + Dimcode string `xml:"dimcode"` + Comment string `xml:"comment"` + Tooltip string `xml:"tooltip"` + UpdateDate string `xml:"update_date"` + DownloadDate string `xml:"download_date"` + ImportDate string `xml:"import_date"` + SourcesystemCd string `xml:"sourcesystem_cd"` + ValuetypeCd string `xml:"valuetype_cd"` +} + +// Modifier is an i2b2 XML modifier. +type Modifier struct { + Concept + AppliedPath string `xml:"applied_path"` +} + +// OntRespModifiersMessageBody is the message_body of the i2b2 get_modifiers response message. +type OntRespModifiersMessageBody struct { + XMLName xml.Name `xml:"message_body"` + Modifiers []Modifier `xml:"modifiers>modifier"` +} diff --git a/pkg/i2b2/models/util.go b/pkg/i2b2/models/util.go new file mode 100644 index 0000000..a106bf4 --- /dev/null +++ b/pkg/i2b2/models/util.go @@ -0,0 +1,11 @@ +package models + +import "strings" + +func ConvertPathToI2b2Format(path string) string { + return `\` + strings.Replace(path, "/", `\`, -1) +} + +func ConvertPathFromI2b2Format(path string) string { + return strings.Replace(strings.Replace(path, `\`, "/", -1), "//", "/", 1) +} From d0fec19e5c582937245988f5d34276d18addb7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:17:40 +0100 Subject: [PATCH 14/81] add i2b2 XML client definition --- pkg/i2b2/client.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pkg/i2b2/client.go diff --git a/pkg/i2b2/client.go b/pkg/i2b2/client.go new file mode 100644 index 0000000..66d8bf1 --- /dev/null +++ b/pkg/i2b2/client.go @@ -0,0 +1,68 @@ +package i2b2 + +import ( + "bytes" + "encoding/xml" + "fmt" + "io/ioutil" + "net/http" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2/models" + "github.com/sirupsen/logrus" +) + +// todo: define API of what's needed from geco +// todo: then define the query functions that take as arguments the API models +// todo: then adapt the existing XML request to be made +// todo: add tests and integration with CI + +// Client is an i2b2 client for its XML API +type Client struct { + + // logger is the logger from GeCo + logger *logrus.Logger + + // ci contains the connection information to the i2b2 instance + ci models.ConnectionInfo + + // ontMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 + ontMaxElements string +} + +// xmlRequest makes an HTTP POST request to i2b2 +func (c Client) xmlRequest(endpoint string, xmlRequest interface{}, xmlResponse *models.Response) error { + reqUrl := c.ci.HiveURL + endpoint + c.logger.Infof("i2b2 XML request to %v", reqUrl) + + // marshal request + marshaledRequest, err := xml.MarshalIndent(xmlRequest, " ", " ") + if err != nil { + return fmt.Errorf("marshalling i2b2 request marshalling: %v", err) + } + marshaledRequest = append([]byte(xml.Header), marshaledRequest...) + c.logger.Debugf("i2b2 request:\n%v", string(marshaledRequest)) + + // execute HTTP request + httpResponse, err := http.Post(reqUrl, "application/xml", bytes.NewBuffer(marshaledRequest)) + if err != nil { + return fmt.Errorf("making HTTP POST of XML request: %v", err) + } + defer httpResponse.Body.Close() + + // unmarshal response + httpBody, err := ioutil.ReadAll(httpResponse.Body) + if err != nil { + return fmt.Errorf("reading XML response: %v", err) + } + c.logger.Debugf("i2b2 response:\n%v", string(httpBody)) + + if err := xml.Unmarshal(httpBody, xmlResponse); err != nil { + return fmt.Errorf("unmarshalling XML response: %v", err) + } + + // check i2b2 request status + if err := xmlResponse.CheckStatus(); err != nil { + return fmt.Errorf("found error in i2b2 response: %v", err) + } + return nil +} From 1648c52d7eaff0e9cc9367f12c4936df2ffd5916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:18:07 +0100 Subject: [PATCH 15/81] add i2b2 data source plugin definition --- pkg/i2b2datasource/i2b2-data-source.go | 34 +++++++++++++++++++++ pkg/i2b2datasource/i2b2-data-source_test.go | 33 ++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 pkg/i2b2datasource/i2b2-data-source.go create mode 100644 pkg/i2b2datasource/i2b2-data-source_test.go diff --git a/pkg/i2b2datasource/i2b2-data-source.go b/pkg/i2b2datasource/i2b2-data-source.go new file mode 100644 index 0000000..3b34894 --- /dev/null +++ b/pkg/i2b2datasource/i2b2-data-source.go @@ -0,0 +1,34 @@ +package i2b2datasource + +import ( + "fmt" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2" + "github.com/ldsec/geco/pkg/datamanager" + "github.com/sirupsen/logrus" +) + +type I2b2DataSource struct { + + // dm is the GeCo data manager + dm *datamanager.DataManager + + // logger is the logger from GeCo + logger *logrus.Logger + + // i2b2C + i2b2Client i2b2.Client + +} + +func (ds I2b2DataSource) Init(dm *datamanager.DataManager, config map[string]string) error { + ds.dm = dm + + fmt.Println("called init") + return nil +} + +func (ds I2b2DataSource) Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) { + fmt.Println("called query") + return nil, nil +} diff --git a/pkg/i2b2datasource/i2b2-data-source_test.go b/pkg/i2b2datasource/i2b2-data-source_test.go new file mode 100644 index 0000000..e0b2af4 --- /dev/null +++ b/pkg/i2b2datasource/i2b2-data-source_test.go @@ -0,0 +1,33 @@ +package i2b2datasource + +import ( + "testing" + + "github.com/ldsec/geco/pkg/common/configuration" + "github.com/ldsec/geco/pkg/datamanager" + "github.com/stretchr/testify/require" +) + +func TestDataManager(t *testing.T) { + + ds := I2b2DataSource{} + + dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) + require.NoError(t, err) + + err = ds.Init(dm, nil) + require.NoError(t, err) + + doId, err := dm.AddDataObject(datamanager.NewFloatVector([]float64{0, 1, 2, 3, 4}), false) + require.NoError(t, err) + t.Logf("test data object ID is %v", doId) + + do, err := dm.GetDataObject(doId) + require.NoError(t, err) + require.Equal(t, doId, do.ID) + require.Equal(t, datamanager.FloatVector, do.Type) + + data, err := do.FloatVector() + require.NoError(t, err) + require.InDeltaSlice(t, []float64{0, 1, 2, 3, 4}, data, 0.0001) +} From 89d1bdffb7e9bb8fd7c4945b5e7272e07379c8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:18:33 +0100 Subject: [PATCH 16/81] add go.mod def --- go.mod | 79 ++++++++ go.sum | 620 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 699 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b006cc8 --- /dev/null +++ b/go.mod @@ -0,0 +1,79 @@ +module github.com/ldsec/geco-i2b2-data-source + +go 1.17 + +replace github.com/ldsec/geco => ./third_party/geco + +require ( + github.com/ldsec/geco v0.0.1 + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/Nerzal/gocloak/v9 v9.0.3 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/aws/aws-sdk-go v1.34.28 // indirect + github.com/creasty/defaults v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect + github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect + github.com/go-openapi/analysis v0.20.1 // indirect + github.com/go-openapi/errors v0.20.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/loads v0.20.3 // indirect + github.com/go-openapi/runtime v0.20.0 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.20.3 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-openapi/validate v0.20.3 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator v9.31.0+incompatible // indirect + github.com/go-resty/resty/v2 v2.3.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/goccy/go-json v0.7.10 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/ldsec/lattigo/v2 v2.0.0 // indirect + github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect + github.com/lestrrat-go/blackmagic v1.0.0 // indirect + github.com/lestrrat-go/httpcc v1.0.0 // indirect + github.com/lestrrat-go/iter v1.0.1 // indirect + github.com/lestrrat-go/jwx v1.2.9 // indirect + github.com/lestrrat-go/option v1.0.0 // indirect + github.com/lib/pq v1.9.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/montanaflynn/stats v0.6.3 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/philippgille/gokv v0.6.0 // indirect + github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 // indirect + github.com/philippgille/gokv/file v0.6.0 // indirect + github.com/philippgille/gokv/s3 v0.6.0 // indirect + github.com/philippgille/gokv/syncmap v0.6.0 // indirect + github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/segmentio/ksuid v1.0.3 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + go.dedis.ch/fixbuf v1.0.3 // indirect + go.dedis.ch/kyber/v3 v3.0.13 // indirect + go.dedis.ch/onet/v3 v3.2.3 // indirect + go.mongodb.org/mongo-driver v1.7.3 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect + golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gonum.org/v1/gonum v0.7.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..acaae5a --- /dev/null +++ b/go.sum @@ -0,0 +1,620 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Nerzal/gocloak/v9 v9.0.3 h1:45Jiw+H2TIrSsMboO4t6bUozMAzxeQDcwIWvnJ/KJsU= +github.com/Nerzal/gocloak/v9 v9.0.3/go.mod h1:AmXlpCliq31R1r/18ei7/ML5mfJDmJ1pTBoFU79DDZg= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/carbocation/handlers v0.0.0-20140528190747-c939c6d9ef31/go.mod h1:iGISoFvZYz358DFlmHvYFlh4CgRdzPLXB2NJE48x6lY= +github.com/carbocation/interpose v0.0.0-20161206215253-723534742ba3/go.mod h1:4PGcghc3ZjA/uozANO8lCHo/gnHyMsm8iFYppSkVE/M= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM= +github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 h1:d/cVoZOrJPJHKH1NdeUjyVAWKp4OpOT+Q+6T1sH7jeU= +github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d h1:1iy2qD6JEhHKKhUOA9IWs7mjco7lnw2qx8FsRI2wirE= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= +github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dre1080/recovr v1.0.3/go.mod h1:QV2VG2MZQYPszdtZ9bsZTpTdWTP81Mer3UHQXqo5bfY= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= +github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= +github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.20.1 h1:zdVbw8yoD4SWZeq+cWdGgquaB0W4VrsJvDJHJND/Ktc= +github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8= +github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= +github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= +github.com/go-openapi/loads v0.20.3 h1:VnuSSPx0bbSmSLUwltC6ss45tWyWzfvIeAeCk73B6N4= +github.com/go-openapi/loads v0.20.3/go.mod h1:r3u+N8rngPey6DHjYj9G4Wf61heNZjTQX2UjdIvUbn0= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= +github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.20.0 h1:DEV4oYH28MqakaabtbxH0cjvlzFegi/15kfUVCfiZW0= +github.com/go-openapi/runtime v0.20.0/go.mod h1:2WnLRxMiOUWNN0UZskSkxW0+WXdfB1KmqRKCFH+ZWYk= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= +github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= +github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= +github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/strfmt v0.20.3 h1:YVG4ZgPZ00km/lRHrIf7c6cKL5/4FAUtG2T9RxWAgDY= +github.com/go-openapi/strfmt v0.20.3/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= +github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= +github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= +github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= +github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= +github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= +github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= +github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So= +github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec= +github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c/go.mod h1:cHMBumiwaaRxRQ6NT8sU3zQSkXbYaPjbBcXa8UgTzAE= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/honeycombio/beeline-go v0.4.11/go.mod h1:xZNfHCpYQFjVS4Xuc8eMKhBclLnW/PvBz5L1Dwcp81Y= +github.com/honeycombio/libhoney-go v1.12.3/go.mod h1:xzRFVBkKkFnse+rTORNz5bTRhyVpKbZAQCyG3hxd2jY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/interpose/middleware v0.0.0-20150216143757-05ed56ed52fa/go.mod h1:eMb40EJpwUTKSRRKJ3sol3zWoy49dJXNxx7bdciFeYo= +github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/justinas/nosurf v1.1.1/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ldsec/lattigo/v2 v2.0.0 h1:P3rzbXFrc45swwl0u/zVCfcSDuIjTpIGueEDVQQzgEY= +github.com/ldsec/lattigo/v2 v2.0.0/go.mod h1:Iu8ol3XIWyF54Ka5bBBWNnNR9kYtymmuYo1Y6Hxzr+U= +github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0 h1:VRlFRZ8WcHm3m5CVAalz6g/pW+tX832/gf8vpKQFLBM= +github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0/go.mod h1:pMeb26+SHl1pJJvMVSNz7jW/lKaTYRtB6e5qIYQ9h6s= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= +github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4= +github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= +github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc= +github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= +github.com/lestrrat-go/iter v1.0.1 h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A= +github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= +github.com/lestrrat-go/jwx v1.2.9 h1:kS8kLI4oaBYJJ6u6rpbPI0tDYVCqo0P5u8vv1zoQ49U= +github.com/lestrrat-go/jwx v1.2.9/go.mod h1:25DcLbNWArPA/Ew5CcBmewl32cJKxOk5cbepBsIJFzw= +github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/meatballhat/negroni-logrus v1.1.0/go.mod h1:1yuzU2YqJx1Fh4UJ2nAt2rBa0rZoLxfpXQL/BXpiU0g= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk= +github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/philippgille/gokv v0.0.0-20191001201555-5ac9a20de634/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= +github.com/philippgille/gokv v0.5.1-0.20191011213304-eb77f15b9c61/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= +github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK2c= +github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= +github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI= +github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= +github.com/philippgille/gokv/file v0.6.0 h1:ySYotRmkwaJLDkNSdT7Q0iDQzKHhSdq+ornlBXWgKzI= +github.com/philippgille/gokv/file v0.6.0/go.mod h1:L5ulK3F64mxW+8OvYFGE5bowupGO73JdQBh4qE2bgEw= +github.com/philippgille/gokv/s3 v0.6.0 h1:AdQv7H7P73TdHW1h0i6fFgK7uKTwYhYx66vmsnvRoIo= +github.com/philippgille/gokv/s3 v0.6.0/go.mod h1:VX24mhIyCpdir7WrhUo1yJGuw6aSbNIuDgjUJGVhqKA= +github.com/philippgille/gokv/syncmap v0.6.0 h1:2eWC2J6mTyUsl687WuGoYPIiyqFiTBZU7hSKPlr0mK4= +github.com/philippgille/gokv/syncmap v0.6.0/go.mod h1:ZekkiO1XY9XbjRv9iunxA6+POW9Tw/QHsLO9xAHEaxo= +github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= +github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= +github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o= +github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= +github.com/phyber/negroni-gzip v1.0.0/go.mod h1:poOYjiFVKpeib8SnUpOgfQGStKNGLKsM8l09lOTNeyw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= +github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.20.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= +go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= +go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= +go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= +go.dedis.ch/kyber/v3 v3.0.12/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= +go.dedis.ch/kyber/v3 v3.0.13 h1:s5Lm8p2/CsTMueQHCN24gPpZ4couBBeKU7r2Yl6r32o= +go.dedis.ch/kyber/v3 v3.0.13/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= +go.dedis.ch/onet/v3 v3.2.3 h1:Ps05qwaT/In9WgoJxJimKW7FBpkGAxj+AkDIhe+HdAs= +go.dedis.ch/onet/v3 v3.2.3/go.mod h1:rKEuXJJiH44kj1VwlgmjT9PnqVItC5jos1W4/t1TMXQ= +go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= +go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= +go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/satori/go.uuid.v1 v1.2.0/go.mod h1:kjjdhYBBaa5W5DYP+OcVG3fRM6VWu14hqDYST4Zvw+E= +gopkg.in/tylerb/graceful.v1 v1.2.15/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From b4c07894c0a06664bf37cc1a52bed338c432f954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:18:50 +0100 Subject: [PATCH 17/81] add gitignore in build --- build/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 build/.gitignore diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..140f8cf --- /dev/null +++ b/build/.gitignore @@ -0,0 +1 @@ +*.so From 673c6d4c063262e0e74bc03e80c823bd1237106c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:19:08 +0100 Subject: [PATCH 18/81] add test for plugin --- internal/plugin_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 internal/plugin_test.go diff --git a/internal/plugin_test.go b/internal/plugin_test.go new file mode 100644 index 0000000..7beb5fb --- /dev/null +++ b/internal/plugin_test.go @@ -0,0 +1,31 @@ +package internal + +import ( + "plugin" + "testing" + + "github.com/ldsec/geco-i2b2-data-source/pkg" + "github.com/ldsec/geco/pkg/common/configuration" + "github.com/ldsec/geco/pkg/datamanager" + "github.com/stretchr/testify/require" +) + +func TestPlugin(t *testing.T) { + p, err := plugin.Open("../build/geco-i2b2-data-source.so") + require.NoError(t, err) + + dsSymbol, err := p.Lookup("DataSourcePlugin") + require.NoError(t, err) + + ds, ok := dsSymbol.(*pkg.DataSource) + require.True(t, ok) + + dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) + require.NoError(t, err) + + err = (*ds).Init(dm, nil) + require.NoError(t, err) + + _, err = (*ds).Query("", "", nil, nil) + require.NoError(t, err) +} From 8a162672b2f9d29a3d43e5b6e93d207ed7d87289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:20:07 +0100 Subject: [PATCH 19/81] add data source plugin for geco --- cmd/geco-i2b2-data-source/data-source-plugin.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 cmd/geco-i2b2-data-source/data-source-plugin.go diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go new file mode 100644 index 0000000..8a2e0c8 --- /dev/null +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/ldsec/geco-i2b2-data-source/pkg" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource" +) + +// DataSourcePlugin exports an instance of the GeCo i2b2 data source for the plugin. +var DataSourcePlugin pkg.DataSource = i2b2datasource.I2b2DataSource{} From 7eb100d525839a465397f47c6678de5cd1c7aa89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Sun, 5 Dec 2021 15:20:28 +0100 Subject: [PATCH 20/81] update data source interface implementation --- pkg/i2b2datasource/i2b2-data-source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/i2b2datasource/i2b2-data-source.go b/pkg/i2b2datasource/i2b2-data-source.go index 3b34894..55b5e2e 100644 --- a/pkg/i2b2datasource/i2b2-data-source.go +++ b/pkg/i2b2datasource/i2b2-data-source.go @@ -21,7 +21,7 @@ type I2b2DataSource struct { } -func (ds I2b2DataSource) Init(dm *datamanager.DataManager, config map[string]string) error { +func (ds I2b2DataSource) Init(dm *datamanager.DataManager, logger *logrus.Logger, config map[string]string) error { ds.dm = dm fmt.Println("called init") From 8caa1d1e3696766a0ac5b76abb1ff43d46c490f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 12:30:28 +0100 Subject: [PATCH 21/81] implementation of searchConcept, searchModifier and corresponding i2b2 XML API --- build/package/i2b2/sql/90-test-data.sh | 197 +++++++++++++++++ go.mod | 2 +- go.sum | 2 + pkg/geco-data-source.go | 2 +- pkg/i2b2/models/util.go | 11 - pkg/{i2b2 => i2b2api}/client.go | 27 ++- pkg/{i2b2 => i2b2api}/models/api_tmp.go | 74 ------- pkg/{i2b2 => i2b2api}/models/common.go | 49 ++--- pkg/{i2b2 => i2b2api}/models/crc_pdo.go | 62 +++--- pkg/{i2b2 => i2b2api}/models/crc_psm.go | 2 +- pkg/{i2b2 => i2b2api}/models/ont.go | 151 +++++++------ pkg/i2b2api/models/util.go | 38 ++++ pkg/i2b2api/ont_client.go | 73 +++++++ pkg/i2b2datasource/datasource.go | 133 ++++++++++++ pkg/i2b2datasource/datasource_test.go | 49 +++++ pkg/i2b2datasource/i2b2-data-source.go | 34 --- pkg/i2b2datasource/i2b2-data-source_test.go | 33 --- pkg/i2b2datasource/models/common.go | 25 +++ pkg/i2b2datasource/models/search.go | 62 ++++++ pkg/i2b2datasource/search.go | 179 +++++++++++++++ pkg/i2b2datasource/search_test.go | 229 ++++++++++++++++++++ 21 files changed, 1134 insertions(+), 300 deletions(-) create mode 100644 build/package/i2b2/sql/90-test-data.sh delete mode 100644 pkg/i2b2/models/util.go rename pkg/{i2b2 => i2b2api}/client.go (67%) rename pkg/{i2b2 => i2b2api}/models/api_tmp.go (56%) rename pkg/{i2b2 => i2b2api}/models/common.go (85%) rename pkg/{i2b2 => i2b2api}/models/crc_pdo.go (64%) rename pkg/{i2b2 => i2b2api}/models/crc_psm.go (99%) rename pkg/{i2b2 => i2b2api}/models/ont.go (59%) create mode 100644 pkg/i2b2api/models/util.go create mode 100644 pkg/i2b2api/ont_client.go create mode 100644 pkg/i2b2datasource/datasource.go create mode 100644 pkg/i2b2datasource/datasource_test.go delete mode 100644 pkg/i2b2datasource/i2b2-data-source.go delete mode 100644 pkg/i2b2datasource/i2b2-data-source_test.go create mode 100644 pkg/i2b2datasource/models/common.go create mode 100644 pkg/i2b2datasource/models/search.go create mode 100644 pkg/i2b2datasource/search.go create mode 100644 pkg/i2b2datasource/search_test.go diff --git a/build/package/i2b2/sql/90-test-data.sh b/build/package/i2b2/sql/90-test-data.sh new file mode 100644 index 0000000..fae27f7 --- /dev/null +++ b/build/package/i2b2/sql/90-test-data.sh @@ -0,0 +1,197 @@ +#!/bin/bash +set -Eeuo pipefail +# set up data in the database for end-to-end tests + +### description of the test data +# 4 patients: 1 (real), 2 (real), 3 (real), 4 (dummy) +# 3 concepts: 1, 2, 3 +# observation_fact: p1: c1; p2: c1, c2; p3: c2, c3; p4: c1, c2, c3 +# the same data is replicated on all nodes + +psql $PSQL_PARAMS -d "$I2B2_DB_NAME" <<-EOSQL + + -- i2b2metadata.schemes + insert into i2b2metadata.schemes(c_key, c_name, c_description) values('TEST:', 'Test', 'Test scheme.'); + + -- i2b2metadata.table_access + insert into i2b2metadata.table_access (c_table_cd, c_table_name, c_protected_access, c_hlevel, c_fullname, c_name, + c_synonym_cd, c_visualattributes, c_facttablecolumn, c_dimtablename, + c_columnname, c_columndatatype, c_operator, c_dimcode, c_tooltip) VALUES + ('TEST', 'TEST', 'N', '0', '\test\', 'Test Ontology', + 'N', 'CA', 'concept_cd', 'concept_dimension', 'concept_path', 'T', 'LIKE', '\test\', 'Test'); + + -- i2b2metadata.test + CREATE TABLE i2b2metadata.test( + c_hlevel numeric(22,0) not null, + c_fullname character varying(900) not null, + c_name character varying(2000) not null, + c_synonym_cd character(1) not null, + c_visualattributes character(3) not null, + c_totalnum numeric(22,0), + c_basecode character varying(450), + c_metadataxml text, + c_facttablecolumn character varying(50) not null, + c_tablename character varying(50) not null, + c_columnname character varying(50) not null, + c_columndatatype character varying(50) not null, + c_operator character varying(10) not null, + c_dimcode character varying(900) not null, + c_comment text, + c_tooltip character varying(900), + update_date date not null, + download_date date, + import_date date, + sourcesystem_cd character varying(50), + valuetype_cd character varying(50), + m_applied_path character varying(900) not null, + m_exclusion_cd character varying(900), + c_path character varying(700), + c_symbol character varying(50), + pcori_basecode character varying(50) + ); + ALTER TABLE ONLY i2b2metadata.test ADD CONSTRAINT fullname_pk_10 PRIMARY KEY (c_fullname); + ALTER TABLE ONLY i2b2metadata.test ADD CONSTRAINT basecode_un_10 UNIQUE (c_basecode); + ALTER TABLE i2b2metadata.test OWNER TO $I2B2_DB_USER; + + insert into i2b2metadata.test + (c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_totalnum, + c_facttablecolumn, c_tablename, c_columnname, c_columndatatype, c_operator, + c_dimcode, c_comment, c_tooltip, update_date, download_date, import_date, + valuetype_cd, m_applied_path, c_basecode, c_metadataxml) values + ( + '0', '\test\', 'Test', 'N', 'CA', '0', + 'concept_cd', 'concept_dimension', 'concept_path', + 'T', 'LIKE', '\test\', 'Test', '\test\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '@', '', NULL + ), ( + '1', '\test\1\', 'Concept 1', 'N', 'LA', '0', + 'concept_cd', 'concept_dimension', 'concept_path', + 'T', 'LIKE', '\test\1\', 'Concept 1', '\test\1\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '@', 'TEST:1', '<?xml version="1.0"?><ValueMetadata></ValueMetadata>' + ), ( + '1', '\test\2\', 'Concept 2', 'N', 'LA', '0', + 'concept_cd', 'concept_dimension', 'concept_path', + 'T', 'LIKE', '\test\2\', 'Concept 2', '\test\2\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '@', 'TEST:2', NULL + ), ( + '1', '\test\3\', 'Concept 3', 'N', 'LA', '0', + 'concept_cd', 'concept_dimension', 'concept_path', + 'T', 'LIKE', '\test\3\', 'Concept 3', '\test\3\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '@', 'TEST:3', NULL + ), ( + '0', '\modifiers\', 'Modifiers test', 'N', 'DA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\', 'Modifiers Test', '\modifiers\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '\test\%', 'TEST:4', NULL + ), ( + '1', '\modifiers\1\', 'Modifier 1', 'N', 'RA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\1\', 'Modifier 1', '\modifiers\1\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '\test\1\', 'TEST:5', '<?xml version="1.0"?><ValueMetadata></ValueMetadata>' + ), ( + '1', '\modifiers\2\', 'Modifier 2', 'N', 'RA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\2\', 'Modifier 2', '\modifiers\2\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '\test\2\', 'TEST:6', NULL + ), ( + '1', '\modifiers\3\', 'Modifier 3', 'N', 'RA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\3\', 'Modifier 3', '\modifiers\3\', + 'NOW()', 'NOW()', 'NOW()', 'TEST', '\test\3\', 'TEST:7', NULL + ), ( + '1', '\modifiers\2text\', 'Modifier 2 text', 'N', 'RA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\2text\', 'Modifier 2 text', '\modifiers\2text\', + 'NOW()', 'NOW()', 'NOW()', 'T', '\test\2\', 'TEST:8', NULL + ), ( + '1', '\modifiers\3text\', 'Modifier 3 text', 'N', 'RA', '0', + 'modifier_cd', 'modifier_dimension', 'modifier_path', + 'T', 'LIKE', '\modifiers\3text\', 'Modifier 3 text', '\modifiers\3text\', + 'NOW()', 'NOW()', 'NOW()', 'T', '\test\3\', 'TEST:9', NULL + ); + + -- i2b2demodata.concept_dimension + insert into i2b2demodata.concept_dimension + (concept_path, concept_cd, import_date, upload_id) values + ('\test\', '', 'NOW()', '1'), + ('\test\1\', 'TEST:1', 'NOW()', '1'), + ('\test\2\', 'TEST:2', 'NOW()', '1'), + ('\test\3\', 'TEST:3', 'NOW()', '1'); + + -- i2b2demodata.modifier_dimension + insert into i2b2demodata.modifier_dimension + (modifier_path, modifier_cd, import_date, upload_id) values + ('\modifiers\', 'TEST:4', 'NOW()', '1'), + ('\modifiers\1\', 'TEST:5', 'NOW()', '1'), + ('\modifiers\2\', 'TEST:6', 'NOW()', '1'), + ('\modifiers\3\', 'TEST:7', 'NOW()', '1'), + ('\modifiers\2text\', 'TEST:8', 'NOW()', '1'), + ('\modifiers\3text\', 'TEST:9', 'NOW()', '1'); + + -- i2b2demodata.provider_dimension + insert into i2b2demodata.provider_dimension + (provider_id, provider_path, name_char, import_date, upload_id) values + ('test', '\test\', 'test', 'NOW()', '1'); + + -- i2b2demodata.patient_dimension + insert into i2b2demodata.patient_dimension + (patient_num, import_date, upload_id) values + ('1', 'NOW()', '1'), + ('2', 'NOW()', '1'), + ('3', 'NOW()', '1'), + ('4', 'NOW()', '1'); + + -- i2b2demodata.patient_mapping + insert into i2b2demodata.patient_mapping + (patient_ide, patient_ide_source, patient_num, project_id, import_date, upload_id) values + ('test1', 'test', '1', 'Demo', 'NOW()', '1'), + ('test2', 'test', '2', 'Demo', 'NOW()', '1'), + ('test3', 'test', '3', 'Demo', 'NOW()', '1'), + ('test4', 'test', '4', 'Demo', 'NOW()', '1'); + + -- i2b2demodata.visit_dimension + insert into i2b2demodata.visit_dimension + (encounter_num, patient_num, import_date, upload_id) values + ('1', '1', 'NOW()', '1'), + ('2', '2', 'NOW()', '1'), + ('3', '3', 'NOW()', '1'), + ('4', '4', 'NOW()', '1'); + + -- i2b2demodata.encounter_mapping + insert into i2b2demodata.encounter_mapping + (encounter_ide, encounter_ide_source, project_id, encounter_num, patient_ide, patient_ide_source, import_date, upload_id) values + ('test1', 'test', 'Demo', '1', 'test1', 'test', 'NOW()', '1'), + ('test2', 'test', 'Demo', '2', 'test2', 'test', 'NOW()', '1'), + ('test3', 'test', 'Demo', '3', 'test3', 'test', 'NOW()', '1'), + ('test4', 'test', 'Demo', '4', 'test4', 'test', 'NOW()', '1'); + + -- i2b2demodata.observation_fact + insert into i2b2demodata.observation_fact + (encounter_num, patient_num, concept_cd, provider_id, start_date, modifier_cd, instance_num, import_date, upload_id, valtype_cd, tval_char, nval_num) values + ('1', '1', 'TEST:1', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '10'), + ('1', '1', 'TEST:1', 'test', 'NOW()', 'TEST:5', '1', 'NOW()', '1', 'N', 'E', '10'), + ('1', '1', 'TEST:2', 'test', 'NOW()', 'TEST:8', '1', 'NOW()', '1', 'T', 'bcde', NULL), + ('1', '1', 'TEST:3', 'test', 'NOW()', 'TEST:9', '1', 'NOW()', '1', 'T', 'ab', NULL), + + ('2', '2', 'TEST:1', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '20'), + ('2', '2', 'TEST:1', 'test', 'NOW()', 'TEST:4', '1', 'NOW()', '1', 'N', 'E', '20'), + ('2', '2', 'TEST:2', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '50'), + ('2', '2', 'TEST:2', 'test', 'NOW()', 'TEST:6', '1', 'NOW()', '1', 'N', 'E', '5'), + ('2', '2', 'TEST:2', 'test', 'NOW()', 'TEST:8', '1', 'NOW()', '1', 'T', 'abc', NULL), + ('2', '2', 'TEST:3', 'test', 'NOW()', 'TEST:9', '1', 'NOW()', '1', 'T', 'def', NULL), + + ('3', '3', 'TEST:1', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '30'), + ('3', '3', 'TEST:1', 'test', 'NOW()', 'TEST:4', '1', 'NOW()', '1', 'N', 'E', '15'), + ('3', '3', 'TEST:1', 'test', 'NOW()', 'TEST:5', '1', 'NOW()', '1', 'N', 'E', '15'), + ('3', '3', 'TEST:2', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '25'), + ('3', '3', 'TEST:2', 'test', 'NOW()', 'TEST:4', '1', 'NOW()', '1', 'N', 'E', '30'), + ('3', '3', 'TEST:2', 'test', 'NOW()', 'TEST:6', '1', 'NOW()', '1', 'N', 'E', '15'), + ('3', '3', 'TEST:2', 'test', 'NOW()', 'TEST:8', '1', 'NOW()', '1', 'T', 'de', NULL), + ('3', '3', 'TEST:3', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '77'), + ('3', '3', 'TEST:3', 'test', 'NOW()', 'TEST:4', '1', 'NOW()', '1', 'N', 'E', '66'), + ('3', '3', 'TEST:3', 'test', 'NOW()', 'TEST:7', '1', 'NOW()', '1', 'N', 'E', '88'), + ('3', '3', 'TEST:3', 'test', 'NOW()', 'TEST:9', '1', 'NOW()', '1', 'T', 'abcdef', NULL), + + ('4', '4', 'TEST:3', 'test', 'NOW()', '@', '1', 'NOW()', '1', 'N', 'E', '20'), + ('4', '4', 'TEST:3', 'test', 'NOW()', 'TEST:7', '1', 'NOW()', '1', 'N', 'E', '10'); +EOSQL diff --git a/go.mod b/go.mod index b006cc8..d175029 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/lestrrat-go/option v1.0.0 // indirect github.com/lib/pq v1.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/montanaflynn/stats v0.6.3 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect diff --git a/go.sum b/go.sum index acaae5a..bced2ad 100644 --- a/go.sum +++ b/go.sum @@ -329,6 +329,8 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk= diff --git a/pkg/geco-data-source.go b/pkg/geco-data-source.go index 8e2cd2c..3b44eff 100644 --- a/pkg/geco-data-source.go +++ b/pkg/geco-data-source.go @@ -12,7 +12,7 @@ import ( type DataSource interface { // Init the data source with the provided configuration. - Init(dm *datamanager.DataManager, logger *logrus.Logger, config map[string]string) error + Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) error // Query data source with a specific operation. // todo: parameters and results are json marshallable -> some other interface instead? check what swagger provides or not diff --git a/pkg/i2b2/models/util.go b/pkg/i2b2/models/util.go deleted file mode 100644 index a106bf4..0000000 --- a/pkg/i2b2/models/util.go +++ /dev/null @@ -1,11 +0,0 @@ -package models - -import "strings" - -func ConvertPathToI2b2Format(path string) string { - return `\` + strings.Replace(path, "/", `\`, -1) -} - -func ConvertPathFromI2b2Format(path string) string { - return strings.Replace(strings.Replace(path, `\`, "/", -1), "//", "/", 1) -} diff --git a/pkg/i2b2/client.go b/pkg/i2b2api/client.go similarity index 67% rename from pkg/i2b2/client.go rename to pkg/i2b2api/client.go index 66d8bf1..ff86bf1 100644 --- a/pkg/i2b2/client.go +++ b/pkg/i2b2api/client.go @@ -1,4 +1,4 @@ -package i2b2 +package i2b2api import ( "bytes" @@ -7,7 +7,7 @@ import ( "io/ioutil" "net/http" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" "github.com/sirupsen/logrus" ) @@ -19,20 +19,19 @@ import ( // Client is an i2b2 client for its XML API type Client struct { - // logger is the logger from GeCo - logger *logrus.Logger + // Logger is the Logger from GeCo + Logger logrus.FieldLogger - // ci contains the connection information to the i2b2 instance - ci models.ConnectionInfo - - // ontMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 - ontMaxElements string + // Ci contains the connection information to the i2b2 instance + Ci models.ConnectionInfo } // xmlRequest makes an HTTP POST request to i2b2 -func (c Client) xmlRequest(endpoint string, xmlRequest interface{}, xmlResponse *models.Response) error { - reqUrl := c.ci.HiveURL + endpoint - c.logger.Infof("i2b2 XML request to %v", reqUrl) +func (c Client) xmlRequest(endpoint string, xmlRequest *models.Request, xmlResponse *models.Response) error { + reqUrl := c.Ci.HiveURL + endpoint + c.Logger.Infof("i2b2 XML request to %v", reqUrl) + + xmlRequest.SetConnectionInfo(c.Ci) // marshal request marshaledRequest, err := xml.MarshalIndent(xmlRequest, " ", " ") @@ -40,7 +39,7 @@ func (c Client) xmlRequest(endpoint string, xmlRequest interface{}, xmlResponse return fmt.Errorf("marshalling i2b2 request marshalling: %v", err) } marshaledRequest = append([]byte(xml.Header), marshaledRequest...) - c.logger.Debugf("i2b2 request:\n%v", string(marshaledRequest)) + c.Logger.Debugf("i2b2 request:\n%v", string(marshaledRequest)) // execute HTTP request httpResponse, err := http.Post(reqUrl, "application/xml", bytes.NewBuffer(marshaledRequest)) @@ -54,7 +53,7 @@ func (c Client) xmlRequest(endpoint string, xmlRequest interface{}, xmlResponse if err != nil { return fmt.Errorf("reading XML response: %v", err) } - c.logger.Debugf("i2b2 response:\n%v", string(httpBody)) + c.Logger.Debugf("i2b2 response:\n%v", string(httpBody)) if err := xml.Unmarshal(httpBody, xmlResponse); err != nil { return fmt.Errorf("unmarshalling XML response: %v", err) diff --git a/pkg/i2b2/models/api_tmp.go b/pkg/i2b2api/models/api_tmp.go similarity index 56% rename from pkg/i2b2/models/api_tmp.go rename to pkg/i2b2api/models/api_tmp.go index 1bb678e..fbeee5e 100644 --- a/pkg/i2b2/models/api_tmp.go +++ b/pkg/i2b2api/models/api_tmp.go @@ -73,77 +73,3 @@ const ( // APITimingSameinstancenum captures enum value "sameinstancenum" APITimingSameinstancenum APITiming = "sameinstancenum" ) - -// Metadataxml todo: should be changed according to the API exposed -type Metadataxml struct { - - // value metadata - ValueMetadata *MetadataxmlValueMetadata `json:"ValueMetadata,omitempty"` -} - - -// MetadataxmlValueMetadata todo: should be changed according to the API exposed -type MetadataxmlValueMetadata struct { - - // children encrypt i ds - ChildrenEncryptIDs string `json:"ChildrenEncryptIDs,omitempty"` - - // creation date time - CreationDateTime string `json:"CreationDateTime,omitempty"` - - // data type - DataType string `json:"DataType,omitempty"` - - // encrypted type - EncryptedType string `json:"EncryptedType,omitempty"` - - // enum values - EnumValues string `json:"EnumValues,omitempty"` - - // flagstouse - Flagstouse string `json:"Flagstouse,omitempty"` - - // node encrypt ID - NodeEncryptID string `json:"NodeEncryptID,omitempty"` - - // oktousevalues - Oktousevalues string `json:"Oktousevalues,omitempty"` - - // test ID - TestID string `json:"TestID,omitempty"` - - // test name - TestName string `json:"TestName,omitempty"` - - // unit values - UnitValues *UnitValues `json:"UnitValues,omitempty"` - - // version - Version string `json:"Version,omitempty"` -} - -// UnitValues todo: should be changed according to the API exposed -type UnitValues struct { - - // converting units - ConvertingUnits []*UnitValuesConvertingUnitsItems0 `json:"ConvertingUnits"` - - // equal units - EqualUnits []string `json:"EqualUnits"` - - // excluding units - ExcludingUnits []string `json:"ExcludingUnits"` - - // normal units - NormalUnits string `json:"NormalUnits,omitempty"` -} - -// UnitValuesConvertingUnitsItems0 todo: should be changed according to the API exposed -type UnitValuesConvertingUnitsItems0 struct { - - // multiplying factor - MultiplyingFactor string `json:"MultiplyingFactor,omitempty"` - - // units - Units string `json:"Units,omitempty"` -} diff --git a/pkg/i2b2/models/common.go b/pkg/i2b2api/models/common.go similarity index 85% rename from pkg/i2b2/models/common.go rename to pkg/i2b2api/models/common.go index 7267b6f..9a2d85e 100644 --- a/pkg/i2b2/models/common.go +++ b/pkg/i2b2api/models/common.go @@ -7,30 +7,8 @@ import ( "time" ) -// ConnectionInfo contains the data needed for connection to i2b2. -type ConnectionInfo struct { - - // HiveURL is the URL of the i2b2 hive - HiveURL string - - // Domain is the i2b2 login domain - Domain string - - // Username is the i2b2 login username - Username string - - // Password is the i2b2 login password - Password string - - // Project is the i2b2 project ID - Project string - - // WaitTime is the wait time to send with i2b2 requests - WaitTime time.Duration -} - // NewRequest creates a new ready-to-use i2b2 request, with a nil message body. -func NewRequest(ci ConnectionInfo) Request { +func NewRequest() Request { now := time.Now() return Request{ XMLNSMSG: "http://www.i2b2.org/xsd/hive/msg/1.1/", @@ -49,9 +27,9 @@ func NewRequest(ci ConnectionInfo) Request { ReceivingApplicationApplicationVersion: "1.7", ReceivingFacilityFacilityName: "i2b2 hive", DatetimeOfMessage: now.Format(time.RFC3339), - SecurityDomain: ci.Domain, - SecurityUsername: ci.Username, - SecurityPassword: ci.Password, + SecurityDomain: "NOT_SET", + SecurityUsername: "NOT_SET", + SecurityPassword: "NOT_SET", MessageTypeMessageCode: "EQQ", MessageTypeEventType: "Q04", MessageTypeMessageStructure: "EQQ_Q04", @@ -63,21 +41,30 @@ func NewRequest(ci ConnectionInfo) Request { AcceptAcknowledgementType: "messageId", ApplicationAcknowledgementType: "", CountryCode: "CH", - ProjectID: ci.Project, + ProjectID: "NOT_SET", }, RequestHeader: RequestHeader{ - ResultWaittimeMs: strconv.FormatInt(ci.WaitTime.Milliseconds(), 10), + ResultWaittimeMs: "NOT_SET", }, } } // NewRequestWithBody creates a new ready-to-use i2b2 request, with a message body -func NewRequestWithBody(ci ConnectionInfo, body MessageBody) (req Request) { - req = NewRequest(ci) +func NewRequestWithBody(body MessageBody) (req Request) { + req = NewRequest() req.MessageBody = body return } +// SetConnectionInfo sets the i2b2 connection information in the request. +func (req *Request) SetConnectionInfo(ci ConnectionInfo) { + req.MessageHeader.SecurityDomain = ci.Domain + req.MessageHeader.SecurityUsername = ci.Username + req.MessageHeader.SecurityPassword = ci.Password + req.MessageHeader.ProjectID = ci.Project + req.RequestHeader.ResultWaittimeMs = strconv.FormatInt(ci.WaitTime.Milliseconds(), 10) +} + // Request is an i2b2 XML request type Request struct { XMLName xml.Name `xml:"msgns:request"` @@ -101,7 +88,7 @@ type Response struct { MessageBody MessageBody `xml:"message_body"` } -func (response *Response) CheckStatus() error { +func (response Response) CheckStatus() error { if response.ResponseHeader.ResultStatus.Status.Type != "DONE" { return errors.New(response.ResponseHeader.ResultStatus.Status.Text) } diff --git a/pkg/i2b2/models/crc_pdo.go b/pkg/i2b2api/models/crc_pdo.go similarity index 64% rename from pkg/i2b2/models/crc_pdo.go rename to pkg/i2b2api/models/crc_pdo.go index 762fe6e..001eb9c 100644 --- a/pkg/i2b2/models/crc_pdo.go +++ b/pkg/i2b2api/models/crc_pdo.go @@ -4,37 +4,37 @@ import ( "encoding/xml" ) -// NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request -func NewCrcPdoReqFromInputList(ci ConnectionInfo, patientSetID string) Request { - - // PDO header - pdoHeader := PdoHeader{ - PatientSetLimit: "0", - EstimatedTime: "0", - RequestType: "getPDO_fromInputList", - } - - // PDO request - pdoRequest := PdoRequestFromInputList{ - Type: "crcpdons:GetPDOFromInputList_requestType", - Xsi: "http://www.w3.org/2001/XMLSchema-instance", - } - - // set request for patient set ID - pdoRequest.InputList.PatientList.Max = "1000000" - pdoRequest.InputList.PatientList.Min = "0" - pdoRequest.InputList.PatientList.PatientSetCollID = patientSetID - pdoRequest.OutputOption.Name = "none" - pdoRequest.OutputOption.PatientSet.Blob = "false" - pdoRequest.OutputOption.PatientSet.TechData = "false" - pdoRequest.OutputOption.PatientSet.OnlyKeys = "false" - pdoRequest.OutputOption.PatientSet.Select = "using_input_list" - - return NewRequestWithBody(ci, CrcPdoReqFromInputListMessageBody{ - PdoHeader: pdoHeader, - PdoRequest: pdoRequest, - }) -} +// // NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request +// func NewCrcPdoReqFromInputList(ci ConnectionInfo, patientSetID string) Request { +// +// // PDO header +// pdoHeader := PdoHeader{ +// PatientSetLimit: "0", +// EstimatedTime: "0", +// RequestType: "getPDO_fromInputList", +// } +// +// // PDO request +// pdoRequest := PdoRequestFromInputList{ +// Type: "crcpdons:GetPDOFromInputList_requestType", +// Xsi: "http://www.w3.org/2001/XMLSchema-instance", +// } +// +// // set request for patient set ID +// pdoRequest.InputList.PatientList.Max = "1000000" +// pdoRequest.InputList.PatientList.Min = "0" +// pdoRequest.InputList.PatientList.PatientSetCollID = patientSetID +// pdoRequest.OutputOption.Name = "none" +// pdoRequest.OutputOption.PatientSet.Blob = "false" +// pdoRequest.OutputOption.PatientSet.TechData = "false" +// pdoRequest.OutputOption.PatientSet.OnlyKeys = "false" +// pdoRequest.OutputOption.PatientSet.Select = "using_input_list" +// +// return NewRequestWithBody(ci, CrcPdoReqFromInputListMessageBody{ +// PdoHeader: pdoHeader, +// PdoRequest: pdoRequest, +// }) +// } // --- request diff --git a/pkg/i2b2/models/crc_psm.go b/pkg/i2b2api/models/crc_psm.go similarity index 99% rename from pkg/i2b2/models/crc_psm.go rename to pkg/i2b2api/models/crc_psm.go index 0df25e4..5683221 100644 --- a/pkg/i2b2/models/crc_psm.go +++ b/pkg/i2b2api/models/crc_psm.go @@ -96,7 +96,7 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ }) } - return NewRequestWithBody(ci, CrcPsmReqFromQueryDefMessageBody{ + return NewRequestWithBody(CrcPsmReqFromQueryDefMessageBody{ PsmHeader: psmHeader, PsmRequest: psmRequest, }) diff --git a/pkg/i2b2/models/ont.go b/pkg/i2b2api/models/ont.go similarity index 59% rename from pkg/i2b2/models/ont.go rename to pkg/i2b2api/models/ont.go index 12863d5..f440e27 100644 --- a/pkg/i2b2/models/ont.go +++ b/pkg/i2b2api/models/ont.go @@ -4,36 +4,34 @@ import ( "encoding/xml" ) -// // NewOntReqGetTermInfoMessageBody returns a new request object for i2b2 get term info (information about node) -// func NewOntReqGetTermInfoMessageBody(path string) Request { -// body := OntReqGetTermInfoMessageBody{} -// body.GetTermInfo.Hiddens = "false" -// body.GetTermInfo.Blob = "true" -// body.GetTermInfo.Synonyms = "false" -// body.GetTermInfo.Max = utilserver.I2b2OntMaxElements -// body.GetTermInfo.Type = "core" -// body.GetTermInfo.Self = path -// -// return NewRequestWithBody(body) -// } -// -// // NewOntReqGetModifierInfoMessageBody returns a new request object for i2b2 get modifier info (information about node). -// // A modifier is identified by its own path (field self in XML API) and its applied path. -// func NewOntReqGetModifierInfoMessageBody(path string, appliedPath string) Request { -// body := OntReqGetModifierInfoMessageBody{} -// -// body.GetModifierInfo.Hiddens = "false" -// body.GetModifierInfo.Blob = "true" -// body.GetModifierInfo.Synonyms = "false" -// body.GetModifierInfo.Type = "core" -// body.GetModifierInfo.Self = path -// body.GetModifierInfo.AppliedPath = appliedPath -// -// return NewRequestWithBody(body) -// } - -// NewOntReqGetCategoriesMessageBody returns a new request object for i2b2 categories (ontology root nodes) -func NewOntReqGetCategoriesMessageBody(ci ConnectionInfo) Request { +func NewOntReqGetTermInfoMessageBody(ontMaxElements, path string) OntReqGetTermInfoMessageBody { + body := OntReqGetTermInfoMessageBody{} + + body.GetTermInfo.Hiddens = "false" + body.GetTermInfo.Blob = "true" + body.GetTermInfo.Synonyms = "false" + body.GetTermInfo.Max = ontMaxElements + body.GetTermInfo.Type = "core" + body.GetTermInfo.Self = path + + return body +} + +func NewOntReqGetModifierInfoMessageBody(path string, appliedPath string) OntReqGetModifierInfoMessageBody { + body := OntReqGetModifierInfoMessageBody{} + + body.GetModifierInfo.Hiddens = "false" + body.GetModifierInfo.Blob = "true" + body.GetModifierInfo.Synonyms = "false" + body.GetModifierInfo.Type = "core" + body.GetModifierInfo.Self = path + body.GetModifierInfo.AppliedPath = appliedPath + + return body +} + +// NewOntReqGetCategoriesMessageBody returns a new message body for a request object for i2b2 categories (ontology root nodes) +func NewOntReqGetCategoriesMessageBody() OntReqGetCategoriesMessageBody { body := OntReqGetCategoriesMessageBody{} body.GetCategories.Hiddens = "false" @@ -41,11 +39,11 @@ func NewOntReqGetCategoriesMessageBody(ci ConnectionInfo) Request { body.GetCategories.Synonyms = "false" body.GetCategories.Type = "core" - return NewRequestWithBody(ci, body) + return body } -// NewOntReqGetChildrenMessageBody returns a new request object for i2b2 children of a node -func NewOntReqGetChildrenMessageBody(ci ConnectionInfo, ontMaxElements, parent string) Request { +// NewOntReqGetChildrenMessageBody returns a new message body for a request object for i2b2 children of a node +func NewOntReqGetChildrenMessageBody(ontMaxElements, parent string) OntReqGetChildrenMessageBody { body := OntReqGetChildrenMessageBody{} body.GetChildren.Hiddens = "false" @@ -56,11 +54,11 @@ func NewOntReqGetChildrenMessageBody(ci ConnectionInfo, ontMaxElements, parent s body.GetChildren.Parent = parent - return NewRequestWithBody(ci, body) + return body } // NewOntReqGetModifiersMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path -func NewOntReqGetModifiersMessageBody(ci ConnectionInfo, self string) Request { +func NewOntReqGetModifiersMessageBody(self string) OntReqGetModifiersMessageBody { body := OntReqGetModifiersMessageBody{} body.GetModifiers.Blob = "true" @@ -69,11 +67,11 @@ func NewOntReqGetModifiersMessageBody(ci ConnectionInfo, self string) Request { body.GetModifiers.Self = self - return NewRequestWithBody(ci, body) + return body } -// NewOntReqGetModifierChildrenMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path -func NewOntReqGetModifierChildrenMessageBody(ci ConnectionInfo, ontMaxElements, parent, appliedPath, appliedConcept string) Request { +// NewOntReqGetModifierChildrenMessageBody returns a new message body for a request object to get the i2b2 modifiers that apply to the concept path +func NewOntReqGetModifierChildrenMessageBody(ontMaxElements, parent, appliedPath, appliedConcept string) OntReqGetModifierChildrenMessageBody { body := OntReqGetModifierChildrenMessageBody{} body.GetModifierChildren.Blob = "true" @@ -86,7 +84,7 @@ func NewOntReqGetModifierChildrenMessageBody(ci ConnectionInfo, ontMaxElements, body.GetModifierChildren.AppliedPath = appliedPath body.GetModifierChildren.AppliedConcept = appliedConcept - return NewRequestWithBody(ci, body) + return body } // --- request @@ -99,31 +97,22 @@ type baseMessageBody struct { Max string `xml:"max,attr,omitempty"` } -// // OntReqGetTermInfoMessageBody is an i2b2 XML message body for ontology term info request -// type OntReqGetTermInfoMessageBody struct { -// XMLName xml.Name `xml:"message_body"` -// GetTermInfo struct { -// Max string `xml:"max,attr"` -// Hiddens string `xml:"hiddens,attr"` -// Synonyms string `xml:"synonyms,attr"` -// Type string `xml:"type,attr"` -// Blob string `xml:"blob,attr"` -// Self string `xml:"self"` -// } `xml:"ontns:get_term_info"` -// } -// -// // OntReqGetModifierInfoMessageBody is an i2b2 XML message body for ontology modifier info request -// type OntReqGetModifierInfoMessageBody struct { -// XMLName xml.Name `xml:"message_body"` -// GetModifierInfo struct { -// Hiddens string `xml:"hiddens,attr"` -// Synonyms string `xml:"synonyms,attr"` -// Type string `xml:"type,attr"` -// Blob string `xml:"blob,attr"` -// Self string `xml:"self"` -// AppliedPath string `xml:"applied_path"` -// } `xml:"ontns:get_modifier_info"` -// } +type OntReqGetTermInfoMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetTermInfo struct { + baseMessageBody + Self string `xml:"self"` + } `xml:"ontns:get_term_info"` +} + +type OntReqGetModifierInfoMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetModifierInfo struct { + baseMessageBody + Self string `xml:"self"` + AppliedPath string `xml:"applied_path"` + } `xml:"ontns:get_modifier_info"` +} // OntReqGetCategoriesMessageBody is an i2b2 XML message body for ontology categories request type OntReqGetCategoriesMessageBody struct { @@ -170,6 +159,12 @@ type OntRespConceptsMessageBody struct { Concepts []Concept `xml:"concepts>concept"` } +// OntRespModifiersMessageBody is the message_body of the i2b2 get_modifiers response message. +type OntRespModifiersMessageBody struct { + XMLName xml.Name `xml:"message_body"` + Modifiers []Modifier `xml:"modifiers>modifier"` +} + // Concept is an i2b2 XML concept type Concept struct { Level string `xml:"level"` @@ -179,7 +174,7 @@ type Concept struct { Visualattributes string `xml:"visualattributes"` Totalnum string `xml:"totalnum"` Basecode string `xml:"basecode"` - Metadataxml *Metadataxml `xml:"metadataxml"` + Metadataxml *MetadataXML `xml:"metadataxml"` Facttablecolumn string `xml:"facttablecolumn"` Tablename string `xml:"tablename"` Columnname string `xml:"columnname"` @@ -201,8 +196,26 @@ type Modifier struct { AppliedPath string `xml:"applied_path"` } -// OntRespModifiersMessageBody is the message_body of the i2b2 get_modifiers response message. -type OntRespModifiersMessageBody struct { - XMLName xml.Name `xml:"message_body"` - Modifiers []Modifier `xml:"modifiers>modifier"` +type MetadataXML struct { + CreationDateTime string `xml:"ValueMetadata>CreationDateTime"` + DataType string `xml:"ValueMetadata>DataType"` + EnumValues string `xml:"ValueMetadata>EnumValues"` + Flagstouse string `xml:"ValueMetadata>Flagstouse"` + Oktousevalues string `xml:"ValueMetadata>Oktousevalues"` + TestID string `xml:"ValueMetadata>TestID"` + TestName string `xml:"ValueMetadata>TestName"` + UnitValues ValueMetadataUnitValues `xml:"ValueMetadata>UnitValues"` + Version string `xml:"ValueMetadata>Version"` +} + +type ValueMetadataUnitValues struct { + ConvertingUnits []UnitValuesConvertingUnits `xml:"ConvertingUnits"` + EqualUnits []string `xml:"EqualUnits"` + ExcludingUnits []string `xml:"ExcludingUnits"` + NormalUnits string `xml:"NormalUnits"` +} + +type UnitValuesConvertingUnits struct { + MultiplyingFactor string `xml:"MultiplyingFactor"` + Units string `xml:"Units"` } diff --git a/pkg/i2b2api/models/util.go b/pkg/i2b2api/models/util.go new file mode 100644 index 0000000..dcdb476 --- /dev/null +++ b/pkg/i2b2api/models/util.go @@ -0,0 +1,38 @@ +package models + +import ( + "strings" + "time" +) + +// ConnectionInfo contains the data needed for connection to i2b2. +type ConnectionInfo struct { + + // HiveURL is the URL of the i2b2 hive + HiveURL string + + // Domain is the i2b2 login domain + Domain string + + // Username is the i2b2 login username + Username string + + // Password is the i2b2 login password + Password string + + // Project is the i2b2 project ID + Project string + + // WaitTime is the wait time to send with i2b2 requests + WaitTime time.Duration +} + +// ConvertPathToI2b2Format converts a GeCo ontology path to an i2b2 path format +func ConvertPathToI2b2Format(path string) string { + return `\` + strings.Replace(path, "/", `\`, -1) +} + +// ConvertPathFromI2b2Format converts an i2b2 ontology path to a GeCo path format +func ConvertPathFromI2b2Format(path string) string { + return strings.Replace(strings.Replace(path, `\`, "/", -1), "//", "/", 1) +} diff --git a/pkg/i2b2api/ont_client.go b/pkg/i2b2api/ont_client.go new file mode 100644 index 0000000..775a833 --- /dev/null +++ b/pkg/i2b2api/ont_client.go @@ -0,0 +1,73 @@ +package i2b2api + +import ( + "fmt" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" +) + +// OntGetCategories makes an i2b2 API request to /OntologyService/getCategories. +func (c Client) OntGetCategories(reqMsgBody *models.OntReqGetCategoriesMessageBody) (*models.OntRespConceptsMessageBody, error) { + return c.ontConceptsRequest("/OntologyService/getCategories", reqMsgBody) +} + +// OntGetChildren makes an i2b2 API request to /OntologyService/getChildren. +func (c Client) OntGetChildren(reqMsgBody *models.OntReqGetChildrenMessageBody) (*models.OntRespConceptsMessageBody, error) { + return c.ontConceptsRequest("/OntologyService/getChildren", reqMsgBody) +} + +// OntGetTermInfo makes an i2b2 API request to /OntologyService/getTermInfo. +func (c Client) OntGetTermInfo(reqMsgBody *models.OntReqGetTermInfoMessageBody) (*models.OntRespConceptsMessageBody, error) { + return c.ontConceptsRequest("/OntologyService/getTermInfo", reqMsgBody) +} + +// ontConceptsRequest makes an i2b2 API request to a service under /OntologyService/ returning a models.OntRespConceptsMessageBody. +func (c Client) ontConceptsRequest(endpoint string, reqMsgBody models.MessageBody) (*models.OntRespConceptsMessageBody, error) { + xmlRequest := models.NewRequestWithBody(reqMsgBody) + xmlResponse := &models.Response{ + MessageBody: &models.OntRespConceptsMessageBody{}, + } + + if err := c.xmlRequest(endpoint, &xmlRequest, xmlResponse); err != nil { + return nil, fmt.Errorf("making XML request: %v", err) + } + + if mb, ok := xmlResponse.MessageBody.(*models.OntRespConceptsMessageBody); !ok { + return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) + } else { + return mb, nil + } +} + +// OntGetModifiers makes an i2b2 API request to /OntologyService/getModifiers. +func (c Client) OntGetModifiers(reqMsgBody *models.OntReqGetModifiersMessageBody) (*models.OntRespModifiersMessageBody, error) { + return c.ontModifiersRequest("/OntologyService/getModifiers", reqMsgBody) +} + +// OntGetModifierChildren makes an i2b2 API request to /OntologyService/getModifierChildren. +func (c Client) OntGetModifierChildren(reqMsgBody *models.OntReqGetModifierChildrenMessageBody) (*models.OntRespModifiersMessageBody, error) { + return c.ontModifiersRequest("/OntologyService/getModifierChildren", reqMsgBody) +} + +// OntGetModifierInfo makes an i2b2 API request to /OntologyService/getModifierInfo. +func (c Client) OntGetModifierInfo(reqMsgBody *models.OntReqGetModifierInfoMessageBody) (*models.OntRespModifiersMessageBody, error) { + return c.ontModifiersRequest("/OntologyService/getModifierInfo", reqMsgBody) +} + +// ontModifiersRequest makes an i2b2 API request to a service under /OntologyService/ returning a models.OntRespModifiersMessageBody. +func (c Client) ontModifiersRequest(endpoint string, reqMsgBody models.MessageBody) (*models.OntRespModifiersMessageBody, error) { + xmlRequest := models.NewRequestWithBody(reqMsgBody) + xmlResponse := &models.Response{ + MessageBody: &models.OntRespModifiersMessageBody{}, + } + + if err := c.xmlRequest(endpoint, &xmlRequest, xmlResponse); err != nil { + return nil, fmt.Errorf("making XML request: %v", err) + } + + if mb, ok := xmlResponse.MessageBody.(*models.OntRespModifiersMessageBody); !ok { + return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) + } else { + return mb, nil + } +} diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go new file mode 100644 index 0000000..e1cacbd --- /dev/null +++ b/pkg/i2b2datasource/datasource.go @@ -0,0 +1,133 @@ +package i2b2datasource + +import ( + "fmt" + "time" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api" + i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/ldsec/geco/pkg/datamanager" + "github.com/mitchellh/mapstructure" + "github.com/sirupsen/logrus" +) + +// todo: linting, remove from models (both packages) + +type queryType string +const ( + searchConceptQueryType queryType = "searchConcept" + searchModifierQueryType queryType = "searchModifier" + exploreQueryQueryType queryType = "exploreQuery" + getCohortsQueryType queryType = "getCohorts" + addCohortQueryType queryType = "addCohort" + deleteCohortQueryType queryType = "deleteCohort" + survivalQueryQueryType queryType = "survivalQuery" + searchOntologyQueryType queryType = "searchOntology" +) + +type I2b2DataSource struct { + + // init is true if the I2b2DataSource has been initialized. + init bool + + // dm is the GeCo data manager + dm *datamanager.DataManager + + // logger is the logger from GeCo + logger logrus.FieldLogger + + // i2b2Client is the i2b2 client + i2b2Client i2b2api.Client + + // i2b2OntMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 + i2b2OntMaxElements string +} + + +func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) (err error) { + fmt.Println("called init") + + ds.dm = dm + ds.logger = logger + + // todo: config keys + // i2b2.api.username + // i2b2.db.xxx + // datasource.db.xxx + + + // parse i2b2 API connection info and initialize i2b2 client + ci := i2b2apimodels.ConnectionInfo{ + HiveURL: config["i2b2.api.url"], + Domain: config["i2b2.api.domain"], + Username: config["i2b2.api.username"], + Password: config["i2b2.api.password"], + Project: config["i2b2.api.project"], + } + + if ci.WaitTime, err = time.ParseDuration(config["i2b2.api.wait-time"]); err != nil { + err = fmt.Errorf("parsing i2b2 wait time: %v", err) + logger.Error(err) + return err + } + + ds.i2b2Client = i2b2api.Client{ + Logger: logger, + Ci:ci, + } + ds.i2b2OntMaxElements = config["i2b2.api.ont-max-elements"] + + ds.init = true + ds.logger.Infof("initialized i2b2 data source for %v", ci.HiveURL) + return nil +} + +// todo: exists in GeCo, can be exposed by SDK code +func (ds I2b2DataSource) logError(errMsg string, causedBy error) (err error) { + if causedBy == nil { + err = fmt.Errorf("%v", errMsg) + } else { + err = fmt.Errorf("%v: %v", errMsg, causedBy) + } + ds.logger.Error(err) + return err +} + +func (ds I2b2DataSource) Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) { + if !ds.init { + panic(fmt.Errorf("data source is not initialized")) + } + ds.logger.Infof("executing operation %v for user %v", operation, userID) + ds.logger.Debugf("parameters: %+v", parameters) + ds.logger.Debugf("resultsSharedIds: %+v", resultsSharedIds) + + // todo: decoder might need squash param set + + results = make(map[string]interface{}) + switch queryType(operation) { + case searchConceptQueryType: + decodedParams := &models.SearchConceptParameters{} + if err := mapstructure.Decode(parameters, decodedParams); err != nil { + return nil, ds.logError("decoding parameters", err) + } else if searchResults, err := ds.SearchConcept(decodedParams); err != nil { + return nil, ds.logError("executing query", err) + } else if err := mapstructure.Decode(searchResults, &results); err != nil { + return nil, ds.logError("encoding results", err) + } + + case searchModifierQueryType: + decodedParams := &models.SearchModifierParameters{} + if err := mapstructure.Decode(parameters, decodedParams); err != nil { + return nil, ds.logError("decoding parameters", err) + } else if searchResults, err := ds.SearchModifier(decodedParams); err != nil { + return nil, ds.logError("executing query", err) + } else if err := mapstructure.Decode(searchResults, &results); err != nil { + return nil, ds.logError("encoding results", err) + } + + default: + return + } + return +} diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go new file mode 100644 index 0000000..6a33a3d --- /dev/null +++ b/pkg/i2b2datasource/datasource_test.go @@ -0,0 +1,49 @@ +package i2b2datasource + +import ( + "testing" + + "github.com/ldsec/geco/pkg/common/configuration" + "github.com/ldsec/geco/pkg/datamanager" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func GetTestDataSource(t *testing.T) I2b2DataSource { + ds := I2b2DataSource{} + + dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) + require.NoError(t, err) + + config := make(map[string]string) + config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" + config["i2b2.api.domain"] = "i2b2demo" + config["i2b2.api.username"] = "demo" + config["i2b2.api.password"] = "changeme" + config["i2b2.api.project"] = "Demo" + config["i2b2.api.wait-time"] = "10s" + config["i2b2.api.ont-max-elements"] = "200" + + logrus.StandardLogger().SetLevel(logrus.DebugLevel) + err = ds.Init(dm, logrus.StandardLogger(), config) + require.NoError(t, err) + return ds +} + +func TestDataManager(t *testing.T) { + + ds := GetTestDataSource(t) + + doId, err := ds.dm.AddDataObject(datamanager.NewFloatVector([]float64{0, 1, 2, 3, 4}), false) + require.NoError(t, err) + t.Logf("test data object ID is %v", doId) + + do, err := ds.dm.GetDataObject(doId) + require.NoError(t, err) + require.Equal(t, doId, do.ID) + require.Equal(t, datamanager.FloatVector, do.Type) + + data, err := do.FloatVector() + require.NoError(t, err) + require.InDeltaSlice(t, []float64{0, 1, 2, 3, 4}, data, 0.0001) +} diff --git a/pkg/i2b2datasource/i2b2-data-source.go b/pkg/i2b2datasource/i2b2-data-source.go deleted file mode 100644 index 55b5e2e..0000000 --- a/pkg/i2b2datasource/i2b2-data-source.go +++ /dev/null @@ -1,34 +0,0 @@ -package i2b2datasource - -import ( - "fmt" - - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2" - "github.com/ldsec/geco/pkg/datamanager" - "github.com/sirupsen/logrus" -) - -type I2b2DataSource struct { - - // dm is the GeCo data manager - dm *datamanager.DataManager - - // logger is the logger from GeCo - logger *logrus.Logger - - // i2b2C - i2b2Client i2b2.Client - -} - -func (ds I2b2DataSource) Init(dm *datamanager.DataManager, logger *logrus.Logger, config map[string]string) error { - ds.dm = dm - - fmt.Println("called init") - return nil -} - -func (ds I2b2DataSource) Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) { - fmt.Println("called query") - return nil, nil -} diff --git a/pkg/i2b2datasource/i2b2-data-source_test.go b/pkg/i2b2datasource/i2b2-data-source_test.go deleted file mode 100644 index e0b2af4..0000000 --- a/pkg/i2b2datasource/i2b2-data-source_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package i2b2datasource - -import ( - "testing" - - "github.com/ldsec/geco/pkg/common/configuration" - "github.com/ldsec/geco/pkg/datamanager" - "github.com/stretchr/testify/require" -) - -func TestDataManager(t *testing.T) { - - ds := I2b2DataSource{} - - dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) - require.NoError(t, err) - - err = ds.Init(dm, nil) - require.NoError(t, err) - - doId, err := dm.AddDataObject(datamanager.NewFloatVector([]float64{0, 1, 2, 3, 4}), false) - require.NoError(t, err) - t.Logf("test data object ID is %v", doId) - - do, err := dm.GetDataObject(doId) - require.NoError(t, err) - require.Equal(t, doId, do.ID) - require.Equal(t, datamanager.FloatVector, do.Type) - - data, err := do.FloatVector() - require.NoError(t, err) - require.InDeltaSlice(t, []float64{0, 1, 2, 3, 4}, data, 0.0001) -} diff --git a/pkg/i2b2datasource/models/common.go b/pkg/i2b2datasource/models/common.go new file mode 100644 index 0000000..7a33ab9 --- /dev/null +++ b/pkg/i2b2datasource/models/common.go @@ -0,0 +1,25 @@ +package models + +import "fmt" + +type parseError error + +func recoverParseError(retErr *error) { + if r := recover(); r != nil { + if parseErr, ok := r.(parseError); !ok { + panic(r) // panic was not caused by a parse error + } else { + *retErr = fmt.Errorf("parsing value: %v", parseErr) + } + } +} + +func getString(params map[string]interface{}, key string) string { + if stringVal, ok := params[key]; !ok { + panic(parseError(fmt.Errorf("key \"%v\" not found in params", key))) + } else if stringCast, ok := stringVal.(string); !ok { + panic(parseError(fmt.Errorf("key \"%v\" is not a string (%T)", key, stringVal))) + } else { + return stringCast + } +} diff --git a/pkg/i2b2datasource/models/search.go b/pkg/i2b2datasource/models/search.go new file mode 100644 index 0000000..2c3410a --- /dev/null +++ b/pkg/i2b2datasource/models/search.go @@ -0,0 +1,62 @@ +package models + +// --- parameters + +// func NewSearchConceptParameters(params map[string]interface{}) (model SearchConceptParameters, retErr error) { +// defer recoverParseError(&retErr) +// +// return SearchConceptParameters{ +// Path: getString(params, "path"), +// Operation: getString(params, "operation"), +// }, nil +// } + +type SearchConceptParameters struct { + Path string + Operation string // children | info | concept +} + +// func NewSearchModifierParameters(params map[string]interface{}) (model SearchModifierParameters, retErr error) { +// defer recoverParseError(&retErr) +// +// return SearchModifierParameters{ +// SearchConceptParameters: SearchConceptParameters{ +// Path: getString(params, "path"), +// Operation: getString(params, "operation"), +// }, +// AppliedPath: getString(params, "appliedPath"), +// AppliedConcept: getString(params, "appliedConcept"), +// }, nil +// } + +type SearchModifierParameters struct { + SearchConceptParameters + AppliedPath string + AppliedConcept string +} + +// --- results + +type SearchResults struct { + SearchResults []SearchResult +} + +type SearchResult struct { + Path string + AppliedPath string + Name string + DisplayName string + Code string + Comment string + Type string // concept | concept_container | concept_folder | modifier | modifier_container | modifier_folder | genomic_annotation + Leaf bool + Metadata SearchResultMetadata +} + +type SearchResultMetadata struct { + DataType string // PosInteger | Integer | Float | PosFloat | Enum | String + Oktousevalues string // Y + UnitValues struct { + NormalUnits string + } +} diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go new file mode 100644 index 0000000..94d06f4 --- /dev/null +++ b/pkg/i2b2datasource/search.go @@ -0,0 +1,179 @@ +package i2b2datasource + +import ( + "fmt" + "strings" + + i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" +) + +// SearchConcept retrieves the info about or the children of the concept identified by path. +func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) (*models.SearchResults, error) { + + // make the appropriate request to i2b2 + path := strings.TrimSpace(params.Path) + i2b2FormatPath := i2b2apimodels.ConvertPathToI2b2Format(path) + var resp *i2b2apimodels.OntRespConceptsMessageBody + var err error + + if len(path) == 0 { + return nil, fmt.Errorf("empty path") + } + + switch params.Operation { + case "info": + if path == "/" { + return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + } else { + req := i2b2apimodels.NewOntReqGetTermInfoMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetTermInfo(&req); err != nil { + return nil, fmt.Errorf("requesting term info: %v", err) + } + } + + case "children": + if path == "/" { + req := i2b2apimodels.NewOntReqGetCategoriesMessageBody() + if resp, err = ds.i2b2Client.OntGetCategories(&req); err != nil { + return nil, fmt.Errorf("requesting categories: %v", err) + } + + } else { + req := i2b2apimodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetChildren(&req); err != nil { + return nil, fmt.Errorf("requesting children: %v", err) + } + } + + default: + return nil, fmt.Errorf("invalid search operation: %v", params.Operation) + } + + // generate result from response + searchResults := make([]models.SearchResult, 0, len(resp.Concepts)) + for _, concept := range resp.Concepts { + searchResults = append(searchResults, parseI2b2Concept(concept)) + } + return &models.SearchResults{SearchResults: searchResults}, nil +} + +func parseI2b2Concept(concept i2b2apimodels.Concept) models.SearchResult { + + parsed := models.SearchResult{ + Name: concept.Name, + DisplayName: concept.Name, + Code: concept.Basecode, + Path: i2b2apimodels.ConvertPathFromI2b2Format(concept.Key), + AppliedPath: "@", + Comment: concept.Comment, + } + + parsed.Type, parsed.Leaf = parseI2b2VisualAttributes(concept.Visualattributes) + + if concept.Metadataxml != nil { + parsed.Metadata = models.SearchResultMetadata{ + DataType: concept.Metadataxml.DataType, + Oktousevalues: concept.Metadataxml.Oktousevalues, + UnitValues: struct { + NormalUnits string + }{ + NormalUnits: concept.Metadataxml.UnitValues.NormalUnits, + }, + } + } + + return parsed +} + +func parseI2b2VisualAttributes(visualAttributes string) (kind string, leaf bool) { + switch visualAttributes[0] { + // i2b2 leaf + case 'L': + return "concept", true + case 'R': + return "modifier", true + + // i2b2 container + case 'C': + return "concept_container", false + case 'O': + return "modifier_container", false + + // i2b2 folder (& default) + default: + fallthrough + case 'F': + return "concept_folder", false + case 'D': + return "modifier_folder", false + } +} + +// SearchModifier retrieves the info about or the children of the modifier identified by path. +func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) (*models.SearchResults, error) { + + // make the appropriate request to i2b2 + path := strings.TrimSpace(params.Path) + i2b2FormatPath := i2b2apimodels.ConvertPathToI2b2Format(path) + var resp *i2b2apimodels.OntRespModifiersMessageBody + var err error + + if len(path) == 0 { + return nil, fmt.Errorf("empty path") + } + + switch params.Operation { + case "concept": + if path == "/" { + return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + + } else { + req := i2b2apimodels.NewOntReqGetModifiersMessageBody(i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetModifiers(&req); err != nil { + return nil, fmt.Errorf("requesting modifiers of concept: %v", err) + } + } + + case "info": + if path == "/" { + return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + + } else { + i2b2FormatAppliedPath := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedPath))[1:] + req := i2b2apimodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) + if resp, err = ds.i2b2Client.OntGetModifierInfo(&req); err != nil { + return nil, fmt.Errorf("requesting info of modifier: %v", err) + } + } + + case "children": + if path == "/" { + return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + + } else { + i2b2FormatAppliedPath := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedPath))[1:] + i2b2FormatAppliedConcept := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) + req := i2b2apimodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) + if resp, err = ds.i2b2Client.OntGetModifierChildren(&req); err != nil { + return nil, fmt.Errorf("requesting children of modifier: %v", err) + } + } + + default: + return nil, fmt.Errorf("invalid search operation: %v", params.Operation) + } + + // generate result from response + searchResults := make([]models.SearchResult, 0, len(resp.Modifiers)) + for _, modifier := range resp.Modifiers { + searchResults = append(searchResults, parseI2b2Modifier(modifier)) + } + return &models.SearchResults{SearchResults: searchResults}, nil +} + +func parseI2b2Modifier(modifier i2b2apimodels.Modifier) models.SearchResult { + res := parseI2b2Concept(modifier.Concept) + res.AppliedPath = i2b2apimodels.ConvertPathFromI2b2Format(modifier.AppliedPath) + return res +} diff --git a/pkg/i2b2datasource/search_test.go b/pkg/i2b2datasource/search_test.go new file mode 100644 index 0000000..66eede2 --- /dev/null +++ b/pkg/i2b2datasource/search_test.go @@ -0,0 +1,229 @@ +package i2b2datasource + +import ( + "testing" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/stretchr/testify/require" +) + +func TestSearchConceptInfo(t *testing.T) { + ds := GetTestDataSource(t) + + searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/", + Operation: "info", + }) + require.NoError(t, err) + require.Empty(t, searchResults.SearchResults) + + searchResults, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/", + Operation: "info", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/test/") + + searchResults, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/1", + Operation: "info", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/test/1") +} + +func TestSearchConceptChildren(t *testing.T) { + ds := GetTestDataSource(t) + + searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/", + Operation: "children", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/test/") + + searchResults, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/", + Operation: "children", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 3, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/test/") + + searchResults, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/2", + Operation: "children", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Empty(t, searchResults.SearchResults) +} + +func TestSearchConceptError(t *testing.T) { + ds := GetTestDataSource(t) + + _, err := ds.SearchConcept(&models.SearchConceptParameters{ + Path: "", + Operation: "children", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "empty path") + + _, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/", + Operation: "xxxxxx", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid search operation") + + _, err = ds.SearchConcept(&models.SearchConceptParameters{ + Path: "/TEST/test/4433", + Operation: "children", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "error") +} + +func TestSearchModifierConcept(t *testing.T) { + ds := GetTestDataSource(t) + + searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/test/1/", + Operation: "concept", + }, + AppliedPath: "", + AppliedConcept: "", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/") + + searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/test/2/", + Operation: "concept", + }, + AppliedPath: "", + AppliedConcept: "", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 2, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/2") + require.Contains(t, searchResults.SearchResults[1].Path, "/TEST/modifiers/2") +} + +func TestSearchModifierInfo(t *testing.T) { + ds := GetTestDataSource(t) + + searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "info", + }, + AppliedPath: "/test/%", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/") + + searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/1", + Operation: "info", + }, + AppliedPath: "/test/%", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/1") +} + +func TestSearchModifierChildren(t *testing.T) { + ds := GetTestDataSource(t) + + searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "children", + }, + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/1/", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 1, len(searchResults.SearchResults)) + require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/1/") + + searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "children", + }, + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/2/", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 2, len(searchResults.SearchResults)) + + searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "children", + }, + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", + }) + require.NoError(t, err) + t.Logf("%+v", searchResults.SearchResults) + require.Equal(t, 2, len(searchResults.SearchResults)) +} + +func TestSearchModifierError(t *testing.T) { + ds := GetTestDataSource(t) + + _, err := ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "", + Operation: "children", + }, + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "empty path") + + _, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "xxxxx", + }, + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid search operation") + + _, err = ds.SearchModifier(&models.SearchModifierParameters{ + SearchConceptParameters: models.SearchConceptParameters{ + Path: "/TEST/modifiers/", + Operation: "children", + }, + AppliedPath: "/test", + AppliedConcept: "/TEST/test/3/", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "error") +} From 8dd2eae57b46d6a13ba64daaa605f2cc2c3f4a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 12:33:52 +0100 Subject: [PATCH 22/81] add/fix test of docker image for ontology query --- test/i2b2/ont_get_categories_req.xml | 2 +- test/i2b2/test_i2b2_docker.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/i2b2/ont_get_categories_req.xml b/test/i2b2/ont_get_categories_req.xml index b7f1fe4..f657d44 100644 --- a/test/i2b2/ont_get_categories_req.xml +++ b/test/i2b2/ont_get_categories_req.xml @@ -48,6 +48,6 @@ <result_waittime_ms>180000</result_waittime_ms> </request_header> <message_body> - <get_categories type="default" blob="false" hiddens="true" synonyms="false"/> + <ontns:get_categories type="core" blob="false" hiddens="true" synonyms="false"/> </message_body> </msgns:request> diff --git a/test/i2b2/test_i2b2_docker.sh b/test/i2b2/test_i2b2_docker.sh index 7671387..b0d7451 100755 --- a/test/i2b2/test_i2b2_docker.sh +++ b/test/i2b2/test_i2b2_docker.sh @@ -24,7 +24,7 @@ postXmlI2b2 PMService/getServices "${MSG}_req.xml" "${MSG}_resp.xml" xmllint --xpath "string(//status)" "${MSG}_resp.xml" | grep "PM processing completed" xmllint --xpath "string(//password)" "${MSG}_resp.xml" | grep "SessionKey" -# todo: with test data, fails with empty DB -#MSG=ont_get_categories -#postXmlI2b2 TBD "${MSG}_req.xml" "${MSG}_resp.xml" -#xmllint --xpath "string(//TBD)" "${MSG}_resp.xml" | grep "TBD" +MSG=ont_get_categories +postXmlI2b2 OntologyService/getCategories "${MSG}_req.xml" "${MSG}_resp.xml" +xmllint --xpath "string(//status)" "${MSG}_resp.xml" | grep "Ontology processing completed" +xmllint --xpath "string(//name)" "${MSG}_resp.xml" | grep "Test Ontology" From e4eb177118ddeedb8020837bd0e122c4aaa61156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 12:41:11 +0100 Subject: [PATCH 23/81] fix plugin declaration and test --- cmd/geco-i2b2-data-source/data-source-plugin.go | 2 +- internal/plugin_test.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go index 8a2e0c8..a1ff663 100644 --- a/cmd/geco-i2b2-data-source/data-source-plugin.go +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -6,4 +6,4 @@ import ( ) // DataSourcePlugin exports an instance of the GeCo i2b2 data source for the plugin. -var DataSourcePlugin pkg.DataSource = i2b2datasource.I2b2DataSource{} +var DataSourcePlugin pkg.DataSource = &i2b2datasource.I2b2DataSource{} diff --git a/internal/plugin_test.go b/internal/plugin_test.go index 7beb5fb..e2e7130 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -7,6 +7,7 @@ import ( "github.com/ldsec/geco-i2b2-data-source/pkg" "github.com/ldsec/geco/pkg/common/configuration" "github.com/ldsec/geco/pkg/datamanager" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -23,7 +24,17 @@ func TestPlugin(t *testing.T) { dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) require.NoError(t, err) - err = (*ds).Init(dm, nil) + config := make(map[string]string) + config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" + config["i2b2.api.domain"] = "i2b2demo" + config["i2b2.api.username"] = "demo" + config["i2b2.api.password"] = "changeme" + config["i2b2.api.project"] = "Demo" + config["i2b2.api.wait-time"] = "10s" + config["i2b2.api.ont-max-elements"] = "200" + + logrus.StandardLogger().SetLevel(logrus.DebugLevel) + err = (*ds).Init(dm, logrus.StandardLogger(), config) require.NoError(t, err) _, err = (*ds).Query("", "", nil, nil) From 3d86c141f5ec840592cc0d3d433c00d483cac29f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 14:21:19 +0100 Subject: [PATCH 24/81] finalize i2b2api package with ont and crc queries for explore queries --- pkg/i2b2api/client.go | 5 -- pkg/i2b2api/crc_client.go | 46 ++++++++++ pkg/i2b2api/models/api_tmp.go | 75 ---------------- pkg/i2b2api/models/common.go | 91 ++++++++++---------- pkg/i2b2api/models/crc_pdo.go | 64 +++++++------- pkg/i2b2api/models/crc_psm.go | 156 ++++++++++++++++++++-------------- pkg/i2b2api/models/ont.go | 4 +- 7 files changed, 218 insertions(+), 223 deletions(-) create mode 100644 pkg/i2b2api/crc_client.go delete mode 100644 pkg/i2b2api/models/api_tmp.go diff --git a/pkg/i2b2api/client.go b/pkg/i2b2api/client.go index ff86bf1..0ed43c0 100644 --- a/pkg/i2b2api/client.go +++ b/pkg/i2b2api/client.go @@ -11,11 +11,6 @@ import ( "github.com/sirupsen/logrus" ) -// todo: define API of what's needed from geco -// todo: then define the query functions that take as arguments the API models -// todo: then adapt the existing XML request to be made -// todo: add tests and integration with CI - // Client is an i2b2 client for its XML API type Client struct { diff --git a/pkg/i2b2api/crc_client.go b/pkg/i2b2api/crc_client.go new file mode 100644 index 0000000..2854761 --- /dev/null +++ b/pkg/i2b2api/crc_client.go @@ -0,0 +1,46 @@ +package i2b2api + +import ( + "fmt" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" +) + +// CrcPsmReqFromQueryDef makes an i2b2 API request to /QueryToolService/request for CRC PSM from query definition. +func (c Client) CrcPsmReqFromQueryDef(reqMsgBody *models.CrcPsmReqFromQueryDefMessageBody) (*models.CrcPsmRespMessageBody, error) { + xmlRequest := models.NewRequestWithBody(reqMsgBody) + xmlResponse := &models.Response{ + MessageBody: &models.CrcPsmRespMessageBody{}, + } + + if err := c.xmlRequest("/QueryToolService/request", &xmlRequest, xmlResponse); err != nil { + return nil, fmt.Errorf("making XML request: %v", err) + } + + if mb, ok := xmlResponse.MessageBody.(*models.CrcPsmRespMessageBody); !ok { + return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) + } else if err := mb.CheckStatus(); err != nil { + return nil, fmt.Errorf("found error in i2b2 CRC PSM response: %v", err) + } else { + return mb, nil + } +} + +// CrcPdoReqFromInputList makes an i2b2 API request to /QueryToolService/pdorequest for CRC PDO from input list. +func (c Client) CrcPdoReqFromInputList(reqMsgBody *models.CrcPdoReqFromInputListMessageBody) (*models.CrcPdoRespMessageBody, error) { + xmlRequest := models.NewRequestWithBody(reqMsgBody) + xmlResponse := &models.Response{ + MessageBody: &models.CrcPdoRespMessageBody{}, + } + + if err := c.xmlRequest("/QueryToolService/pdorequest", &xmlRequest, xmlResponse); err != nil { + return nil, fmt.Errorf("making XML request: %v", err) + } + + if mb, ok := xmlResponse.MessageBody.(*models.CrcPdoRespMessageBody); !ok { + return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) + } else { + return mb, nil + } +} + diff --git a/pkg/i2b2api/models/api_tmp.go b/pkg/i2b2api/models/api_tmp.go deleted file mode 100644 index fbeee5e..0000000 --- a/pkg/i2b2api/models/api_tmp.go +++ /dev/null @@ -1,75 +0,0 @@ -package models - -// todo: those are c/c from swagger-generated medco models, they should be updated according to the definition of the API exposed through geco - -// APIPanel todo: should be changed according to the API exposed -type APIPanel struct { - - // items containing cohort names - CohortItems []string `json:"cohortItems"` - - // items containing i2b2 concepts (and optionally modifiers) - ConceptItems []*APIPanelConceptItemsItems0 `json:"conceptItems"` - - // exclude the i2b2 panel - // Required: true - Not *bool `json:"not"` - - // panel timing - PanelTiming APITiming `json:"panelTiming,omitempty"` -} - -// APIPanelConceptItemsItems0 todo: should be changed according to the API exposed -type APIPanelConceptItemsItems0 struct { - - // encrypted - // Required: true - Encrypted *bool `json:"encrypted"` - - // modifier - Modifier *APIPanelConceptItemsItems0Modifier `json:"modifier,omitempty"` - - // # NUMBER operators EQ: equal NE: not equal GT: greater than GE: greater than or equal LT: less than LE: less than or equal BETWEEN: between (value syntax: "x and y") # TEXT operators IN: in (value syntax: "'x','y','z'") LIKE[exact]: equal LIKE[begin]: begins with LIKE[end]: ends with LIKE[contains]: contains - // - // Enum: [EQ NE GT GE LT LE BETWEEN IN LIKE[exact] LIKE[begin] LIKE[end] LIKE[contains]] - Operator string `json:"operator,omitempty"` - - // query term - // Required: true - // Pattern: ^([\w=-]+)$|^((\/[^\/]+)+\/)$ - QueryTerm *string `json:"queryTerm"` - - // type - // Enum: [NUMBER TEXT] - Type string `json:"type,omitempty"` - - // value - Value string `json:"value,omitempty"` -} - -// APIPanelConceptItemsItems0Modifier todo: should be changed according to the API exposed -type APIPanelConceptItemsItems0Modifier struct { - - // applied path - // Required: true - // Pattern: ^((\/[^\/]+)+\/%?)$ - AppliedPath *string `json:"appliedPath"` - - // modifier key - // Required: true - // Pattern: ^((\/[^\/]+)+\/)$ - ModifierKey *string `json:"modifierKey"` -} - -// APITiming todo: should be changed according to the API exposed -type APITiming string -const ( - // APITimingAny captures enum value "any" - APITimingAny APITiming = "any" - - // APITimingSamevisit captures enum value "samevisit" - APITimingSamevisit APITiming = "samevisit" - - // APITimingSameinstancenum captures enum value "sameinstancenum" - APITimingSameinstancenum APITiming = "sameinstancenum" -) diff --git a/pkg/i2b2api/models/common.go b/pkg/i2b2api/models/common.go index 9a2d85e..9c31492 100644 --- a/pkg/i2b2api/models/common.go +++ b/pkg/i2b2api/models/common.go @@ -7,6 +7,52 @@ import ( "time" ) + +// MessageBody is an i2b2 XML generic body +type MessageBody interface{} + +// MessageHeader is an i2b2 XML header embedded in a request or response +type MessageHeader struct { + XMLName xml.Name `xml:"message_header"` + + I2b2VersionCompatible string `xml:"i2b2_version_compatible"` + Hl7VersionCompatible string `xml:"hl7_version_compatible"` + + SendingApplicationApplicationName string `xml:"sending_application>application_name"` + SendingApplicationApplicationVersion string `xml:"sending_application>application_version"` + + SendingFacilityFacilityName string `xml:"sending_facility>facility_name"` + + ReceivingApplicationApplicationName string `xml:"receiving_application>application_name"` + ReceivingApplicationApplicationVersion string `xml:"receiving_application>application_version"` + + ReceivingFacilityFacilityName string `xml:"receiving_facility>facility_name"` + + DatetimeOfMessage string `xml:"datetime_of_message"` + + SecurityDomain string `xml:"security>domain"` + SecurityUsername string `xml:"security>username"` + SecurityPassword string `xml:"security>password"` + + MessageTypeMessageCode string `xml:"message_type>message_code"` + MessageTypeEventType string `xml:"message_type>event_type"` + MessageTypeMessageStructure string `xml:"message_type>message_structure"` + + MessageControlIDSessionID string `xml:"message_control_id>session_id"` + MessageControlIDMessageNum string `xml:"message_control_id>message_num"` + MessageControlIDInstanceNum string `xml:"message_control_id>instance_num"` + + ProcessingIDProcessingID string `xml:"processing_id>processing_id"` + ProcessingIDProcessingMode string `xml:"processing_id>processing_mode"` + + AcceptAcknowledgementType string `xml:"accept_acknowledgement_type"` + ApplicationAcknowledgementType string `xml:"application_acknowledgement_type"` + CountryCode string `xml:"country_code"` + ProjectID string `xml:"project_id"` +} + +// --- request + // NewRequest creates a new ready-to-use i2b2 request, with a nil message body. func NewRequest() Request { now := time.Now() @@ -95,52 +141,14 @@ func (response Response) CheckStatus() error { return nil } -// MessageHeader is an i2b2 XML header embedded in a request or response -type MessageHeader struct { - XMLName xml.Name `xml:"message_header"` - - I2b2VersionCompatible string `xml:"i2b2_version_compatible"` - Hl7VersionCompatible string `xml:"hl7_version_compatible"` - - SendingApplicationApplicationName string `xml:"sending_application>application_name"` - SendingApplicationApplicationVersion string `xml:"sending_application>application_version"` - - SendingFacilityFacilityName string `xml:"sending_facility>facility_name"` - - ReceivingApplicationApplicationName string `xml:"receiving_application>application_name"` - ReceivingApplicationApplicationVersion string `xml:"receiving_application>application_version"` - - ReceivingFacilityFacilityName string `xml:"receiving_facility>facility_name"` - - DatetimeOfMessage string `xml:"datetime_of_message"` - - SecurityDomain string `xml:"security>domain"` - SecurityUsername string `xml:"security>username"` - SecurityPassword string `xml:"security>password"` - - MessageTypeMessageCode string `xml:"message_type>message_code"` - MessageTypeEventType string `xml:"message_type>event_type"` - MessageTypeMessageStructure string `xml:"message_type>message_structure"` - - MessageControlIDSessionID string `xml:"message_control_id>session_id"` - MessageControlIDMessageNum string `xml:"message_control_id>message_num"` - MessageControlIDInstanceNum string `xml:"message_control_id>instance_num"` - - ProcessingIDProcessingID string `xml:"processing_id>processing_id"` - ProcessingIDProcessingMode string `xml:"processing_id>processing_mode"` - - AcceptAcknowledgementType string `xml:"accept_acknowledgement_type"` - ApplicationAcknowledgementType string `xml:"application_acknowledgement_type"` - CountryCode string `xml:"country_code"` - ProjectID string `xml:"project_id"` -} - // RequestHeader is an i2b2 XML header embedded in a request type RequestHeader struct { XMLName xml.Name `xml:"request_header"` ResultWaittimeMs string `xml:"result_waittime_ms"` } +// --- response + // ResponseHeader is an i2b2 XML header embedded in a response type ResponseHeader struct { XMLName xml.Name `xml:"response_header"` @@ -166,6 +174,3 @@ type ResponseHeader struct { } `xml:"conditions"` } `xml:"result_status"` } - -// MessageBody is an i2b2 XML generic body -type MessageBody interface{} diff --git a/pkg/i2b2api/models/crc_pdo.go b/pkg/i2b2api/models/crc_pdo.go index 001eb9c..c89475b 100644 --- a/pkg/i2b2api/models/crc_pdo.go +++ b/pkg/i2b2api/models/crc_pdo.go @@ -4,40 +4,40 @@ import ( "encoding/xml" ) -// // NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request -// func NewCrcPdoReqFromInputList(ci ConnectionInfo, patientSetID string) Request { -// -// // PDO header -// pdoHeader := PdoHeader{ -// PatientSetLimit: "0", -// EstimatedTime: "0", -// RequestType: "getPDO_fromInputList", -// } -// -// // PDO request -// pdoRequest := PdoRequestFromInputList{ -// Type: "crcpdons:GetPDOFromInputList_requestType", -// Xsi: "http://www.w3.org/2001/XMLSchema-instance", -// } -// -// // set request for patient set ID -// pdoRequest.InputList.PatientList.Max = "1000000" -// pdoRequest.InputList.PatientList.Min = "0" -// pdoRequest.InputList.PatientList.PatientSetCollID = patientSetID -// pdoRequest.OutputOption.Name = "none" -// pdoRequest.OutputOption.PatientSet.Blob = "false" -// pdoRequest.OutputOption.PatientSet.TechData = "false" -// pdoRequest.OutputOption.PatientSet.OnlyKeys = "false" -// pdoRequest.OutputOption.PatientSet.Select = "using_input_list" -// -// return NewRequestWithBody(ci, CrcPdoReqFromInputListMessageBody{ -// PdoHeader: pdoHeader, -// PdoRequest: pdoRequest, -// }) -// } - // --- request +// NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request +func NewCrcPdoReqFromInputList(patientSetID string) CrcPdoReqFromInputListMessageBody { + + // PDO header + pdoHeader := PdoHeader{ + PatientSetLimit: "0", + EstimatedTime: "0", + RequestType: "getPDO_fromInputList", + } + + // PDO request + pdoRequest := PdoRequestFromInputList{ + Type: "crcpdons:GetPDOFromInputList_requestType", + Xsi: "http://www.w3.org/2001/XMLSchema-instance", + } + + // set request for patient set ID + pdoRequest.InputList.PatientList.Max = "1000000" + pdoRequest.InputList.PatientList.Min = "0" + pdoRequest.InputList.PatientList.PatientSetCollID = patientSetID + pdoRequest.OutputOption.Name = "none" + pdoRequest.OutputOption.PatientSet.Blob = "false" + pdoRequest.OutputOption.PatientSet.TechData = "false" + pdoRequest.OutputOption.PatientSet.OnlyKeys = "false" + pdoRequest.OutputOption.PatientSet.Select = "using_input_list" + + return CrcPdoReqFromInputListMessageBody{ + PdoHeader: pdoHeader, + PdoRequest: pdoRequest, + } +} + // CrcPdoReqFromInputListMessageBody is an i2b2 XML message body for CRC PDO request from input list type CrcPdoReqFromInputListMessageBody struct { XMLName xml.Name `xml:"message_body"` diff --git a/pkg/i2b2api/models/crc_psm.go b/pkg/i2b2api/models/crc_psm.go index 5683221..6826316 100644 --- a/pkg/i2b2api/models/crc_psm.go +++ b/pkg/i2b2api/models/crc_psm.go @@ -2,14 +2,16 @@ package models import ( "encoding/xml" - "errors" + "fmt" "strconv" "strings" ) +// --- request + // NewCrcPsmReqFromQueryDef returns a new request object for i2b2 psm request -func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels []*APIPanel, - resultOutputs []ResultOutputName, queryTiming APITiming) Request { +func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels []Panel, queryTiming Timing, + resultOutputs []ResultOutputName) CrcPsmReqFromQueryDefMessageBody { // PSM header psmHeader := PsmHeader{ @@ -30,63 +32,66 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ QueryName: queryName, QueryID: queryName, QueryDescription: "Query from GeCo i2b2 data source (" + queryName + ")", - QueryTiming: strings.ToUpper(string(queryTiming)), + QueryTiming: string(queryTiming), SpecificityScale: "0", + Panels: queryPanels, } // embed query in request - for p, queryPanel := range queryPanels { - - invert := "0" - if *queryPanel.Not { - invert = "1" - } - - i2b2Panel := Panel{ - PanelNumber: strconv.Itoa(p + 1), - PanelAccuracyScale: "100", - Invert: invert, - PanelTiming: strings.ToUpper(string(queryPanel.PanelTiming)), - TotalItemOccurrences: "1", - } - - for _, queryItem := range queryPanel.ConceptItems { - i2b2Item := Item{ - ItemKey: ConvertPathToI2b2Format(*queryItem.QueryTerm), - } - if queryItem.Operator != "" && queryItem.Modifier == nil { - i2b2Item.ConstrainByValue = &ConstrainByValue{ - ValueType: queryItem.Type, - ValueOperator: queryItem.Operator, - ValueConstraint: queryItem.Value, - } - } - if queryItem.Modifier != nil { - i2b2Item.ConstrainByModifier = &ConstrainByModifier{ - AppliedPath: strings.ReplaceAll(*queryItem.Modifier.AppliedPath, "/", `\`), - ModifierKey: ConvertPathToI2b2Format(*queryItem.Modifier.ModifierKey), - } - if queryItem.Operator != "" { - i2b2Item.ConstrainByModifier.ConstrainByValue = &ConstrainByValue{ - ValueType: queryItem.Type, - ValueOperator: queryItem.Operator, - ValueConstraint: queryItem.Value, - } - } - } - i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) - } - - for _, cohort := range queryPanel.CohortItems { - - i2b2Item := Item{ - ItemKey: cohort, - } - i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) - } - - psmRequest.Panels = append(psmRequest.Panels, i2b2Panel) - } + //for p, queryPanel := range queryPanels { + + // todo: make constructors NewXXX for the models, and use that from data source + + // invert := "0" + // if *queryPanel.Not { + // invert = "1" + // } + // + // i2b2Panel := Panel{ + // PanelNumber: strconv.Itoa(p + 1), + // PanelAccuracyScale: "100", + // Invert: invert, + // PanelTiming: strings.ToUpper(string(queryPanel.PanelTiming)), + // TotalItemOccurrences: "1", + // } + + // for _, queryItem := range queryPanel.ConceptItems { + // i2b2Item := Item{ + // ItemKey: ConvertPathToI2b2Format(*queryItem.QueryTerm), + // } + // if queryItem.Operator != "" && queryItem.Modifier == nil { + // i2b2Item.ConstrainByValue = &ConstrainByValue{ + // ValueType: queryItem.Type, + // ValueOperator: queryItem.Operator, + // ValueConstraint: queryItem.Value, + // } + // } + // if queryItem.Modifier != nil { + // i2b2Item.ConstrainByModifier = &ConstrainByModifier{ + // AppliedPath: strings.ReplaceAll(*queryItem.Modifier.AppliedPath, "/", `\`), + // ModifierKey: ConvertPathToI2b2Format(*queryItem.Modifier.ModifierKey), + // } + // if queryItem.Operator != "" { + // i2b2Item.ConstrainByModifier.ConstrainByValue = &ConstrainByValue{ + // ValueType: queryItem.Type, + // ValueOperator: queryItem.Operator, + // ValueConstraint: queryItem.Value, + // } + // } + // } + // i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) + // } + // + // for _, cohort := range queryPanel.CohortItems { + // + // i2b2Item := Item{ + // ItemKey: cohort, + // } + // i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) + // } + + // psmRequest.Panels = append(psmRequest.Panels, i2b2Panel) + // } // embed result outputs for i, resultOutput := range resultOutputs { @@ -96,14 +101,12 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ }) } - return NewRequestWithBody(CrcPsmReqFromQueryDefMessageBody{ + return CrcPsmReqFromQueryDefMessageBody{ PsmHeader: psmHeader, PsmRequest: psmRequest, - }) + } } -// --- request - // CrcPsmReqFromQueryDefMessageBody is an i2b2 XML message body for CRC PSM request from query definition type CrcPsmReqFromQueryDefMessageBody struct { XMLName xml.Name `xml:"message_body"` @@ -141,6 +144,22 @@ type PsmRequestFromQueryDef struct { ResultOutputs []ResultOutput `xml:"result_output_list>result_output"` } +func NewPanel(panelNb int, not bool, timing Timing, items []Item) Panel { + invert := "0" + if not { + invert = "1" + } + + return Panel{ + PanelNumber: strconv.Itoa(panelNb + 1), + PanelAccuracyScale: "100", + Invert: invert, + Items: items, + PanelTiming: string(timing), + TotalItemOccurrences: "1", + } +} + // Panel is an i2b2 XML panel type Panel struct { PanelNumber string `xml:"panel_number"` @@ -187,8 +206,6 @@ type ResultOutput struct { // ResultOutputName is an i2b2 XML requested result type value type ResultOutputName string - -// enumerated values of ResultOutputName const ( Patientset ResultOutputName = "PATIENTSET" PatientEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" @@ -199,6 +216,13 @@ const ( PatientRaceCountXML ResultOutputName = "PATIENT_RACE_COUNT_XML" ) +type Timing string +const ( + TimingAny Timing = "ANY" + TimingSameVisit Timing = "SAMEVISIT" + TimingSameInstanceNum Timing = "SAMEINSTANCENUM" +) + // --- response // CrcPsmRespMessageBody is an i2b2 XML message body for CRC PSM response @@ -243,7 +267,7 @@ type CrcPsmRespMessageBody struct { } `xml:"response"` } -func (mb CrcPsmRespMessageBody) checkStatus() error { +func (mb CrcPsmRespMessageBody) CheckStatus() error { var errorMessages []string for _, status := range mb.Response.Status { if status.Type == "ERROR" || status.Type == "FATAL_ERROR" { @@ -252,7 +276,7 @@ func (mb CrcPsmRespMessageBody) checkStatus() error { } if len(errorMessages) != 0 { - return errors.New(strings.Join(errorMessages, "; ")) + return fmt.Errorf(strings.Join(errorMessages, "; ")) } return nil } @@ -276,9 +300,9 @@ type QueryResultInstance struct { } `xml:"query_status_type"` } -func (qri QueryResultInstance) checkStatus() error { +func (qri QueryResultInstance) CheckStatus() error { if qri.QueryStatusType.StatusTypeID != "3" { - return errors.New("i2b2 result instance does not have finished status") + return fmt.Errorf("i2b2 result instance does not have finished status: %v / %v / %v", qri.QueryStatusType.StatusTypeID, qri.QueryStatusType.Name, qri.QueryStatusType.Description) } return nil } diff --git a/pkg/i2b2api/models/ont.go b/pkg/i2b2api/models/ont.go index f440e27..f0206fe 100644 --- a/pkg/i2b2api/models/ont.go +++ b/pkg/i2b2api/models/ont.go @@ -4,6 +4,8 @@ import ( "encoding/xml" ) +// --- request + func NewOntReqGetTermInfoMessageBody(ontMaxElements, path string) OntReqGetTermInfoMessageBody { body := OntReqGetTermInfoMessageBody{} @@ -87,8 +89,6 @@ func NewOntReqGetModifierChildrenMessageBody(ontMaxElements, parent, appliedPath return body } -// --- request - type baseMessageBody struct { Hiddens string `xml:"hiddens,attr,omitempty"` Synonyms string `xml:"synonyms,attr,omitempty"` From 39a3db033964f2f74044c7af3fdb6849f5bdd5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:21:44 +0100 Subject: [PATCH 25/81] i2b2 api models modifications --- pkg/i2b2api/models/crc_psm.go | 70 ++++------------------------------- pkg/i2b2api/models/util.go | 5 +++ 2 files changed, 12 insertions(+), 63 deletions(-) diff --git a/pkg/i2b2api/models/crc_psm.go b/pkg/i2b2api/models/crc_psm.go index 6826316..1888d35 100644 --- a/pkg/i2b2api/models/crc_psm.go +++ b/pkg/i2b2api/models/crc_psm.go @@ -37,62 +37,6 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ Panels: queryPanels, } - // embed query in request - //for p, queryPanel := range queryPanels { - - // todo: make constructors NewXXX for the models, and use that from data source - - // invert := "0" - // if *queryPanel.Not { - // invert = "1" - // } - // - // i2b2Panel := Panel{ - // PanelNumber: strconv.Itoa(p + 1), - // PanelAccuracyScale: "100", - // Invert: invert, - // PanelTiming: strings.ToUpper(string(queryPanel.PanelTiming)), - // TotalItemOccurrences: "1", - // } - - // for _, queryItem := range queryPanel.ConceptItems { - // i2b2Item := Item{ - // ItemKey: ConvertPathToI2b2Format(*queryItem.QueryTerm), - // } - // if queryItem.Operator != "" && queryItem.Modifier == nil { - // i2b2Item.ConstrainByValue = &ConstrainByValue{ - // ValueType: queryItem.Type, - // ValueOperator: queryItem.Operator, - // ValueConstraint: queryItem.Value, - // } - // } - // if queryItem.Modifier != nil { - // i2b2Item.ConstrainByModifier = &ConstrainByModifier{ - // AppliedPath: strings.ReplaceAll(*queryItem.Modifier.AppliedPath, "/", `\`), - // ModifierKey: ConvertPathToI2b2Format(*queryItem.Modifier.ModifierKey), - // } - // if queryItem.Operator != "" { - // i2b2Item.ConstrainByModifier.ConstrainByValue = &ConstrainByValue{ - // ValueType: queryItem.Type, - // ValueOperator: queryItem.Operator, - // ValueConstraint: queryItem.Value, - // } - // } - // } - // i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) - // } - // - // for _, cohort := range queryPanel.CohortItems { - // - // i2b2Item := Item{ - // ItemKey: cohort, - // } - // i2b2Panel.Items = append(i2b2Panel.Items, i2b2Item) - // } - - // psmRequest.Panels = append(psmRequest.Panels, i2b2Panel) - // } - // embed result outputs for i, resultOutput := range resultOutputs { psmRequest.ResultOutputs = append(psmRequest.ResultOutputs, ResultOutput{ @@ -207,13 +151,13 @@ type ResultOutput struct { // ResultOutputName is an i2b2 XML requested result type value type ResultOutputName string const ( - Patientset ResultOutputName = "PATIENTSET" - PatientEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" - PatientCountXML ResultOutputName = "PATIENT_COUNT_XML" - PatientGenderCountXML ResultOutputName = "PATIENT_GENDER_COUNT_XML" - PatientAgeCountXML ResultOutputName = "PATIENT_AGE_COUNT_XML" - PatientVitalstatusCountXML ResultOutputName = "PATIENT_VITALSTATUS_COUNT_XML" - PatientRaceCountXML ResultOutputName = "PATIENT_RACE_COUNT_XML" + ResultOutputPatientSet ResultOutputName = "PATIENTSET" + ResultOutputEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" + ResultOutputCount ResultOutputName = "PATIENT_COUNT_XML" + ResultOutputGenderCount ResultOutputName = "PATIENT_GENDER_COUNT_XML" + ResultOutputAgeCount ResultOutputName = "PATIENT_AGE_COUNT_XML" + ResultOutputVitalStatusCount ResultOutputName = "PATIENT_VITALSTATUS_COUNT_XML" + ResultOutputRaceCount ResultOutputName = "PATIENT_RACE_COUNT_XML" ) type Timing string diff --git a/pkg/i2b2api/models/util.go b/pkg/i2b2api/models/util.go index dcdb476..c071814 100644 --- a/pkg/i2b2api/models/util.go +++ b/pkg/i2b2api/models/util.go @@ -27,6 +27,11 @@ type ConnectionInfo struct { WaitTime time.Duration } +// ConvertAppliedPathToI2b2Format converts a GeCo applied path to an i2b2 applied path format +func ConvertAppliedPathToI2b2Format(path string) string { + return strings.Replace(path, "/", `\`, -1) +} + // ConvertPathToI2b2Format converts a GeCo ontology path to an i2b2 path format func ConvertPathToI2b2Format(path string) string { return `\` + strings.Replace(path, "/", `\`, -1) From ca57396cbb55f0c4537d434ca8bac2de8d5eba85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:22:29 +0100 Subject: [PATCH 26/81] add models for explore query --- pkg/i2b2datasource/models/explore_query.go | 77 ++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pkg/i2b2datasource/models/explore_query.go diff --git a/pkg/i2b2datasource/models/explore_query.go b/pkg/i2b2datasource/models/explore_query.go new file mode 100644 index 0000000..5db1328 --- /dev/null +++ b/pkg/i2b2datasource/models/explore_query.go @@ -0,0 +1,77 @@ +package models + +import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + +// --- parameters + +type ExploreQueryParameters struct { + Id string + Definition ExploreQueryDefinition +} + +type ExploreQueryDefinition struct { + Timing string // any | samevisit | sameinstancenum + Panels []Panel +} + +type Panel struct { + Not bool + Timing string // any | samevisit | sameinstancenum + CohortItems []string + ConceptItems []ConceptItem +} + +type ConceptItem struct { + QueryTerm string + Operator string // EQ | NE | GT | GE | LT | LE | BETWEEN | IN | LIKE[exact] | LIKE[begin] | LIKE[end] | LIKE[contains] + Value string + Type string // NUMBER | TEXT + Modifier struct { + Key string + AppliedPath string + } +} + +func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2apimodels.Panel, i2b2ApiTiming i2b2apimodels.Timing) { + i2b2ApiTiming = i2b2apimodels.Timing(d.Timing) + + for panelIdx, panel := range d.Panels { + var i2b2ApiItems []i2b2apimodels.Item + + for _, item := range panel.ConceptItems { + i2b2ApiItem := i2b2apimodels.Item{ItemKey: i2b2apimodels.ConvertPathToI2b2Format(item.QueryTerm)} + + if item.Operator != "" && item.Modifier.Key == "" { + i2b2ApiItem.ConstrainByValue = &i2b2apimodels.ConstrainByValue{ + ValueType: item.Type, + ValueOperator: item.Operator, + ValueConstraint: item.Value, + } + } + + if item.Modifier.Key != "" { + i2b2ApiItem.ConstrainByModifier = &i2b2apimodels.ConstrainByModifier{ + ModifierKey: i2b2apimodels.ConvertPathToI2b2Format(item.Modifier.Key), + AppliedPath: i2b2apimodels.ConvertAppliedPathToI2b2Format(item.Modifier.AppliedPath), + } + if item.Operator != "" { + i2b2ApiItem.ConstrainByModifier.ConstrainByValue = &i2b2apimodels.ConstrainByValue{ + ValueType: item.Type, + ValueOperator: item.Operator, + ValueConstraint: item.Value, + } + } + } + i2b2ApiItems = append(i2b2ApiItems, i2b2ApiItem) + } + + for _, cohort := range panel.CohortItems { + i2b2ApiItems = append(i2b2ApiItems, i2b2apimodels.Item{ItemKey: cohort}) + } + + i2b2ApiPanels = append(i2b2ApiPanels, + i2b2apimodels.NewPanel(panelIdx, panel.Not, i2b2apimodels.Timing(panel.Timing), i2b2ApiItems), + ) + } + return +} From 72d2492466331405685266dbb08216e0deae167c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:23:02 +0100 Subject: [PATCH 27/81] change conversion of i2b2 path --- pkg/i2b2datasource/search.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go index 94d06f4..aec1160 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/i2b2datasource/search.go @@ -140,7 +140,7 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } else { - i2b2FormatAppliedPath := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedPath))[1:] + i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) req := i2b2apimodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) if resp, err = ds.i2b2Client.OntGetModifierInfo(&req); err != nil { return nil, fmt.Errorf("requesting info of modifier: %v", err) @@ -152,7 +152,7 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } else { - i2b2FormatAppliedPath := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedPath))[1:] + i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) i2b2FormatAppliedConcept := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) req := i2b2apimodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) if resp, err = ds.i2b2Client.OntGetModifierChildren(&req); err != nil { From 0b99f41dc7a630b742c7e74be5ef7543e85b3a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:23:35 +0100 Subject: [PATCH 28/81] add explore query implementation --- pkg/i2b2datasource/explore_query.go | 97 ++++++++ pkg/i2b2datasource/explore_query_test.go | 285 +++++++++++++++++++++++ 2 files changed, 382 insertions(+) create mode 100644 pkg/i2b2datasource/explore_query.go create mode 100644 pkg/i2b2datasource/explore_query_test.go diff --git a/pkg/i2b2datasource/explore_query.go b/pkg/i2b2datasource/explore_query.go new file mode 100644 index 0000000..1bcd999 --- /dev/null +++ b/pkg/i2b2datasource/explore_query.go @@ -0,0 +1,97 @@ +package i2b2datasource + +import ( + "fmt" + "strconv" + + i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" +) + +// ExploreQuery makes an explore query, i.e. two i2b2 CRC queries, a PSM and a PDO query. +func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (patientCount uint64, patientList []uint64, err error) { + + if i2b2PatientCount, i2b2PatientSetID, err := ds.doExploreQuery(params); err != nil { + return 0, nil, err + } else if i2b2PatientIDs, err := ds.getPatientIDs(i2b2PatientSetID); err != nil { + return 0, nil, err + } else if patientCount, err = strconv.ParseUint(i2b2PatientCount, 10, 64); err != nil { + return 0, nil, fmt.Errorf("parsing patient count: %v", err) + } else { + for _, patientID := range i2b2PatientIDs { + if parsedPatientID, err := strconv.ParseUint(patientID, 10, 64); err != nil { + return 0, nil, fmt.Errorf("parsing patient ID: %v", err) + } else { + patientList = append(patientList, parsedPatientID) + } + } + } + + return +} + +// doExploreQuery requests the explore query to the i2b2 CRC and parse its results. +func (ds I2b2DataSource) doExploreQuery(params *models.ExploreQueryParameters) (patientCount, patientSetID string, err error) { + + // build query + panels, timing := params.Definition.ToI2b2APIModel() + psmReq := i2b2apimodels.NewCrcPsmReqFromQueryDef( + ds.i2b2Client.Ci, + params.Id, + panels, + timing, + []i2b2apimodels.ResultOutputName{i2b2apimodels.ResultOutputPatientSet, i2b2apimodels.ResultOutputCount}, + ) + + // request query + var psmResp *i2b2apimodels.CrcPsmRespMessageBody + if psmResp, err = ds.i2b2Client.CrcPsmReqFromQueryDef(&psmReq); err != nil { + return "", "", fmt.Errorf("requesting PSM query: %v", err) + } + + // extract results from result instances + for i, qri := range psmResp.Response.QueryResultInstances { + + // check status + if err := qri.CheckStatus(); err != nil { + return "", "", fmt.Errorf("found error in query result instance %v: %v", i, err) + } + + // extract result + switch qri.QueryResultType.Name { + case string(i2b2apimodels.ResultOutputPatientSet): + if patientSetID != "" { + ds.logger.Warnf("unexpected additional patient set result in i2b2 CRC PSM response (previous: %v)", patientSetID) + } + patientSetID = qri.ResultInstanceID + + case string(i2b2apimodels.ResultOutputCount): + if patientCount != "" { + ds.logger.Warnf("unexpected additional patient count result in i2b2 CRC PSM response (previous: %v)", patientCount) + } + patientCount = qri.SetSize + + default: + ds.logger.Warnf("unexpected result in i2b2 CRC PSM response: %v", qri.QueryResultType.Name) + } + } + + if patientCount == "" || patientSetID == "" { + return "", "", fmt.Errorf("missing result from i2b2 PSM response: patientCount=%v, patientSetID=%v", patientCount, patientSetID) + } + return +} + +// getPatientIDs retrieves the list of patient IDs from the i2b2 CRC using a patient set ID. +func (ds I2b2DataSource) getPatientIDs(patientSetID string) (patientIDs []string, err error) { + pdoReq := i2b2apimodels.NewCrcPdoReqFromInputList(patientSetID) + var pdoResp *i2b2apimodels.CrcPdoRespMessageBody + if pdoResp, err = ds.i2b2Client.CrcPdoReqFromInputList(&pdoReq); err != nil { + return nil, fmt.Errorf("requesting PDO query: %v", err) + } + + for _, patient := range pdoResp.Response.PatientData.PatientSet.Patient { + patientIDs = append(patientIDs, patient.PatientID) + } + return +} diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/i2b2datasource/explore_query_test.go new file mode 100644 index 0000000..354642d --- /dev/null +++ b/pkg/i2b2datasource/explore_query_test.go @@ -0,0 +1,285 @@ +package i2b2datasource + +import ( + "testing" + + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/stretchr/testify/require" +) + +func TestExploreQueryConcept(t *testing.T) { + ds := GetTestDataSource(t) + + count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "0", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 3, count) + require.Subset(t, []uint64{1, 2, 3}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/3/", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 4, count) + require.Subset(t, []uint64{1, 2, 3, 4}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + }}, + },{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/2/", + }}, + },{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/3/", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 3, count) + require.Subset(t, []uint64{1, 2, 3}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + },{ + QueryTerm: "/TEST/test/3/", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 4, count) + require.Subset(t, []uint64{1, 2, 3, 4}, patientList) +} + +func TestExploreQueryConceptValue(t *testing.T) { + ds := GetTestDataSource(t) + + count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "0", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + Operator: "EQ", + Value: "10", + Type: "NUMBER", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 1, count) + require.Subset(t, []uint64{1}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + Operator: "EQ", + Value: "10", + Type: "NUMBER", + },{ + QueryTerm: "/TEST/test/3/", + Operator: "EQ", + Value: "20", + Type: "NUMBER", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 2, count) + require.Subset(t, []uint64{1, 4}, patientList) + + // todo: maybe it's because of modifier!!! + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/3/", + Operator: "LIKE[contains]", + Value: "c", + Type: "TEXT", + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 2, count) + require.Subset(t, []uint64{1, 4}, patientList) +} + +func TestExploreQueryModifier(t *testing.T) { + ds := GetTestDataSource(t) + + count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "0", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/1/", + AppliedPath: "/test/1/", + }, + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 2, count) + require.Subset(t, []uint64{1, 3}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/1/", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/1/", + AppliedPath: "/test/1/", + }, + },{ + QueryTerm: "/TEST/test/2/", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/2text/", + AppliedPath: "/test/2/", + }, + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 3, count) + require.Subset(t, []uint64{1, 2, 3}, patientList) +} + +func TestExploreQueryModifierValue(t *testing.T) { + ds := GetTestDataSource(t) + + count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "0", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/2/", + Operator: "LIKE[contains]", + Value: "cd", + Type: "TEXT", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/2text/", + AppliedPath: "/test/2/", + }, + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 1, count) + require.Subset(t, []uint64{1}, patientList) + + count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + Id: "1", + Definition: models.ExploreQueryDefinition{ + Panels: []models.Panel{{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/3/", + Operator: "LIKE[exact]", + Value: "def", + Type: "TEXT", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/3text/", + AppliedPath: "/test/3/", + }, + }}, + },{ + Not: false, + ConceptItems: []models.ConceptItem{{ + QueryTerm: "/TEST/test/2/", + Operator: "LIKE[begin]", + Value: "a", + Type: "TEXT", + Modifier: struct { + Key string + AppliedPath string + }{ + Key: "/TEST/modifiers/2text/", + AppliedPath: "/test/2/", + }, + }}, + }}, + }, + }) + require.NoError(t, err) + t.Logf("results: count=%v, patientList=%v", count, patientList) + require.EqualValues(t, 1, count) + require.Subset(t, []uint64{2}, patientList) +} From 54e81e8e74145f3d673890fcf8a0014ba4a15ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:24:39 +0100 Subject: [PATCH 29/81] add data object functions; update to data source for explore query --- pkg/i2b2datasource/dataobjects.go | 41 ++++++++++++++++++++++++++++ pkg/i2b2datasource/datasource.go | 45 +++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 pkg/i2b2datasource/dataobjects.go diff --git a/pkg/i2b2datasource/dataobjects.go b/pkg/i2b2datasource/dataobjects.go new file mode 100644 index 0000000..75d1fc8 --- /dev/null +++ b/pkg/i2b2datasource/dataobjects.go @@ -0,0 +1,41 @@ +package i2b2datasource + +import ( + "fmt" + + "github.com/ldsec/geco/pkg/datamanager" + gecomodels "github.com/ldsec/geco/pkg/models" +) + +// todo: geco data manager needs to allow data objects that are int and int vectors, it is now temporarily stored as float! + +// storeIntValue stores an integer value in the data manager. +func (ds I2b2DataSource) storeIntValue(value uint64, doSharedID string) error { + + do := datamanager.NewFloatVector([]float64{float64(value)}) // todo: need to be different type of data object + if doId, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false); err != nil { + return fmt.Errorf("adding data object: %v", err) + } else { + ds.logger.Infof("added data object to data manager (%v)", doId) + } + + return nil +} + +// storeIntVector stores a vector of integers in the data manager. +func (ds I2b2DataSource) storeIntVector(values []uint64, doSharedID string) error { + + var floats []float64 + for _, value := range values { + floats = append(floats, float64(value)) + } + + do := datamanager.NewFloatVector(floats) // todo: need to be different type of data object + if doId, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false); err != nil { + return fmt.Errorf("adding data object: %v", err) + } else { + ds.logger.Infof("added data object to data manager (%v)", doId) + } + + return nil +} diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go index e1cacbd..2ead801 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/i2b2datasource/datasource.go @@ -16,14 +16,19 @@ import ( type queryType string const ( - searchConceptQueryType queryType = "searchConcept" - searchModifierQueryType queryType = "searchModifier" - exploreQueryQueryType queryType = "exploreQuery" - getCohortsQueryType queryType = "getCohorts" - addCohortQueryType queryType = "addCohort" - deleteCohortQueryType queryType = "deleteCohort" - survivalQueryQueryType queryType = "survivalQuery" - searchOntologyQueryType queryType = "searchOntology" + queryTypeSearchConcept queryType = "searchConcept" + queryTypeSearchModifier queryType = "searchModifier" + queryTypeExploreQuery queryType = "exploreQuery" + queryTypeGetCohorts queryType = "getCohorts" + queryTypeAddCohort queryType = "addCohort" + queryTypeDeleteCohort queryType = "deleteCohort" + queryTypeSurvivalQuery queryType = "survivalQuery" + queryTypeSearchOntology queryType = "searchOntology" +) + +const ( + sharedIdExploreQueryCount string = "count" + sharedIdExploreQueryPatientList string = "patientList" ) type I2b2DataSource struct { @@ -106,7 +111,7 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s results = make(map[string]interface{}) switch queryType(operation) { - case searchConceptQueryType: + case queryTypeSearchConcept: decodedParams := &models.SearchConceptParameters{} if err := mapstructure.Decode(parameters, decodedParams); err != nil { return nil, ds.logError("decoding parameters", err) @@ -116,7 +121,7 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s return nil, ds.logError("encoding results", err) } - case searchModifierQueryType: + case queryTypeSearchModifier: decodedParams := &models.SearchModifierParameters{} if err := mapstructure.Decode(parameters, decodedParams); err != nil { return nil, ds.logError("decoding parameters", err) @@ -126,8 +131,26 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s return nil, ds.logError("encoding results", err) } + case queryTypeExploreQuery: + countSharedID, countOK := resultsSharedIds[sharedIdExploreQueryCount] + patientListSharedID, patientListOK := resultsSharedIds[sharedIdExploreQueryPatientList] + if !countOK || !patientListOK { + return nil, ds.logError("missing results shared ID", nil) + } + + decodedParams := &models.ExploreQueryParameters{} + if err := mapstructure.Decode(parameters, decodedParams); err != nil { + return nil, ds.logError("decoding parameters", err) + } else if count, patientList, err := ds.ExploreQuery(decodedParams); err != nil { + return nil, ds.logError("executing query", err) + } else if err := ds.storeIntValue(count, countSharedID); err != nil { + return nil, ds.logError("storing count", err) + } else if err := ds.storeIntVector(patientList, patientListSharedID); err != nil { + return nil, ds.logError("storing patient list", err) + } + default: - return + return nil, ds.logError(fmt.Sprintf("unknown query requested (%v)", operation), nil) } return } From 984394ebb37718373415504e3ee3bbde67546c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 9 Dec 2021 18:24:53 +0100 Subject: [PATCH 30/81] cosmetic --- build/package/i2b2/sql/90-test-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/i2b2/sql/90-test-data.sh b/build/package/i2b2/sql/90-test-data.sh index fae27f7..10d932a 100644 --- a/build/package/i2b2/sql/90-test-data.sh +++ b/build/package/i2b2/sql/90-test-data.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail # set up data in the database for end-to-end tests ### description of the test data -# 4 patients: 1 (real), 2 (real), 3 (real), 4 (dummy) +# 4 patients: 1, 2, 3, 4 # 3 concepts: 1, 2, 3 # observation_fact: p1: c1; p2: c1, c2; p3: c2, c3; p4: c1, c2, c3 # the same data is replicated on all nodes From e451dbcd5e9a6bb9257273a964f4e247c1a37834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 07:24:54 +0100 Subject: [PATCH 31/81] data source test cleanup --- pkg/i2b2datasource/datasource_test.go | 4 +-- pkg/i2b2datasource/explore_query_test.go | 32 +++++------------------- pkg/i2b2datasource/search_test.go | 14 +++++------ 3 files changed, 15 insertions(+), 35 deletions(-) diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go index 6a33a3d..c9c91e5 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/i2b2datasource/datasource_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -func GetTestDataSource(t *testing.T) I2b2DataSource { +func getTestDataSource(t *testing.T) I2b2DataSource { ds := I2b2DataSource{} dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) @@ -32,7 +32,7 @@ func GetTestDataSource(t *testing.T) I2b2DataSource { func TestDataManager(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) doId, err := ds.dm.AddDataObject(datamanager.NewFloatVector([]float64{0, 1, 2, 3, 4}), false) require.NoError(t, err) diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/i2b2datasource/explore_query_test.go index 354642d..168b23d 100644 --- a/pkg/i2b2datasource/explore_query_test.go +++ b/pkg/i2b2datasource/explore_query_test.go @@ -8,7 +8,7 @@ import ( ) func TestExploreQueryConcept(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ Id: "0", @@ -43,7 +43,7 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3, 4}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "2", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -69,7 +69,7 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "3", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -88,7 +88,7 @@ func TestExploreQueryConcept(t *testing.T) { } func TestExploreQueryConceptValue(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ Id: "0", @@ -132,30 +132,10 @@ func TestExploreQueryConceptValue(t *testing.T) { t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 2, count) require.Subset(t, []uint64{1, 4}, patientList) - - // todo: maybe it's because of modifier!!! - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", - Definition: models.ExploreQueryDefinition{ - Panels: []models.Panel{{ - Not: false, - ConceptItems: []models.ConceptItem{{ - QueryTerm: "/TEST/test/3/", - Operator: "LIKE[contains]", - Value: "c", - Type: "TEXT", - }}, - }}, - }, - }) - require.NoError(t, err) - t.Logf("results: count=%v, patientList=%v", count, patientList) - require.EqualValues(t, 2, count) - require.Subset(t, []uint64{1, 4}, patientList) } func TestExploreQueryModifier(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ Id: "0", @@ -214,7 +194,7 @@ func TestExploreQueryModifier(t *testing.T) { } func TestExploreQueryModifierValue(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ Id: "0", diff --git a/pkg/i2b2datasource/search_test.go b/pkg/i2b2datasource/search_test.go index 66eede2..fea4c78 100644 --- a/pkg/i2b2datasource/search_test.go +++ b/pkg/i2b2datasource/search_test.go @@ -8,7 +8,7 @@ import ( ) func TestSearchConceptInfo(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "/", @@ -37,7 +37,7 @@ func TestSearchConceptInfo(t *testing.T) { } func TestSearchConceptChildren(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "/", @@ -67,7 +67,7 @@ func TestSearchConceptChildren(t *testing.T) { } func TestSearchConceptError(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) _, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "", @@ -92,7 +92,7 @@ func TestSearchConceptError(t *testing.T) { } func TestSearchModifierConcept(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -123,7 +123,7 @@ func TestSearchModifierConcept(t *testing.T) { } func TestSearchModifierInfo(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -151,7 +151,7 @@ func TestSearchModifierInfo(t *testing.T) { } func TestSearchModifierChildren(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -192,7 +192,7 @@ func TestSearchModifierChildren(t *testing.T) { } func TestSearchModifierError(t *testing.T) { - ds := GetTestDataSource(t) + ds := getTestDataSource(t) _, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ From 79ba4ae7c2caa5ae3229f88d9469fa2670a27786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 07:25:57 +0100 Subject: [PATCH 32/81] data source search model modification --- pkg/i2b2datasource/models/search.go | 12 +++++++----- pkg/i2b2datasource/search.go | 8 ++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/i2b2datasource/models/search.go b/pkg/i2b2datasource/models/search.go index 2c3410a..0f291e9 100644 --- a/pkg/i2b2datasource/models/search.go +++ b/pkg/i2b2datasource/models/search.go @@ -54,9 +54,11 @@ type SearchResult struct { } type SearchResultMetadata struct { - DataType string // PosInteger | Integer | Float | PosFloat | Enum | String - Oktousevalues string // Y - UnitValues struct { - NormalUnits string - } + DataType string // PosInteger | Integer | Float | PosFloat | Enum | String + OkToUseValues string // Y + UnitValues MetadataUnitValues +} + +type MetadataUnitValues struct { + NormalUnits string } diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go index aec1160..c77e3b3 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/i2b2datasource/search.go @@ -74,12 +74,8 @@ func parseI2b2Concept(concept i2b2apimodels.Concept) models.SearchResult { if concept.Metadataxml != nil { parsed.Metadata = models.SearchResultMetadata{ DataType: concept.Metadataxml.DataType, - Oktousevalues: concept.Metadataxml.Oktousevalues, - UnitValues: struct { - NormalUnits string - }{ - NormalUnits: concept.Metadataxml.UnitValues.NormalUnits, - }, + OkToUseValues: concept.Metadataxml.Oktousevalues, + UnitValues: models.MetadataUnitValues{NormalUnits: concept.Metadataxml.UnitValues.NormalUnits}, } } From 46be9ab1e8af031ab96cdbb0b92fc0584e10f577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 07:37:37 +0100 Subject: [PATCH 33/81] go fmt --- pkg/geco-data-source.go | 1 - pkg/i2b2api/crc_client.go | 1 - pkg/i2b2api/models/common.go | 1 - pkg/i2b2api/models/crc_psm.go | 12 ++-- pkg/i2b2api/models/ont.go | 70 +++++++++++----------- pkg/i2b2datasource/datasource.go | 13 ++-- pkg/i2b2datasource/explore_query_test.go | 58 +++++++++--------- pkg/i2b2datasource/models/explore_query.go | 2 +- pkg/i2b2datasource/models/search.go | 20 ++++--- pkg/i2b2datasource/search.go | 2 +- pkg/i2b2datasource/search_test.go | 50 ++++++++-------- 11 files changed, 115 insertions(+), 115 deletions(-) diff --git a/pkg/geco-data-source.go b/pkg/geco-data-source.go index 3b44eff..38cde48 100644 --- a/pkg/geco-data-source.go +++ b/pkg/geco-data-source.go @@ -15,6 +15,5 @@ type DataSource interface { Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) error // Query data source with a specific operation. - // todo: parameters and results are json marshallable -> some other interface instead? check what swagger provides or not Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) } diff --git a/pkg/i2b2api/crc_client.go b/pkg/i2b2api/crc_client.go index 2854761..bbb5ea2 100644 --- a/pkg/i2b2api/crc_client.go +++ b/pkg/i2b2api/crc_client.go @@ -43,4 +43,3 @@ func (c Client) CrcPdoReqFromInputList(reqMsgBody *models.CrcPdoReqFromInputList return mb, nil } } - diff --git a/pkg/i2b2api/models/common.go b/pkg/i2b2api/models/common.go index 9c31492..638348f 100644 --- a/pkg/i2b2api/models/common.go +++ b/pkg/i2b2api/models/common.go @@ -7,7 +7,6 @@ import ( "time" ) - // MessageBody is an i2b2 XML generic body type MessageBody interface{} diff --git a/pkg/i2b2api/models/crc_psm.go b/pkg/i2b2api/models/crc_psm.go index 1888d35..856427d 100644 --- a/pkg/i2b2api/models/crc_psm.go +++ b/pkg/i2b2api/models/crc_psm.go @@ -34,7 +34,7 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ QueryDescription: "Query from GeCo i2b2 data source (" + queryName + ")", QueryTiming: string(queryTiming), SpecificityScale: "0", - Panels: queryPanels, + Panels: queryPanels, } // embed result outputs @@ -98,7 +98,7 @@ func NewPanel(panelNb int, not bool, timing Timing, items []Item) Panel { PanelNumber: strconv.Itoa(panelNb + 1), PanelAccuracyScale: "100", Invert: invert, - Items: items, + Items: items, PanelTiming: string(timing), TotalItemOccurrences: "1", } @@ -150,6 +150,7 @@ type ResultOutput struct { // ResultOutputName is an i2b2 XML requested result type value type ResultOutputName string + const ( ResultOutputPatientSet ResultOutputName = "PATIENTSET" ResultOutputEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" @@ -161,10 +162,11 @@ const ( ) type Timing string + const ( - TimingAny Timing = "ANY" - TimingSameVisit Timing = "SAMEVISIT" - TimingSameInstanceNum Timing = "SAMEINSTANCENUM" + TimingAny Timing = "ANY" + TimingSameVisit Timing = "SAMEVISIT" + TimingSameInstanceNum Timing = "SAMEINSTANCENUM" ) // --- response diff --git a/pkg/i2b2api/models/ont.go b/pkg/i2b2api/models/ont.go index f0206fe..8cbd8a9 100644 --- a/pkg/i2b2api/models/ont.go +++ b/pkg/i2b2api/models/ont.go @@ -101,7 +101,7 @@ type OntReqGetTermInfoMessageBody struct { XMLName xml.Name `xml:"message_body"` GetTermInfo struct { baseMessageBody - Self string `xml:"self"` + Self string `xml:"self"` } `xml:"ontns:get_term_info"` } @@ -167,27 +167,27 @@ type OntRespModifiersMessageBody struct { // Concept is an i2b2 XML concept type Concept struct { - Level string `xml:"level"` - Key string `xml:"key"` - Name string `xml:"name"` - SynonymCd string `xml:"synonym_cd"` - Visualattributes string `xml:"visualattributes"` - Totalnum string `xml:"totalnum"` - Basecode string `xml:"basecode"` - Metadataxml *MetadataXML `xml:"metadataxml"` - Facttablecolumn string `xml:"facttablecolumn"` - Tablename string `xml:"tablename"` - Columnname string `xml:"columnname"` - Columndatatype string `xml:"columndatatype"` - Operator string `xml:"operator"` - Dimcode string `xml:"dimcode"` - Comment string `xml:"comment"` - Tooltip string `xml:"tooltip"` - UpdateDate string `xml:"update_date"` - DownloadDate string `xml:"download_date"` - ImportDate string `xml:"import_date"` - SourcesystemCd string `xml:"sourcesystem_cd"` - ValuetypeCd string `xml:"valuetype_cd"` + Level string `xml:"level"` + Key string `xml:"key"` + Name string `xml:"name"` + SynonymCd string `xml:"synonym_cd"` + Visualattributes string `xml:"visualattributes"` + Totalnum string `xml:"totalnum"` + Basecode string `xml:"basecode"` + Metadataxml *MetadataXML `xml:"metadataxml"` + Facttablecolumn string `xml:"facttablecolumn"` + Tablename string `xml:"tablename"` + Columnname string `xml:"columnname"` + Columndatatype string `xml:"columndatatype"` + Operator string `xml:"operator"` + Dimcode string `xml:"dimcode"` + Comment string `xml:"comment"` + Tooltip string `xml:"tooltip"` + UpdateDate string `xml:"update_date"` + DownloadDate string `xml:"download_date"` + ImportDate string `xml:"import_date"` + SourcesystemCd string `xml:"sourcesystem_cd"` + ValuetypeCd string `xml:"valuetype_cd"` } // Modifier is an i2b2 XML modifier. @@ -197,25 +197,25 @@ type Modifier struct { } type MetadataXML struct { - CreationDateTime string `xml:"ValueMetadata>CreationDateTime"` - DataType string `xml:"ValueMetadata>DataType"` - EnumValues string `xml:"ValueMetadata>EnumValues"` - Flagstouse string `xml:"ValueMetadata>Flagstouse"` - Oktousevalues string `xml:"ValueMetadata>Oktousevalues"` - TestID string `xml:"ValueMetadata>TestID"` - TestName string `xml:"ValueMetadata>TestName"` - UnitValues ValueMetadataUnitValues `xml:"ValueMetadata>UnitValues"` - Version string `xml:"ValueMetadata>Version"` + CreationDateTime string `xml:"ValueMetadata>CreationDateTime"` + DataType string `xml:"ValueMetadata>DataType"` + EnumValues string `xml:"ValueMetadata>EnumValues"` + Flagstouse string `xml:"ValueMetadata>Flagstouse"` + Oktousevalues string `xml:"ValueMetadata>Oktousevalues"` + TestID string `xml:"ValueMetadata>TestID"` + TestName string `xml:"ValueMetadata>TestName"` + UnitValues ValueMetadataUnitValues `xml:"ValueMetadata>UnitValues"` + Version string `xml:"ValueMetadata>Version"` } type ValueMetadataUnitValues struct { ConvertingUnits []UnitValuesConvertingUnits `xml:"ConvertingUnits"` - EqualUnits []string `xml:"EqualUnits"` - ExcludingUnits []string `xml:"ExcludingUnits"` - NormalUnits string `xml:"NormalUnits"` + EqualUnits []string `xml:"EqualUnits"` + ExcludingUnits []string `xml:"ExcludingUnits"` + NormalUnits string `xml:"NormalUnits"` } type UnitValuesConvertingUnits struct { MultiplyingFactor string `xml:"MultiplyingFactor"` - Units string `xml:"Units"` + Units string `xml:"Units"` } diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go index 2ead801..0b5bac4 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/i2b2datasource/datasource.go @@ -15,19 +15,20 @@ import ( // todo: linting, remove from models (both packages) type queryType string + const ( queryTypeSearchConcept queryType = "searchConcept" queryTypeSearchModifier queryType = "searchModifier" queryTypeExploreQuery queryType = "exploreQuery" queryTypeGetCohorts queryType = "getCohorts" - queryTypeAddCohort queryType = "addCohort" - queryTypeDeleteCohort queryType = "deleteCohort" - queryTypeSurvivalQuery queryType = "survivalQuery" + queryTypeAddCohort queryType = "addCohort" + queryTypeDeleteCohort queryType = "deleteCohort" + queryTypeSurvivalQuery queryType = "survivalQuery" queryTypeSearchOntology queryType = "searchOntology" ) const ( - sharedIdExploreQueryCount string = "count" + sharedIdExploreQueryCount string = "count" sharedIdExploreQueryPatientList string = "patientList" ) @@ -49,7 +50,6 @@ type I2b2DataSource struct { i2b2OntMaxElements string } - func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) (err error) { fmt.Println("called init") @@ -61,7 +61,6 @@ func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldL // i2b2.db.xxx // datasource.db.xxx - // parse i2b2 API connection info and initialize i2b2 client ci := i2b2apimodels.ConnectionInfo{ HiveURL: config["i2b2.api.url"], @@ -79,7 +78,7 @@ func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldL ds.i2b2Client = i2b2api.Client{ Logger: logger, - Ci:ci, + Ci: ci, } ds.i2b2OntMaxElements = config["i2b2.api.ont-max-elements"] diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/i2b2datasource/explore_query_test.go index 168b23d..4c97961 100644 --- a/pkg/i2b2datasource/explore_query_test.go +++ b/pkg/i2b2datasource/explore_query_test.go @@ -11,10 +11,10 @@ func TestExploreQueryConcept(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + Id: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", }}, @@ -27,10 +27,10 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/3/", }}, @@ -43,20 +43,20 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3, 4}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "2", + Id: "2", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", }}, - },{ - Not: false, + }, { + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/2/", }}, - },{ - Not: false, + }, { + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/3/", }}, @@ -69,13 +69,13 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "3", + Id: "3", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", - },{ + }, { QueryTerm: "/TEST/test/3/", }}, }}, @@ -91,10 +91,10 @@ func TestExploreQueryConceptValue(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + Id: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", Operator: "EQ", @@ -110,16 +110,16 @@ func TestExploreQueryConceptValue(t *testing.T) { require.Subset(t, []uint64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", Operator: "EQ", Value: "10", Type: "NUMBER", - },{ + }, { QueryTerm: "/TEST/test/3/", Operator: "EQ", Value: "20", @@ -138,10 +138,10 @@ func TestExploreQueryModifier(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + Id: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", Modifier: struct { @@ -161,10 +161,10 @@ func TestExploreQueryModifier(t *testing.T) { require.Subset(t, []uint64{1, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/1/", Modifier: struct { @@ -174,7 +174,7 @@ func TestExploreQueryModifier(t *testing.T) { Key: "/TEST/modifiers/1/", AppliedPath: "/test/1/", }, - },{ + }, { QueryTerm: "/TEST/test/2/", Modifier: struct { Key string @@ -197,10 +197,10 @@ func TestExploreQueryModifierValue(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + Id: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/2/", Operator: "LIKE[contains]", @@ -223,10 +223,10 @@ func TestExploreQueryModifierValue(t *testing.T) { require.Subset(t, []uint64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + Id: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ - Not: false, + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/3/", Operator: "LIKE[exact]", @@ -240,8 +240,8 @@ func TestExploreQueryModifierValue(t *testing.T) { AppliedPath: "/test/3/", }, }}, - },{ - Not: false, + }, { + Not: false, ConceptItems: []models.ConceptItem{{ QueryTerm: "/TEST/test/2/", Operator: "LIKE[begin]", diff --git a/pkg/i2b2datasource/models/explore_query.go b/pkg/i2b2datasource/models/explore_query.go index 5db1328..a9307e1 100644 --- a/pkg/i2b2datasource/models/explore_query.go +++ b/pkg/i2b2datasource/models/explore_query.go @@ -5,7 +5,7 @@ import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" // --- parameters type ExploreQueryParameters struct { - Id string + Id string Definition ExploreQueryDefinition } diff --git a/pkg/i2b2datasource/models/search.go b/pkg/i2b2datasource/models/search.go index 0f291e9..70b374f 100644 --- a/pkg/i2b2datasource/models/search.go +++ b/pkg/i2b2datasource/models/search.go @@ -12,7 +12,7 @@ package models // } type SearchConceptParameters struct { - Path string + Path string Operation string // children | info | concept } @@ -29,9 +29,11 @@ type SearchConceptParameters struct { // }, nil // } +// todo: maybe squash model together if OK with serializing etc. + type SearchModifierParameters struct { SearchConceptParameters - AppliedPath string + AppliedPath string AppliedConcept string } @@ -42,15 +44,15 @@ type SearchResults struct { } type SearchResult struct { - Path string + Path string AppliedPath string - Name string + Name string DisplayName string - Code string - Comment string - Type string // concept | concept_container | concept_folder | modifier | modifier_container | modifier_folder | genomic_annotation - Leaf bool - Metadata SearchResultMetadata + Code string + Comment string + Type string // concept | concept_container | concept_folder | modifier | modifier_container | modifier_folder | genomic_annotation + Leaf bool + Metadata SearchResultMetadata } type SearchResultMetadata struct { diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go index c77e3b3..d42894e 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/i2b2datasource/search.go @@ -75,7 +75,7 @@ func parseI2b2Concept(concept i2b2apimodels.Concept) models.SearchResult { parsed.Metadata = models.SearchResultMetadata{ DataType: concept.Metadataxml.DataType, OkToUseValues: concept.Metadataxml.Oktousevalues, - UnitValues: models.MetadataUnitValues{NormalUnits: concept.Metadataxml.UnitValues.NormalUnits}, + UnitValues: models.MetadataUnitValues{NormalUnits: concept.Metadataxml.UnitValues.NormalUnits}, } } diff --git a/pkg/i2b2datasource/search_test.go b/pkg/i2b2datasource/search_test.go index fea4c78..b909385 100644 --- a/pkg/i2b2datasource/search_test.go +++ b/pkg/i2b2datasource/search_test.go @@ -99,12 +99,12 @@ func TestSearchModifierConcept(t *testing.T) { Path: "/TEST/test/1/", Operation: "concept", }, - AppliedPath: "", - AppliedConcept: "", + AppliedPath: "", + AppliedConcept: "", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 1, len(searchResults.SearchResults)) + require.Equal(t, 1, len(searchResults.SearchResults)) require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/") searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ @@ -112,12 +112,12 @@ func TestSearchModifierConcept(t *testing.T) { Path: "/TEST/test/2/", Operation: "concept", }, - AppliedPath: "", - AppliedConcept: "", + AppliedPath: "", + AppliedConcept: "", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 2, len(searchResults.SearchResults)) + require.Equal(t, 2, len(searchResults.SearchResults)) require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/2") require.Contains(t, searchResults.SearchResults[1].Path, "/TEST/modifiers/2") } @@ -130,11 +130,11 @@ func TestSearchModifierInfo(t *testing.T) { Path: "/TEST/modifiers/", Operation: "info", }, - AppliedPath: "/test/%", + AppliedPath: "/test/%", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 1, len(searchResults.SearchResults)) + require.Equal(t, 1, len(searchResults.SearchResults)) require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/") searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ @@ -142,11 +142,11 @@ func TestSearchModifierInfo(t *testing.T) { Path: "/TEST/modifiers/1", Operation: "info", }, - AppliedPath: "/test/%", + AppliedPath: "/test/%", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 1, len(searchResults.SearchResults)) + require.Equal(t, 1, len(searchResults.SearchResults)) require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/1") } @@ -158,12 +158,12 @@ func TestSearchModifierChildren(t *testing.T) { Path: "/TEST/modifiers/", Operation: "children", }, - AppliedPath: "/test/%", - AppliedConcept: "/TEST/test/1/", + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/1/", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 1, len(searchResults.SearchResults)) + require.Equal(t, 1, len(searchResults.SearchResults)) require.Contains(t, searchResults.SearchResults[0].Path, "/TEST/modifiers/1/") searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ @@ -171,24 +171,24 @@ func TestSearchModifierChildren(t *testing.T) { Path: "/TEST/modifiers/", Operation: "children", }, - AppliedPath: "/test/%", - AppliedConcept: "/TEST/test/2/", + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/2/", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 2, len(searchResults.SearchResults)) + require.Equal(t, 2, len(searchResults.SearchResults)) searchResults, err = ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ Path: "/TEST/modifiers/", Operation: "children", }, - AppliedPath: "/test/%", - AppliedConcept: "/TEST/test/3/", + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", }) require.NoError(t, err) t.Logf("%+v", searchResults.SearchResults) - require.Equal(t, 2, len(searchResults.SearchResults)) + require.Equal(t, 2, len(searchResults.SearchResults)) } func TestSearchModifierError(t *testing.T) { @@ -199,8 +199,8 @@ func TestSearchModifierError(t *testing.T) { Path: "", Operation: "children", }, - AppliedPath: "/test/%", - AppliedConcept: "/TEST/test/3/", + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", }) require.Error(t, err) require.Contains(t, err.Error(), "empty path") @@ -210,8 +210,8 @@ func TestSearchModifierError(t *testing.T) { Path: "/TEST/modifiers/", Operation: "xxxxx", }, - AppliedPath: "/test/%", - AppliedConcept: "/TEST/test/3/", + AppliedPath: "/test/%", + AppliedConcept: "/TEST/test/3/", }) require.Error(t, err) require.Contains(t, err.Error(), "invalid search operation") @@ -221,8 +221,8 @@ func TestSearchModifierError(t *testing.T) { Path: "/TEST/modifiers/", Operation: "children", }, - AppliedPath: "/test", - AppliedConcept: "/TEST/test/3/", + AppliedPath: "/test", + AppliedConcept: "/TEST/test/3/", }) require.Error(t, err) require.Contains(t, err.Error(), "error") From 71722880e0d384ac9a6708cd3af791b868a1d24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 07:37:59 +0100 Subject: [PATCH 34/81] add test attempt for data source --- pkg/i2b2datasource/datasource_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go index c9c91e5..5192464 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/i2b2datasource/datasource_test.go @@ -47,3 +47,20 @@ func TestDataManager(t *testing.T) { require.NoError(t, err) require.InDeltaSlice(t, []float64{0, 1, 2, 3, 4}, data, 0.0001) } + +func TestQuery(t *testing.T) { + ds := getTestDataSource(t) + params := make(map[string]interface{}) + params["path"] = "/" + params["operation"] = "children" + res, err := ds.Query("testUser", "searchConcept", params, nil) + require.NoError(t, err) + t.Logf("result: %+v", res) + t.Logf("result: %T", res["SearchResults"]) + + t.Logf(res["SearchResults"].([]interface{})[0].(string)) + // require.EqualValues(t, ) + + //res["searchResult"] + +} From 61c98974466ac0808c25afdba22e4866c460bc6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 08:16:04 +0100 Subject: [PATCH 35/81] consistency changes and go lint --- pkg/geco-data-source.go | 2 +- pkg/i2b2api/client.go | 10 +- pkg/i2b2api/crc_client.go | 6 +- pkg/i2b2api/models/common.go | 15 +-- pkg/i2b2api/models/crc_pdo.go | 10 +- pkg/i2b2api/models/crc_psm.go | 28 +++--- pkg/i2b2api/models/ont.go | 67 +++++++------ pkg/i2b2api/ont_client.go | 12 +-- pkg/i2b2datasource/dataobjects.go | 15 +-- pkg/i2b2datasource/datasource.go | 65 ++++++------ pkg/i2b2datasource/explore_query.go | 10 +- pkg/i2b2datasource/explore_query_test.go | 20 ++-- pkg/i2b2datasource/models/explore_query.go | 7 +- pkg/i2b2datasource/models/search.go | 66 +++++++++++++ pkg/i2b2datasource/search.go | 110 +++++---------------- 15 files changed, 238 insertions(+), 205 deletions(-) diff --git a/pkg/geco-data-source.go b/pkg/geco-data-source.go index 38cde48..b6f18ad 100644 --- a/pkg/geco-data-source.go +++ b/pkg/geco-data-source.go @@ -5,7 +5,7 @@ import ( "github.com/sirupsen/logrus" ) -// todo: move this interface to geco once finalized +// todo: move this interface to geco once finalized, e.g. sdk // DataSource defines a GeCo data source plugin. The plugin must export a variable named DataSourcePlugin of the // type DataSource to be compatible. diff --git a/pkg/i2b2api/client.go b/pkg/i2b2api/client.go index 0ed43c0..172ff4c 100644 --- a/pkg/i2b2api/client.go +++ b/pkg/i2b2api/client.go @@ -11,7 +11,7 @@ import ( "github.com/sirupsen/logrus" ) -// Client is an i2b2 client for its XML API +// Client is an i2b2 client for its XML API. type Client struct { // Logger is the Logger from GeCo @@ -21,10 +21,10 @@ type Client struct { Ci models.ConnectionInfo } -// xmlRequest makes an HTTP POST request to i2b2 +// xmlRequest makes an HTTP POST request with an XML payload to i2b2. func (c Client) xmlRequest(endpoint string, xmlRequest *models.Request, xmlResponse *models.Response) error { - reqUrl := c.Ci.HiveURL + endpoint - c.Logger.Infof("i2b2 XML request to %v", reqUrl) + reqURL := c.Ci.HiveURL + endpoint + c.Logger.Infof("i2b2 XML request to %v", reqURL) xmlRequest.SetConnectionInfo(c.Ci) @@ -37,7 +37,7 @@ func (c Client) xmlRequest(endpoint string, xmlRequest *models.Request, xmlRespo c.Logger.Debugf("i2b2 request:\n%v", string(marshaledRequest)) // execute HTTP request - httpResponse, err := http.Post(reqUrl, "application/xml", bytes.NewBuffer(marshaledRequest)) + httpResponse, err := http.Post(reqURL, "application/xml", bytes.NewBuffer(marshaledRequest)) if err != nil { return fmt.Errorf("making HTTP POST of XML request: %v", err) } diff --git a/pkg/i2b2api/crc_client.go b/pkg/i2b2api/crc_client.go index bbb5ea2..d547a34 100644 --- a/pkg/i2b2api/crc_client.go +++ b/pkg/i2b2api/crc_client.go @@ -37,9 +37,9 @@ func (c Client) CrcPdoReqFromInputList(reqMsgBody *models.CrcPdoReqFromInputList return nil, fmt.Errorf("making XML request: %v", err) } - if mb, ok := xmlResponse.MessageBody.(*models.CrcPdoRespMessageBody); !ok { + mb, ok := xmlResponse.MessageBody.(*models.CrcPdoRespMessageBody) + if !ok { return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) - } else { - return mb, nil } + return mb, nil } diff --git a/pkg/i2b2api/models/common.go b/pkg/i2b2api/models/common.go index 638348f..8bacb74 100644 --- a/pkg/i2b2api/models/common.go +++ b/pkg/i2b2api/models/common.go @@ -7,10 +7,10 @@ import ( "time" ) -// MessageBody is an i2b2 XML generic body +// MessageBody is an i2b2 XML generic body. type MessageBody interface{} -// MessageHeader is an i2b2 XML header embedded in a request or response +// MessageHeader is an i2b2 XML header embedded in a request or response. type MessageHeader struct { XMLName xml.Name `xml:"message_header"` @@ -94,7 +94,7 @@ func NewRequest() Request { } } -// NewRequestWithBody creates a new ready-to-use i2b2 request, with a message body +// NewRequestWithBody creates a new ready-to-use i2b2 request, with a message body. func NewRequestWithBody(body MessageBody) (req Request) { req = NewRequest() req.MessageBody = body @@ -110,7 +110,7 @@ func (req *Request) SetConnectionInfo(ci ConnectionInfo) { req.RequestHeader.ResultWaittimeMs = strconv.FormatInt(ci.WaitTime.Milliseconds(), 10) } -// Request is an i2b2 XML request +// Request is an i2b2 XML request. type Request struct { XMLName xml.Name `xml:"msgns:request"` XMLNSMSG string `xml:"xmlns:msgns,attr"` @@ -124,7 +124,7 @@ type Request struct { MessageBody MessageBody `xml:"message_body"` } -// Response is an i2b2 XML response +// Response is an i2b2 XML response. type Response struct { XMLName xml.Name `xml:"response"` MessageHeader MessageHeader `xml:"message_header"` @@ -133,6 +133,7 @@ type Response struct { MessageBody MessageBody `xml:"message_body"` } +// CheckStatus returns an error if the status of the Response is not successful. func (response Response) CheckStatus() error { if response.ResponseHeader.ResultStatus.Status.Type != "DONE" { return errors.New(response.ResponseHeader.ResultStatus.Status.Text) @@ -140,7 +141,7 @@ func (response Response) CheckStatus() error { return nil } -// RequestHeader is an i2b2 XML header embedded in a request +// RequestHeader is an i2b2 XML header embedded in a request. type RequestHeader struct { XMLName xml.Name `xml:"request_header"` ResultWaittimeMs string `xml:"result_waittime_ms"` @@ -148,7 +149,7 @@ type RequestHeader struct { // --- response -// ResponseHeader is an i2b2 XML header embedded in a response +// ResponseHeader is an i2b2 XML header embedded in a response. type ResponseHeader struct { XMLName xml.Name `xml:"response_header"` Info struct { diff --git a/pkg/i2b2api/models/crc_pdo.go b/pkg/i2b2api/models/crc_pdo.go index c89475b..2f10e6b 100644 --- a/pkg/i2b2api/models/crc_pdo.go +++ b/pkg/i2b2api/models/crc_pdo.go @@ -6,7 +6,7 @@ import ( // --- request -// NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request +// NewCrcPdoReqFromInputList returns a new request object for i2b2 pdo request. func NewCrcPdoReqFromInputList(patientSetID string) CrcPdoReqFromInputListMessageBody { // PDO header @@ -38,7 +38,7 @@ func NewCrcPdoReqFromInputList(patientSetID string) CrcPdoReqFromInputListMessag } } -// CrcPdoReqFromInputListMessageBody is an i2b2 XML message body for CRC PDO request from input list +// CrcPdoReqFromInputListMessageBody is an i2b2 XML message body for CRC PDO request from input list. type CrcPdoReqFromInputListMessageBody struct { XMLName xml.Name `xml:"message_body"` @@ -46,14 +46,14 @@ type CrcPdoReqFromInputListMessageBody struct { PdoRequest PdoRequestFromInputList `xml:"crcpdons:request"` } -// PdoHeader is an i2b2 XML header for PDO requests +// PdoHeader is an i2b2 XML header for PDO requests. type PdoHeader struct { PatientSetLimit string `xml:"patient_set_limit"` EstimatedTime string `xml:"estimated_time"` RequestType string `xml:"request_type"` } -// PdoRequestFromInputList is an i2b2 XML PDO request - from input list +// PdoRequestFromInputList is an i2b2 XML PDO request - from input list. type PdoRequestFromInputList struct { Type string `xml:"xsi:type,attr"` Xsi string `xml:"xmlns:xsi,attr"` @@ -79,7 +79,7 @@ type PdoRequestFromInputList struct { // --- response -// CrcPdoRespMessageBody is an i2b2 XML message body for CRC PDO response +// CrcPdoRespMessageBody is an i2b2 XML message body for CRC PDO response. type CrcPdoRespMessageBody struct { XMLName xml.Name `xml:"message_body"` diff --git a/pkg/i2b2api/models/crc_psm.go b/pkg/i2b2api/models/crc_psm.go index 856427d..0baf9c3 100644 --- a/pkg/i2b2api/models/crc_psm.go +++ b/pkg/i2b2api/models/crc_psm.go @@ -9,7 +9,7 @@ import ( // --- request -// NewCrcPsmReqFromQueryDef returns a new request object for i2b2 psm request +// NewCrcPsmReqFromQueryDef returns a new request object for i2b2 psm request. func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels []Panel, queryTiming Timing, resultOutputs []ResultOutputName) CrcPsmReqFromQueryDefMessageBody { @@ -51,7 +51,7 @@ func NewCrcPsmReqFromQueryDef(ci ConnectionInfo, queryName string, queryPanels [ } } -// CrcPsmReqFromQueryDefMessageBody is an i2b2 XML message body for CRC PSM request from query definition +// CrcPsmReqFromQueryDefMessageBody is an i2b2 XML message body for CRC PSM request from query definition. type CrcPsmReqFromQueryDefMessageBody struct { XMLName xml.Name `xml:"message_body"` @@ -59,7 +59,7 @@ type CrcPsmReqFromQueryDefMessageBody struct { PsmRequest PsmRequestFromQueryDef `xml:"crcpsmns:request"` } -// PsmHeader is an i2b2 XML header for PSM request +// PsmHeader is an i2b2 XML header for PSM request. type PsmHeader struct { User struct { Text string `xml:",chardata"` @@ -73,7 +73,7 @@ type PsmHeader struct { RequestType string `xml:"request_type"` } -// PsmRequestFromQueryDef is an i2b2 XML PSM request from query definition +// PsmRequestFromQueryDef is an i2b2 XML PSM request from query definition. type PsmRequestFromQueryDef struct { Type string `xml:"xsi:type,attr"` Xsi string `xml:"xmlns:xsi,attr"` @@ -88,6 +88,7 @@ type PsmRequestFromQueryDef struct { ResultOutputs []ResultOutput `xml:"result_output_list>result_output"` } +// NewPanel creates a new Panel. func NewPanel(panelNb int, not bool, timing Timing, items []Item) Panel { invert := "0" if not { @@ -104,7 +105,7 @@ func NewPanel(panelNb int, not bool, timing Timing, items []Item) Panel { } } -// Panel is an i2b2 XML panel +// Panel is an i2b2 XML panel. type Panel struct { PanelNumber string `xml:"panel_number"` PanelAccuracyScale string `xml:"panel_accuracy_scale"` @@ -115,7 +116,7 @@ type Panel struct { Items []Item `xml:"item"` } -// Item is an i2b2 XML item +// Item is an i2b2 XML item. type Item struct { Hlevel string `xml:"hlevel"` ItemName string `xml:"item_name"` @@ -128,29 +129,30 @@ type Item struct { ItemIsSynonym string `xml:"item_is_synonym"` } -// ConstrainByModifier is an i2b2 XML constrain_by_modifier element +// ConstrainByModifier is an i2b2 XML constrain_by_modifier element. type ConstrainByModifier struct { AppliedPath string `xml:"applied_path"` ModifierKey string `xml:"modifier_key"` ConstrainByValue *ConstrainByValue `xml:"constrain_by_value"` } -// ConstrainByValue is an i2b2 XML constrain_by_value element +// ConstrainByValue is an i2b2 XML constrain_by_value element. type ConstrainByValue struct { ValueType string `xml:"value_type"` ValueOperator string `xml:"value_operator"` ValueConstraint string `xml:"value_constraint"` } -// ResultOutput is an i2b2 XML requested result type +// ResultOutput is an i2b2 XML requested result type. type ResultOutput struct { PriorityIndex string `xml:"priority_index,attr"` Name string `xml:"name,attr"` } -// ResultOutputName is an i2b2 XML requested result type value +// ResultOutputName is an i2b2 XML requested result type value. type ResultOutputName string +// Enumerated values for ResultOutputName. const ( ResultOutputPatientSet ResultOutputName = "PATIENTSET" ResultOutputEncounterSet ResultOutputName = "PATIENT_ENCOUNTER_SET" @@ -161,8 +163,10 @@ const ( ResultOutputRaceCount ResultOutputName = "PATIENT_RACE_COUNT_XML" ) +// Timing is an i2b2 XML timing. type Timing string +// Enumerated values for Timing. const ( TimingAny Timing = "ANY" TimingSameVisit Timing = "SAMEVISIT" @@ -213,6 +217,7 @@ type CrcPsmRespMessageBody struct { } `xml:"response"` } +// CheckStatus returns an error if the status of the CrcPsmRespMessageBody is not successful. func (mb CrcPsmRespMessageBody) CheckStatus() error { var errorMessages []string for _, status := range mb.Response.Status { @@ -227,7 +232,7 @@ func (mb CrcPsmRespMessageBody) CheckStatus() error { return nil } -// QueryResultInstance is an i2b2 XML query result instance +// QueryResultInstance is an i2b2 XML query result instance. type QueryResultInstance struct { ResultInstanceID string `xml:"result_instance_id"` QueryInstanceID string `xml:"query_instance_id"` @@ -246,6 +251,7 @@ type QueryResultInstance struct { } `xml:"query_status_type"` } +// CheckStatus returns an error if the status of the QueryResultInstance is not successful. func (qri QueryResultInstance) CheckStatus() error { if qri.QueryStatusType.StatusTypeID != "3" { return fmt.Errorf("i2b2 result instance does not have finished status: %v / %v / %v", qri.QueryStatusType.StatusTypeID, qri.QueryStatusType.Name, qri.QueryStatusType.Description) diff --git a/pkg/i2b2api/models/ont.go b/pkg/i2b2api/models/ont.go index 8cbd8a9..4d5f234 100644 --- a/pkg/i2b2api/models/ont.go +++ b/pkg/i2b2api/models/ont.go @@ -6,6 +6,7 @@ import ( // --- request +// NewOntReqGetTermInfoMessageBody returns a new message body for a request object for concept info. func NewOntReqGetTermInfoMessageBody(ontMaxElements, path string) OntReqGetTermInfoMessageBody { body := OntReqGetTermInfoMessageBody{} @@ -19,20 +20,7 @@ func NewOntReqGetTermInfoMessageBody(ontMaxElements, path string) OntReqGetTermI return body } -func NewOntReqGetModifierInfoMessageBody(path string, appliedPath string) OntReqGetModifierInfoMessageBody { - body := OntReqGetModifierInfoMessageBody{} - - body.GetModifierInfo.Hiddens = "false" - body.GetModifierInfo.Blob = "true" - body.GetModifierInfo.Synonyms = "false" - body.GetModifierInfo.Type = "core" - body.GetModifierInfo.Self = path - body.GetModifierInfo.AppliedPath = appliedPath - - return body -} - -// NewOntReqGetCategoriesMessageBody returns a new message body for a request object for i2b2 categories (ontology root nodes) +// NewOntReqGetCategoriesMessageBody returns a new message body for a request object for i2b2 categories (ontology root nodes). func NewOntReqGetCategoriesMessageBody() OntReqGetCategoriesMessageBody { body := OntReqGetCategoriesMessageBody{} @@ -44,7 +32,7 @@ func NewOntReqGetCategoriesMessageBody() OntReqGetCategoriesMessageBody { return body } -// NewOntReqGetChildrenMessageBody returns a new message body for a request object for i2b2 children of a node +// NewOntReqGetChildrenMessageBody returns a new message body for a request object for i2b2 children of a node. func NewOntReqGetChildrenMessageBody(ontMaxElements, parent string) OntReqGetChildrenMessageBody { body := OntReqGetChildrenMessageBody{} @@ -59,7 +47,21 @@ func NewOntReqGetChildrenMessageBody(ontMaxElements, parent string) OntReqGetChi return body } -// NewOntReqGetModifiersMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path +// NewOntReqGetModifierInfoMessageBody returns a new message body for a request object for modifier info. +func NewOntReqGetModifierInfoMessageBody(path string, appliedPath string) OntReqGetModifierInfoMessageBody { + body := OntReqGetModifierInfoMessageBody{} + + body.GetModifierInfo.Hiddens = "false" + body.GetModifierInfo.Blob = "true" + body.GetModifierInfo.Synonyms = "false" + body.GetModifierInfo.Type = "core" + body.GetModifierInfo.Self = path + body.GetModifierInfo.AppliedPath = appliedPath + + return body +} + +// NewOntReqGetModifiersMessageBody returns a new request object to get the i2b2 modifiers that apply to the concept path. func NewOntReqGetModifiersMessageBody(self string) OntReqGetModifiersMessageBody { body := OntReqGetModifiersMessageBody{} @@ -72,7 +74,7 @@ func NewOntReqGetModifiersMessageBody(self string) OntReqGetModifiersMessageBody return body } -// NewOntReqGetModifierChildrenMessageBody returns a new message body for a request object to get the i2b2 modifiers that apply to the concept path +// NewOntReqGetModifierChildrenMessageBody returns a new message body for a request object to get the children of a modifier. func NewOntReqGetModifierChildrenMessageBody(ontMaxElements, parent, appliedPath, appliedConcept string) OntReqGetModifierChildrenMessageBody { body := OntReqGetModifierChildrenMessageBody{} @@ -97,6 +99,7 @@ type baseMessageBody struct { Max string `xml:"max,attr,omitempty"` } +// OntReqGetTermInfoMessageBody is an i2b2 XML message for ontology concept info request. type OntReqGetTermInfoMessageBody struct { XMLName xml.Name `xml:"message_body"` GetTermInfo struct { @@ -105,16 +108,7 @@ type OntReqGetTermInfoMessageBody struct { } `xml:"ontns:get_term_info"` } -type OntReqGetModifierInfoMessageBody struct { - XMLName xml.Name `xml:"message_body"` - GetModifierInfo struct { - baseMessageBody - Self string `xml:"self"` - AppliedPath string `xml:"applied_path"` - } `xml:"ontns:get_modifier_info"` -} - -// OntReqGetCategoriesMessageBody is an i2b2 XML message body for ontology categories request +// OntReqGetCategoriesMessageBody is an i2b2 XML message body for ontology categories request. type OntReqGetCategoriesMessageBody struct { XMLName xml.Name `xml:"message_body"` GetCategories struct { @@ -122,7 +116,7 @@ type OntReqGetCategoriesMessageBody struct { } `xml:"ontns:get_categories"` } -// OntReqGetChildrenMessageBody is an i2b2 XML message for ontology children request +// OntReqGetChildrenMessageBody is an i2b2 XML message for ontology children request. type OntReqGetChildrenMessageBody struct { XMLName xml.Name `xml:"message_body"` GetChildren struct { @@ -131,7 +125,17 @@ type OntReqGetChildrenMessageBody struct { } `xml:"ontns:get_children"` } -// OntReqGetModifiersMessageBody is an i2b2 XML message for ontology modifiers request +// OntReqGetModifierInfoMessageBody is an i2b2 XML message for ontology modifier info request. +type OntReqGetModifierInfoMessageBody struct { + XMLName xml.Name `xml:"message_body"` + GetModifierInfo struct { + baseMessageBody + Self string `xml:"self"` + AppliedPath string `xml:"applied_path"` + } `xml:"ontns:get_modifier_info"` +} + +// OntReqGetModifiersMessageBody is an i2b2 XML message for ontology modifiers request. type OntReqGetModifiersMessageBody struct { XMLName xml.Name `xml:"message_body"` GetModifiers struct { @@ -140,7 +144,7 @@ type OntReqGetModifiersMessageBody struct { } `xml:"ontns:get_modifiers"` } -// OntReqGetModifierChildrenMessageBody is an i2b2 XML message for ontology modifier children request +// OntReqGetModifierChildrenMessageBody is an i2b2 XML message for ontology modifier children request. type OntReqGetModifierChildrenMessageBody struct { XMLName xml.Name `xml:"message_body"` GetModifierChildren struct { @@ -196,6 +200,7 @@ type Modifier struct { AppliedPath string `xml:"applied_path"` } +// MetadataXML is the metadata of a Concept. type MetadataXML struct { CreationDateTime string `xml:"ValueMetadata>CreationDateTime"` DataType string `xml:"ValueMetadata>DataType"` @@ -208,6 +213,7 @@ type MetadataXML struct { Version string `xml:"ValueMetadata>Version"` } +// ValueMetadataUnitValues is part of MetadataXML. type ValueMetadataUnitValues struct { ConvertingUnits []UnitValuesConvertingUnits `xml:"ConvertingUnits"` EqualUnits []string `xml:"EqualUnits"` @@ -215,6 +221,7 @@ type ValueMetadataUnitValues struct { NormalUnits string `xml:"NormalUnits"` } +// UnitValuesConvertingUnits is part of ValueMetadataUnitValues. type UnitValuesConvertingUnits struct { MultiplyingFactor string `xml:"MultiplyingFactor"` Units string `xml:"Units"` diff --git a/pkg/i2b2api/ont_client.go b/pkg/i2b2api/ont_client.go index 775a833..0b40391 100644 --- a/pkg/i2b2api/ont_client.go +++ b/pkg/i2b2api/ont_client.go @@ -32,11 +32,11 @@ func (c Client) ontConceptsRequest(endpoint string, reqMsgBody models.MessageBod return nil, fmt.Errorf("making XML request: %v", err) } - if mb, ok := xmlResponse.MessageBody.(*models.OntRespConceptsMessageBody); !ok { + mb, ok := xmlResponse.MessageBody.(*models.OntRespConceptsMessageBody) + if !ok { return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) - } else { - return mb, nil } + return mb, nil } // OntGetModifiers makes an i2b2 API request to /OntologyService/getModifiers. @@ -65,9 +65,9 @@ func (c Client) ontModifiersRequest(endpoint string, reqMsgBody models.MessageBo return nil, fmt.Errorf("making XML request: %v", err) } - if mb, ok := xmlResponse.MessageBody.(*models.OntRespModifiersMessageBody); !ok { + mb, ok := xmlResponse.MessageBody.(*models.OntRespModifiersMessageBody) + if !ok { return nil, fmt.Errorf("casting message body, got %T", xmlResponse.MessageBody) - } else { - return mb, nil } + return mb, nil } diff --git a/pkg/i2b2datasource/dataobjects.go b/pkg/i2b2datasource/dataobjects.go index 75d1fc8..6a48e3f 100644 --- a/pkg/i2b2datasource/dataobjects.go +++ b/pkg/i2b2datasource/dataobjects.go @@ -13,11 +13,12 @@ import ( func (ds I2b2DataSource) storeIntValue(value uint64, doSharedID string) error { do := datamanager.NewFloatVector([]float64{float64(value)}) // todo: need to be different type of data object - if doId, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false); err != nil { + + doID, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false) + if err != nil { return fmt.Errorf("adding data object: %v", err) - } else { - ds.logger.Infof("added data object to data manager (%v)", doId) } + ds.logger.Infof("added data object to data manager (%v)", doID) return nil } @@ -29,13 +30,13 @@ func (ds I2b2DataSource) storeIntVector(values []uint64, doSharedID string) erro for _, value := range values { floats = append(floats, float64(value)) } - do := datamanager.NewFloatVector(floats) // todo: need to be different type of data object - if doId, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false); err != nil { + + doID, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false) + if err != nil { return fmt.Errorf("adding data object: %v", err) - } else { - ds.logger.Infof("added data object to data manager (%v)", doId) } + ds.logger.Infof("added data object to data manager (%v)", doID) return nil } diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go index 0b5bac4..aff2cba 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/i2b2datasource/datasource.go @@ -12,26 +12,28 @@ import ( "github.com/sirupsen/logrus" ) -// todo: linting, remove from models (both packages) - -type queryType string +// Operation is an operation of the data source supported by I2b2DataSource.Query. +type Operation string +// Enumerated values for Operation. const ( - queryTypeSearchConcept queryType = "searchConcept" - queryTypeSearchModifier queryType = "searchModifier" - queryTypeExploreQuery queryType = "exploreQuery" - queryTypeGetCohorts queryType = "getCohorts" - queryTypeAddCohort queryType = "addCohort" - queryTypeDeleteCohort queryType = "deleteCohort" - queryTypeSurvivalQuery queryType = "survivalQuery" - queryTypeSearchOntology queryType = "searchOntology" + OperationSearchConcept Operation = "searchConcept" + OperationSearchModifier Operation = "searchModifier" + OperationExploreQuery Operation = "exploreQuery" + OperationGetCohorts Operation = "getCohorts" + OperationAddCohort Operation = "addCohort" + OperationDeleteCohort Operation = "deleteCohort" + OperationSurvivalQuery Operation = "survivalQuery" + OperationSearchOntology Operation = "searchOntology" ) +// Values of identifiers for data objects shared IDs. const ( - sharedIdExploreQueryCount string = "count" - sharedIdExploreQueryPatientList string = "patientList" + sharedIDExploreQueryCount string = "count" + sharedIDExploreQueryPatientList string = "patientList" ) +// I2b2DataSource is an i2b2 data source for GeCo. It implements the data source interface. type I2b2DataSource struct { // init is true if the I2b2DataSource has been initialized. @@ -50,6 +52,7 @@ type I2b2DataSource struct { i2b2OntMaxElements string } +// Init implements the data source interface Init function. func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) (err error) { fmt.Println("called init") @@ -87,17 +90,7 @@ func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldL return nil } -// todo: exists in GeCo, can be exposed by SDK code -func (ds I2b2DataSource) logError(errMsg string, causedBy error) (err error) { - if causedBy == nil { - err = fmt.Errorf("%v", errMsg) - } else { - err = fmt.Errorf("%v: %v", errMsg, causedBy) - } - ds.logger.Error(err) - return err -} - +// Query implements the data source interface Query function. func (ds I2b2DataSource) Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) { if !ds.init { panic(fmt.Errorf("data source is not initialized")) @@ -109,8 +102,8 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s // todo: decoder might need squash param set results = make(map[string]interface{}) - switch queryType(operation) { - case queryTypeSearchConcept: + switch Operation(operation) { + case OperationSearchConcept: decodedParams := &models.SearchConceptParameters{} if err := mapstructure.Decode(parameters, decodedParams); err != nil { return nil, ds.logError("decoding parameters", err) @@ -120,7 +113,7 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s return nil, ds.logError("encoding results", err) } - case queryTypeSearchModifier: + case OperationSearchModifier: decodedParams := &models.SearchModifierParameters{} if err := mapstructure.Decode(parameters, decodedParams); err != nil { return nil, ds.logError("decoding parameters", err) @@ -130,9 +123,9 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s return nil, ds.logError("encoding results", err) } - case queryTypeExploreQuery: - countSharedID, countOK := resultsSharedIds[sharedIdExploreQueryCount] - patientListSharedID, patientListOK := resultsSharedIds[sharedIdExploreQueryPatientList] + case OperationExploreQuery: + countSharedID, countOK := resultsSharedIds[sharedIDExploreQueryCount] + patientListSharedID, patientListOK := resultsSharedIds[sharedIDExploreQueryPatientList] if !countOK || !patientListOK { return nil, ds.logError("missing results shared ID", nil) } @@ -153,3 +146,15 @@ func (ds I2b2DataSource) Query(userID string, operation string, parameters map[s } return } + +// logError creates and logs an error. +// todo: exists in GeCo, can be exposed by SDK code +func (ds I2b2DataSource) logError(errMsg string, causedBy error) (err error) { + if causedBy == nil { + err = fmt.Errorf("%v", errMsg) + } else { + err = fmt.Errorf("%v: %v", errMsg, causedBy) + } + ds.logger.Error(err) + return err +} diff --git a/pkg/i2b2datasource/explore_query.go b/pkg/i2b2datasource/explore_query.go index 1bcd999..cc3456f 100644 --- a/pkg/i2b2datasource/explore_query.go +++ b/pkg/i2b2datasource/explore_query.go @@ -19,25 +19,25 @@ func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (pa return 0, nil, fmt.Errorf("parsing patient count: %v", err) } else { for _, patientID := range i2b2PatientIDs { - if parsedPatientID, err := strconv.ParseUint(patientID, 10, 64); err != nil { + parsedPatientID, err := strconv.ParseUint(patientID, 10, 64) + if err != nil { return 0, nil, fmt.Errorf("parsing patient ID: %v", err) - } else { - patientList = append(patientList, parsedPatientID) } + patientList = append(patientList, parsedPatientID) } } return } -// doExploreQuery requests the explore query to the i2b2 CRC and parse its results. +// doExploreQuery requests an explore query to the i2b2 CRC and parse its results. func (ds I2b2DataSource) doExploreQuery(params *models.ExploreQueryParameters) (patientCount, patientSetID string, err error) { // build query panels, timing := params.Definition.ToI2b2APIModel() psmReq := i2b2apimodels.NewCrcPsmReqFromQueryDef( ds.i2b2Client.Ci, - params.Id, + params.ID, panels, timing, []i2b2apimodels.ResultOutputName{i2b2apimodels.ResultOutputPatientSet, i2b2apimodels.ResultOutputCount}, diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/i2b2datasource/explore_query_test.go index 4c97961..1466746 100644 --- a/pkg/i2b2datasource/explore_query_test.go +++ b/pkg/i2b2datasource/explore_query_test.go @@ -11,7 +11,7 @@ func TestExploreQueryConcept(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -27,7 +27,7 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -43,7 +43,7 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3, 4}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "2", + ID: "2", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -69,7 +69,7 @@ func TestExploreQueryConcept(t *testing.T) { require.Subset(t, []uint64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "3", + ID: "3", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -91,7 +91,7 @@ func TestExploreQueryConceptValue(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -110,7 +110,7 @@ func TestExploreQueryConceptValue(t *testing.T) { require.Subset(t, []uint64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -138,7 +138,7 @@ func TestExploreQueryModifier(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -161,7 +161,7 @@ func TestExploreQueryModifier(t *testing.T) { require.Subset(t, []uint64{1, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -197,7 +197,7 @@ func TestExploreQueryModifierValue(t *testing.T) { ds := getTestDataSource(t) count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "0", + ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, @@ -223,7 +223,7 @@ func TestExploreQueryModifierValue(t *testing.T) { require.Subset(t, []uint64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ - Id: "1", + ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ Not: false, diff --git a/pkg/i2b2datasource/models/explore_query.go b/pkg/i2b2datasource/models/explore_query.go index a9307e1..079236f 100644 --- a/pkg/i2b2datasource/models/explore_query.go +++ b/pkg/i2b2datasource/models/explore_query.go @@ -4,16 +4,19 @@ import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" // --- parameters +// ExploreQueryParameters is the parameter for the ExploreQuery operation. type ExploreQueryParameters struct { - Id string + ID string Definition ExploreQueryDefinition } +// ExploreQueryDefinition is the query definition of ExploreQueryParameters. type ExploreQueryDefinition struct { Timing string // any | samevisit | sameinstancenum Panels []Panel } +// Panel is part of an ExploreQueryDefinition. type Panel struct { Not bool Timing string // any | samevisit | sameinstancenum @@ -21,6 +24,7 @@ type Panel struct { ConceptItems []ConceptItem } +// ConceptItem is part of a Panel. type ConceptItem struct { QueryTerm string Operator string // EQ | NE | GT | GE | LT | LE | BETWEEN | IN | LIKE[exact] | LIKE[begin] | LIKE[end] | LIKE[contains] @@ -32,6 +36,7 @@ type ConceptItem struct { } } +// ToI2b2APIModel converts this query definition in the i2b2 API format. func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2apimodels.Panel, i2b2ApiTiming i2b2apimodels.Timing) { i2b2ApiTiming = i2b2apimodels.Timing(d.Timing) diff --git a/pkg/i2b2datasource/models/search.go b/pkg/i2b2datasource/models/search.go index 70b374f..b5719aa 100644 --- a/pkg/i2b2datasource/models/search.go +++ b/pkg/i2b2datasource/models/search.go @@ -1,5 +1,7 @@ package models +import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + // --- parameters // func NewSearchConceptParameters(params map[string]interface{}) (model SearchConceptParameters, retErr error) { @@ -11,6 +13,7 @@ package models // }, nil // } +// SearchConceptParameters is the parameter for the SearchConcept operation. type SearchConceptParameters struct { Path string Operation string // children | info | concept @@ -31,6 +34,7 @@ type SearchConceptParameters struct { // todo: maybe squash model together if OK with serializing etc. +// SearchModifierParameters is the parameter for the SearchModifier operation. type SearchModifierParameters struct { SearchConceptParameters AppliedPath string @@ -39,10 +43,70 @@ type SearchModifierParameters struct { // --- results +// SearchResults is the result of the Search operations. type SearchResults struct { SearchResults []SearchResult } +// NewSearchResultFromI2b2Concept creates a new SearchResult from an i2b2 concept. +func NewSearchResultFromI2b2Concept(concept i2b2apimodels.Concept) SearchResult { + parsed := SearchResult{ + Name: concept.Name, + DisplayName: concept.Name, + Code: concept.Basecode, + Path: i2b2apimodels.ConvertPathFromI2b2Format(concept.Key), + AppliedPath: "@", + Comment: concept.Comment, + } + + // parse i2b2 visual attributes + switch concept.Visualattributes[0] { + // i2b2 leaf + case 'L': + parsed.Type = "concept" + parsed.Leaf = true + case 'R': + parsed.Type = "modifier" + parsed.Leaf = true + + // i2b2 container + case 'C': + parsed.Type = "concept_container" + parsed.Leaf = false + case 'O': + parsed.Type = "modifier_container" + parsed.Leaf = false + + // i2b2 folder (& default) + default: + fallthrough + case 'F': + parsed.Type = "concept_folder" + parsed.Leaf = false + case 'D': + parsed.Type = "modifier_folder" + parsed.Leaf = false + } + + if concept.Metadataxml != nil { + parsed.Metadata = SearchResultMetadata{ + DataType: concept.Metadataxml.DataType, + OkToUseValues: concept.Metadataxml.Oktousevalues, + UnitValues: MetadataUnitValues{NormalUnits: concept.Metadataxml.UnitValues.NormalUnits}, + } + } + + return parsed +} + +// NewSearchResultFromI2b2Modifier creates a new SearchResult from an i2b2 modifier. +func NewSearchResultFromI2b2Modifier(modifier i2b2apimodels.Modifier) SearchResult { + res := NewSearchResultFromI2b2Concept(modifier.Concept) + res.AppliedPath = i2b2apimodels.ConvertPathFromI2b2Format(modifier.AppliedPath) + return res +} + +// SearchResult is a result part of SearchResults. type SearchResult struct { Path string AppliedPath string @@ -55,12 +119,14 @@ type SearchResult struct { Metadata SearchResultMetadata } +// SearchResultMetadata is part of SearchResult. type SearchResultMetadata struct { DataType string // PosInteger | Integer | Float | PosFloat | Enum | String OkToUseValues string // Y UnitValues MetadataUnitValues } +// MetadataUnitValues is part of SearchResultMetadata. type MetadataUnitValues struct { NormalUnits string } diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go index d42894e..afbb411 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/i2b2datasource/search.go @@ -25,11 +25,11 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( case "info": if path == "/" { return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil - } else { - req := i2b2apimodels.NewOntReqGetTermInfoMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) - if resp, err = ds.i2b2Client.OntGetTermInfo(&req); err != nil { - return nil, fmt.Errorf("requesting term info: %v", err) - } + } + + req := i2b2apimodels.NewOntReqGetTermInfoMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetTermInfo(&req); err != nil { + return nil, fmt.Errorf("requesting term info: %v", err) } case "children": @@ -38,12 +38,11 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( if resp, err = ds.i2b2Client.OntGetCategories(&req); err != nil { return nil, fmt.Errorf("requesting categories: %v", err) } + } - } else { - req := i2b2apimodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) - if resp, err = ds.i2b2Client.OntGetChildren(&req); err != nil { - return nil, fmt.Errorf("requesting children: %v", err) - } + req := i2b2apimodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetChildren(&req); err != nil { + return nil, fmt.Errorf("requesting children: %v", err) } default: @@ -53,59 +52,11 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( // generate result from response searchResults := make([]models.SearchResult, 0, len(resp.Concepts)) for _, concept := range resp.Concepts { - searchResults = append(searchResults, parseI2b2Concept(concept)) + searchResults = append(searchResults, models.NewSearchResultFromI2b2Concept(concept)) } return &models.SearchResults{SearchResults: searchResults}, nil } -func parseI2b2Concept(concept i2b2apimodels.Concept) models.SearchResult { - - parsed := models.SearchResult{ - Name: concept.Name, - DisplayName: concept.Name, - Code: concept.Basecode, - Path: i2b2apimodels.ConvertPathFromI2b2Format(concept.Key), - AppliedPath: "@", - Comment: concept.Comment, - } - - parsed.Type, parsed.Leaf = parseI2b2VisualAttributes(concept.Visualattributes) - - if concept.Metadataxml != nil { - parsed.Metadata = models.SearchResultMetadata{ - DataType: concept.Metadataxml.DataType, - OkToUseValues: concept.Metadataxml.Oktousevalues, - UnitValues: models.MetadataUnitValues{NormalUnits: concept.Metadataxml.UnitValues.NormalUnits}, - } - } - - return parsed -} - -func parseI2b2VisualAttributes(visualAttributes string) (kind string, leaf bool) { - switch visualAttributes[0] { - // i2b2 leaf - case 'L': - return "concept", true - case 'R': - return "modifier", true - - // i2b2 container - case 'C': - return "concept_container", false - case 'O': - return "modifier_container", false - - // i2b2 folder (& default) - default: - fallthrough - case 'F': - return "concept_folder", false - case 'D': - return "modifier_folder", false - } -} - // SearchModifier retrieves the info about or the children of the modifier identified by path. func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) (*models.SearchResults, error) { @@ -123,37 +74,34 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) case "concept": if path == "/" { return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + } - } else { - req := i2b2apimodels.NewOntReqGetModifiersMessageBody(i2b2FormatPath) - if resp, err = ds.i2b2Client.OntGetModifiers(&req); err != nil { - return nil, fmt.Errorf("requesting modifiers of concept: %v", err) - } + req := i2b2apimodels.NewOntReqGetModifiersMessageBody(i2b2FormatPath) + if resp, err = ds.i2b2Client.OntGetModifiers(&req); err != nil { + return nil, fmt.Errorf("requesting modifiers of concept: %v", err) } case "info": if path == "/" { return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + } - } else { - i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) - req := i2b2apimodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) - if resp, err = ds.i2b2Client.OntGetModifierInfo(&req); err != nil { - return nil, fmt.Errorf("requesting info of modifier: %v", err) - } + i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) + req := i2b2apimodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) + if resp, err = ds.i2b2Client.OntGetModifierInfo(&req); err != nil { + return nil, fmt.Errorf("requesting info of modifier: %v", err) } case "children": if path == "/" { return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil + } - } else { - i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) - i2b2FormatAppliedConcept := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) - req := i2b2apimodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) - if resp, err = ds.i2b2Client.OntGetModifierChildren(&req); err != nil { - return nil, fmt.Errorf("requesting children of modifier: %v", err) - } + i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) + i2b2FormatAppliedConcept := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) + req := i2b2apimodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) + if resp, err = ds.i2b2Client.OntGetModifierChildren(&req); err != nil { + return nil, fmt.Errorf("requesting children of modifier: %v", err) } default: @@ -163,13 +111,7 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) // generate result from response searchResults := make([]models.SearchResult, 0, len(resp.Modifiers)) for _, modifier := range resp.Modifiers { - searchResults = append(searchResults, parseI2b2Modifier(modifier)) + searchResults = append(searchResults, models.NewSearchResultFromI2b2Modifier(modifier)) } return &models.SearchResults{SearchResults: searchResults}, nil } - -func parseI2b2Modifier(modifier i2b2apimodels.Modifier) models.SearchResult { - res := parseI2b2Concept(modifier.Concept) - res.AppliedPath = i2b2apimodels.ConvertPathFromI2b2Format(modifier.AppliedPath) - return res -} From cf28277f37763b051fd204a6768c1c2d5e12701b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 09:56:44 +0100 Subject: [PATCH 36/81] update CI --- .github/workflows/{i2b2.yml => ci-cd.yml} | 85 +++++++++++++---------- 1 file changed, 47 insertions(+), 38 deletions(-) rename .github/workflows/{i2b2.yml => ci-cd.yml} (50%) diff --git a/.github/workflows/i2b2.yml b/.github/workflows/ci-cd.yml similarity index 50% rename from .github/workflows/i2b2.yml rename to .github/workflows/ci-cd.yml index b7d3334..07b0a0d 100644 --- a/.github/workflows/i2b2.yml +++ b/.github/workflows/ci-cd.yml @@ -1,51 +1,57 @@ -name: Build i2b2 docker image and run some tests -# todo: i2b2 should be built only on request, or on change of something +name: Build and test GeCo i2b2 data source +# todo: i2b2 push when? is build fast enough? on: - push: #todo: determine when to trigger + push: workflow_dispatch: jobs: - i2b2: - name: i2b2 docker image build and test + ci-cd: + name: Build and test GeCo i2b2 data source runs-on: ubuntu-latest # env: -# PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} +# PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} todo: also put if manual trigger steps: - - name: Environment - run: | - go version - env - - name: Setup SSH deploy key for private repositories uses: webfactory/ssh-agent@v0.4.1 with: ssh-private-key: ${{ secrets.GECO_SSH_DEPLOY_KEY }} + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '^1.17' + - name: Checkout code uses: actions/checkout@v2 with: token: ${{ secrets.LDS_PKG_PAT }} submodules: true -# - name: Setup Git and Golang environment for private repositories -# run: | -# git config --global url."git@github.com:".insteadOf "https://github.com/" -# go env -w GOPRIVATE=github.com/ldsec/geco,github.com/ldsec/spindle,github.com/ldsec/geco-i2b2-data-source + - name: Environment + run: | + go version + ./scripts/version.sh + pushd third_party/geco && ./scripts/version.sh && popd + env + + - name: Generate GeCo Swagger files + run: make geco-swagger-gen + + - name: Go sources beautification + run: make go-imports go-lint + + - name: Build data source plugin + run: make go-build-plugin - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 -# - name: Get versions -# id: get_versions -# run: | -# VERSION=$(scripts/version.sh) -# GECO_VERSION=$(cd test/geco && scripts/version.sh) -# echo ::set-output name=version::${VERSION} -# echo ::set-output name=geco_version::${GECO_VERSION} -# echo ::set-output name=i2b2_docker_tag::ghcr.io/ldsec/i2b2-geco:${VERSION} - -# # todo: GET VERSION OF i2b2 SOMEWHERE AND SET THIS AS VERSION ON DOCKER -# use a variant of PUSH? or have it only manual? in terms of being triggered + - name: Get versions + id: get_versions + run: | + DATASOURCE_VERSION=$(scripts/version.sh) + echo ::set-output name=datasource_version::${DATASOURCE_VERSION} + echo ::set-output name=i2b2_docker_image::ghcr.io/ldsec/i2b2-geco:${DATASOURCE_VERSION} - name: Build i2b2 Docker image uses: docker/build-push-action@v2 @@ -54,27 +60,30 @@ jobs: build-args: BUILD=docker load: true ssh: default - # tags: ${{ steps.get_version.outputs.docker_tag }} - tags: ghcr.io/ldsec/i2b2-geco + tags: ${{ steps.get_version.outputs.i2b2_docker_image }} cache-from: type=gha,scope=buildkit cache-to: type=gha,scope=buildkit,mode=max - - name: Start GeCo deployment - run: | - make start-geco-dev-local-3nodes - - - name: Start i2b2 + - name: Start deployments run: | + make geco-docker-compose ARGS="up -d postgresql" make i2b2-docker-compose ARGS="up -d" - - name: Run i2b2 test + - name: Run i2b2 docker image test run: | sudo apt-get install -y libxml2-utils - make test-i2b2 + make i2b2-test + + - name: Run go unit tests + run: make go-unit-tests + + - name: Show GeCo deployment logs + if: always() + run: make geco-docker-compose ARGS="logs" - - name: Show deployment logs + - name: Show i2b2 deployment logs if: always() - run: make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="logs" + run: make i2b2-docker-compose ARGS="logs" # - name: Login to GitHub Container Registry # if: ${{ env.PUSH }} @@ -86,4 +95,4 @@ jobs: # - name: Push GeCo Docker image # if: ${{ env.PUSH }} -# run: docker push ${{ steps.get_version.outputs.i2b2_docker_tag }} +# run: docker push ${{ steps.get_version.outputs.i2b2_docker_image }} From 5516e5408e863b71cbe868676448b70b640ab8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 09:57:35 +0100 Subject: [PATCH 37/81] update makefile --- Makefile | 55 +++++++++++++++++++--------------------------- scripts/version.sh | 8 +++++++ 2 files changed, 31 insertions(+), 32 deletions(-) create mode 100755 scripts/version.sh diff --git a/Makefile b/Makefile index 92dcd95..dd5d4f4 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,32 @@ -#VERSION := $(shell scripts/version.sh) -USER_GROUP := $(shell id -u):$(shell id -g) -DOCKER_IMAGE ?= ghcr.io/ldsec/geco-i2b2-data-source:$(VERSION) - +export DATASOURCE_VERSION := $(shell scripts/version.sh) +export USER_GROUP := $(shell id -u):$(shell id -g) +export I2B2_DOCKER_IMAGE ?= ghcr.io/ldsec/i2b2-geco:$(DATASOURCE_VERSION) export COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 -# todo: set up CI with DB from geco for later , ensure exec times are OK -# todo: assumes geco submodule has been retrioeved: -# todo: gha caching https://evilmartians.com/chronicles/build-images-on-github-actions-with-docker-layer-caching -# -> connect as postgres in geco DB like this -# docker exec -it i2b2postgresql psql -U postgres -# CREATE ROLE i2b2 LOGIN PASSWORD 'i2b2'; -# ALTER USER i2b2 CREATEDB; - -# todo: make clean swagger-gen -# todo pushd third_party/geco && make clean swagger-gen && popd - -start-geco-dev-local-3nodes: - make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="up -d postgresql" - # todo: only the DB at the moment - -# use ARGS to pass to docker-compose arguments, e.g. make docker-compose ARGS="up -d" -i2b2-docker-compose: - cd test/i2b2 && docker-compose -f docker-compose.yml $(ARGS) - # todo: version etc. to pass - -test-i2b2: +.PHONY: test clean +test: go-imports go-lint go-unit-tests i2b2-test +clean: + rm -f ./build/geco-i2b2-data-source.so + +# --- GeCo +.PHONY: geco-start-dev-local-3nodes geco-swagger-gen +geco-docker-compose: + make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="$(ARGS)" +geco-swagger-gen: + cd third_party/geco && make go-swagger-gen + +# --- i2b2 docker +.PHONY: i2b2-docker-compose i2b2-test +i2b2-docker-compose: # use ARGS to pass to docker-compose arguments, e.g. make docker-compose ARGS="up -d" + cd deployments && docker-compose -f i2b2.yml $(ARGS) +i2b2-test: cd test/i2b2 && ./test_i2b2_docker.sh -# --- go source code -.PHONY: build-bin test-go test clean -build-plugin: +# --- go sources +.PHONY: go-build-plugin go-imports go-lint go-unit-tests +go-build-plugin: go build -buildmode=plugin -v -o ./build/ ./cmd/... -test-go: go-imports go-lint go-unit-tests -clean: go-swagger-clean - rm -f ./build/geco-cli ./build/geco-server -.PHONY: go-imports go-lint go-unit-tests go-imports: @echo Checking correct formatting of files @{ \ diff --git a/scripts/version.sh b/scripts/version.sh new file mode 100755 index 0000000..c086ace --- /dev/null +++ b/scripts/version.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +VERSION=$(git describe --tags 2> /dev/null) +if [ $? -eq 0 ]; then + echo "${VERSION}" +else + echo "v0.0.0-dev-$(git describe --tags --always)" +fi From 6bc169604553b5fefe07243030a1a53d66a0d630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 09:57:50 +0100 Subject: [PATCH 38/81] write readme --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/README.md b/README.md index 2061ebf..2f8389e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,62 @@ # GeCo i2b2 Data Source Plugin +## Build +Retrieve git submodule of GeCo: +```shell +make git submodule update --init --recursive +``` + +Generate GeCo Swagger files: +```shell +make geco-swagger-gen +``` + +Build plugin to `build/geco-i2b2-data-source.so`: +```shell +make go-build-plugin +``` + +## Test +### Start depdencies +Start GeCo (only the database for the moment): +```shell +make geco-docker-compose ARGS="up -d postgresql" +``` + +Start i2b2: +```shell +make i2b2-docker-compose ARGS="up -d" +``` + +### Run tests +Test i2b2 docker: +```shell +make i2b2-test +``` + +Run go unit tests i2b2 docker: +```shell +make go-unit tests +``` + +## Development +### Source code organization +- `build/package/i2b2/`: i2b2 docker image definition +- `cmd/geco-i2b2-data-source/`: go main package for the plugin +- `pkg/`: exported go code + - `i2b2api/`: client for i2b2 HTTP XML API + - `i2b2datasource/`: definition of the i2b2 GeCo data source +- `scripts/`: utility scripts +- `test/i2b2/`: test files for the i2b2 docker image +- `third_party/geco/`: git submodule for the GeCo source code + +### Useful commands +Start psql in the running postgresql container: +```shell +make geco-docker-compose ARGS="exec postgresql psql -U postgres" +``` + +### About the GeCo dependency +- GeCo is included in the repository as a submodule (in `third_party/geco`) and the `go.mod` file includes a `replace` directive in order to use it. +- This is notably due to the use of the development deployment of GeCo to be used for the testing of this plugin. +- Ultimately, some definitions from this plugin (e.g. DataSource interface) and from GeCo (e.g. dev deployment) need to be extracted to a public repository to serve as an SDK for data source plugins. From 33fa56adf01c8461cb6df3615ffcc2bd097c6071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 09:58:11 +0100 Subject: [PATCH 39/81] move i2b2 deployment file --- test/i2b2/docker-compose.yml => deployments/i2b2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename test/i2b2/docker-compose.yml => deployments/i2b2.yml (83%) diff --git a/test/i2b2/docker-compose.yml b/deployments/i2b2.yml similarity index 83% rename from test/i2b2/docker-compose.yml rename to deployments/i2b2.yml index 746ff60..5bb39ef 100644 --- a/test/i2b2/docker-compose.yml +++ b/deployments/i2b2.yml @@ -1,9 +1,9 @@ version: '3.9' services: i2b2: - image: ghcr.io/ldsec/i2b2-geco + image: ${I2B2_DOCKER_IMAGE:?} build: - context: ../../build/package/i2b2 + context: ../build/package/i2b2 ports: - "8080:8080" environment: @@ -16,7 +16,7 @@ services: - I2B2_DOMAIN_NAME=i2b2demo - I2B2_SERVICE_PASSWORD=changeme - DEFAULT_USER_PASSWORD=changeme - - AXIS2_LOGLEVEL=INFO + - AXIS2_LOGLEVEL=DEBUG networks: default: From 6e7aaaa6a6143fbbf22f9207d952711ba681e4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 09:58:52 +0100 Subject: [PATCH 40/81] fix tests --- internal/plugin_test.go | 5 ++++- pkg/i2b2datasource/datasource_test.go | 3 ++- pkg/i2b2datasource/search.go | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/plugin_test.go b/internal/plugin_test.go index e2e7130..cef5a45 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -37,6 +37,9 @@ func TestPlugin(t *testing.T) { err = (*ds).Init(dm, logrus.StandardLogger(), config) require.NoError(t, err) - _, err = (*ds).Query("", "", nil, nil) + params := make(map[string]interface{}) + params["path"] = "/" + params["operation"] = "children" + _, err = (*ds).Query("test", "searchConcept", params, nil) require.NoError(t, err) } diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go index 5192464..828526b 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/i2b2datasource/datasource_test.go @@ -58,7 +58,8 @@ func TestQuery(t *testing.T) { t.Logf("result: %+v", res) t.Logf("result: %T", res["SearchResults"]) - t.Logf(res["SearchResults"].([]interface{})[0].(string)) + // todo + //t.Logf(res["SearchResults"].([]interface{})[0].(string)) // require.EqualValues(t, ) //res["searchResult"] diff --git a/pkg/i2b2datasource/search.go b/pkg/i2b2datasource/search.go index afbb411..bd5406e 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/i2b2datasource/search.go @@ -38,6 +38,7 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( if resp, err = ds.i2b2Client.OntGetCategories(&req); err != nil { return nil, fmt.Errorf("requesting categories: %v", err) } + break } req := i2b2apimodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) From e9cd061cb70f3761ef33f92e9387e3df0fcd374f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 10:03:54 +0100 Subject: [PATCH 41/81] CI: add ssh auth for deps --- .github/workflows/ci-cd.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 07b0a0d..7412d9a 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -34,6 +34,11 @@ jobs: pushd third_party/geco && ./scripts/version.sh && popd env + - name: Setup Git and Golang environment for private repositories + run: | + git config --global url."git@github.com:".insteadOf "https://github.com/" + go env -w GOPRIVATE=github.com/ldsec/geco,github.com/ldsec/spindle,github.com/ldsec/geco-i2b2-data-source + - name: Generate GeCo Swagger files run: make geco-swagger-gen From b4beab2fa2fd007159c9195276ef773c53e4fcbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 10:07:19 +0100 Subject: [PATCH 42/81] CI: add spindle ssh key for deps --- .github/workflows/ci-cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 7412d9a..20a026d 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -14,7 +14,9 @@ jobs: - name: Setup SSH deploy key for private repositories uses: webfactory/ssh-agent@v0.4.1 with: - ssh-private-key: ${{ secrets.GECO_SSH_DEPLOY_KEY }} + ssh-private-key: | + ${{ secrets.GECO_SSH_DEPLOY_KEY }} + ${{ secrets.SPINDLE_SSH_DEPLOY_KEY }} - name: Set up Go uses: actions/setup-go@v2 From ccb2b38b1dba05db820e92039cfc88c58ea2f5de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 11:00:55 +0100 Subject: [PATCH 43/81] add description of file --- build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh b/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh index 132c9a5..eda5301 100644 --- a/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh +++ b/build/package/i2b2/pre-init-scripts/10-write-i2b2-datasources.sh @@ -1,5 +1,6 @@ #!/bin/bash set -Eeuo pipefail +# write all the wildfly data sources for the i2b2 cells cat > "$JBOSS_HOME/standalone/deployments/pm-ds.xml" <<EOL <?xml version="1.0" encoding="UTF-8"?> From 6acf76d65e66ac00922fbd2f66dbeaf0281c73df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 11:02:53 +0100 Subject: [PATCH 44/81] add description of file --- build/package/i2b2/I2b2PasswordHash.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/package/i2b2/I2b2PasswordHash.java b/build/package/i2b2/I2b2PasswordHash.java index 0a00a4d..c558816 100644 --- a/build/package/i2b2/I2b2PasswordHash.java +++ b/build/package/i2b2/I2b2PasswordHash.java @@ -1,5 +1,9 @@ import java.security.MessageDigest; +/** + * This Java tool implements the hash algorithm of i2b2. + * This enables setting i2b2 password for users directly in the database. + */ public final class I2b2PasswordHash { public static void main(String[] args) throws Exception { if (args.length != 1) { From 8d82645782b50e090366397038f3e3ab97cc7698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 11:18:43 +0100 Subject: [PATCH 45/81] test CI to be reverted --- .github/workflows/ci-cd.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 20a026d..bef2580 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -18,6 +18,12 @@ jobs: ${{ secrets.GECO_SSH_DEPLOY_KEY }} ${{ secrets.SPINDLE_SSH_DEPLOY_KEY }} + - name: TEST cloning geco + run: git clone git@github.com:ldsec/geco.git + + - name: TEST cloning spindle + run: git clone git@github.com:ldsec/spindle.git + - name: Set up Go uses: actions/setup-go@v2 with: From 9831d1e1fd594ec75e04111c37fc310a3107646f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 10 Dec 2021 11:29:21 +0100 Subject: [PATCH 46/81] test CI to be reverted --- .github/workflows/ci-cd.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index bef2580..d869ca9 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -15,15 +15,15 @@ jobs: uses: webfactory/ssh-agent@v0.4.1 with: ssh-private-key: | - ${{ secrets.GECO_SSH_DEPLOY_KEY }} ${{ secrets.SPINDLE_SSH_DEPLOY_KEY }} - - - name: TEST cloning geco - run: git clone git@github.com:ldsec/geco.git + ${{ secrets.GECO_SSH_DEPLOY_KEY }} - name: TEST cloning spindle run: git clone git@github.com:ldsec/spindle.git + - name: TEST cloning geco + run: git clone git@github.com:ldsec/geco.git + - name: Set up Go uses: actions/setup-go@v2 with: From f1e948c04f5e06c3ca9d140e165ca40a2c470ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Bouy=C3=A9?= <romain.bouye@epfl.ch> Date: Fri, 17 Dec 2021 15:30:58 +0100 Subject: [PATCH 47/81] Fix CI (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update webfactory/ssh-agent@v0.5.4 * Change in .gitmodule address of geco to https * ci: remove token to checkout command * ci: change git config beforehand * ci: add geco ssh-key to checkout * ci: fix geco submodule url * ci: fix geco checkout * ci: manually clone geco * update geco * ci: ignore spindle git lfs * Fix CI - use git submodule (#5) * fix attempt * CI: add back token * CI: set up SSH key for checkout * CI: attempt Co-authored-by: Mickaël Misbach <mickael@misba.ch> --- .github/workflows/ci-cd.yml | 26 ++++++++++---------------- Makefile | 1 + go.mod | 8 ++++---- go.sum | 8 +++----- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index d869ca9..fc2eb2c 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -11,19 +11,6 @@ jobs: # PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} todo: also put if manual trigger steps: - - name: Setup SSH deploy key for private repositories - uses: webfactory/ssh-agent@v0.4.1 - with: - ssh-private-key: | - ${{ secrets.SPINDLE_SSH_DEPLOY_KEY }} - ${{ secrets.GECO_SSH_DEPLOY_KEY }} - - - name: TEST cloning spindle - run: git clone git@github.com:ldsec/spindle.git - - - name: TEST cloning geco - run: git clone git@github.com:ldsec/geco.git - - name: Set up Go uses: actions/setup-go@v2 with: @@ -31,9 +18,16 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + + - name: Setup SSH deploy key for private repositories + uses: webfactory/ssh-agent@v0.5.4 with: - token: ${{ secrets.LDS_PKG_PAT }} - submodules: true + ssh-private-key: | + ${{ secrets.SPINDLE_SSH_DEPLOY_KEY }} + ${{ secrets.GECO_SSH_DEPLOY_KEY }} + + - name: Get submodules + run: git submodule update --init --recursive - name: Environment run: | @@ -45,7 +39,7 @@ jobs: - name: Setup Git and Golang environment for private repositories run: | git config --global url."git@github.com:".insteadOf "https://github.com/" - go env -w GOPRIVATE=github.com/ldsec/geco,github.com/ldsec/spindle,github.com/ldsec/geco-i2b2-data-source + go env -w GOPRIVATE=github.com/ldsec - name: Generate GeCo Swagger files run: make geco-swagger-gen diff --git a/Makefile b/Makefile index dd5d4f4..0a5af5e 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ export DATASOURCE_VERSION := $(shell scripts/version.sh) export USER_GROUP := $(shell id -u):$(shell id -g) export I2B2_DOCKER_IMAGE ?= ghcr.io/ldsec/i2b2-geco:$(DATASOURCE_VERSION) export COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 +export GIT_LFS_SKIP_SMUDGE=1 .PHONY: test clean test: go-imports go-lint go-unit-tests i2b2-test diff --git a/go.mod b/go.mod index d175029..cb86375 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.17 replace github.com/ldsec/geco => ./third_party/geco require ( - github.com/ldsec/geco v0.0.1 + github.com/ldsec/geco v0.0.2-0.20211215101444-5aa7d423be0d + github.com/mitchellh/mapstructure v1.4.3 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 ) @@ -40,7 +42,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/ldsec/lattigo/v2 v2.0.0 // indirect - github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0 // indirect + github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.0 // indirect @@ -50,7 +52,6 @@ require ( github.com/lestrrat-go/option v1.0.0 // indirect github.com/lib/pq v1.9.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/montanaflynn/stats v0.6.3 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -63,7 +64,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/segmentio/ksuid v1.0.3 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v3 v3.0.13 // indirect go.dedis.ch/onet/v3 v3.2.3 // indirect diff --git a/go.sum b/go.sum index bced2ad..23d7fba 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Cryptolens/cryptolens-golang v0.0.0-20210322162237-d3ae4c72721f/go.mod h1:zVXpFHn6wrg4t0PA7g2XT46hlspli2/g7vC9HRXvfwA= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Nerzal/gocloak/v9 v9.0.3 h1:45Jiw+H2TIrSsMboO4t6bUozMAzxeQDcwIWvnJ/KJsU= github.com/Nerzal/gocloak/v9 v9.0.3/go.mod h1:AmXlpCliq31R1r/18ei7/ML5mfJDmJ1pTBoFU79DDZg= @@ -279,7 +280,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -287,8 +287,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ldsec/lattigo/v2 v2.0.0 h1:P3rzbXFrc45swwl0u/zVCfcSDuIjTpIGueEDVQQzgEY= github.com/ldsec/lattigo/v2 v2.0.0/go.mod h1:Iu8ol3XIWyF54Ka5bBBWNnNR9kYtymmuYo1Y6Hxzr+U= -github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0 h1:VRlFRZ8WcHm3m5CVAalz6g/pW+tX832/gf8vpKQFLBM= -github.com/ldsec/spindle v0.0.0-20210923091257-7bc102eb03c0/go.mod h1:pMeb26+SHl1pJJvMVSNz7jW/lKaTYRtB6e5qIYQ9h6s= +github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 h1:RCds6LS6RVHUI/vw64tgVoc/v0qtvtOygSBS2YEo/Bk= +github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4/go.mod h1:/uvhNx7QKG/gNJh5pWYSNWhbO4xffJ6SC4oV91yftp4= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= @@ -327,7 +327,6 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -594,7 +593,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= From 580e2577d67ab855f3dd2223f0ed6d220d32c68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 17 Dec 2021 15:31:52 +0100 Subject: [PATCH 48/81] update geco version --- third_party/geco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/geco b/third_party/geco index d3f5e8b..1fb6d97 160000 --- a/third_party/geco +++ b/third_party/geco @@ -1 +1 @@ -Subproject commit d3f5e8b7c84c87ae81bcf1053bc5f9f7428c9784 +Subproject commit 1fb6d975a45010b3de0032d51df9388ca4f2def8 From d9b585f6fce813022f17c6a73af237e3285f2902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 17 Dec 2021 15:40:55 +0100 Subject: [PATCH 49/81] modifications for new SDK package in geco --- Makefile | 2 +- README.md | 2 +- .../data-source-plugin.go | 6 +- go.mod | 62 +-------- go.sum | 71 ---------- internal/plugin_test.go | 23 ++-- pkg/geco-data-source.go | 19 --- pkg/i2b2datasource/dataobjects.go | 42 ------ pkg/i2b2datasource/datasource.go | 129 +++++++----------- pkg/i2b2datasource/datasource_test.go | 52 ++----- pkg/i2b2datasource/explore_query.go | 6 +- pkg/i2b2datasource/explore_query_test.go | 20 +-- pkg/i2b2datasource/operations.go | 16 +++ 13 files changed, 108 insertions(+), 342 deletions(-) delete mode 100644 pkg/geco-data-source.go delete mode 100644 pkg/i2b2datasource/dataobjects.go create mode 100644 pkg/i2b2datasource/operations.go diff --git a/Makefile b/Makefile index 0a5af5e..2c2c3d3 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ clean: rm -f ./build/geco-i2b2-data-source.so # --- GeCo -.PHONY: geco-start-dev-local-3nodes geco-swagger-gen +.PHONY: geco-docker-compose geco-swagger-gen geco-docker-compose: make -C third_party/geco/deployments/dev-local-3nodes docker-compose ARGS="$(ARGS)" geco-swagger-gen: diff --git a/README.md b/README.md index 2f8389e..d7c8cc9 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ make go-build-plugin ``` ## Test -### Start depdencies +### Start dependencies Start GeCo (only the database for the moment): ```shell make geco-docker-compose ARGS="up -d postgresql" diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go index a1ff663..34e17ce 100644 --- a/cmd/geco-i2b2-data-source/data-source-plugin.go +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -1,9 +1,9 @@ package main import ( - "github.com/ldsec/geco-i2b2-data-source/pkg" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource" + "github.com/ldsec/geco/pkg/sdk" ) -// DataSourcePlugin exports an instance of the GeCo i2b2 data source for the plugin. -var DataSourcePlugin pkg.DataSource = &i2b2datasource.I2b2DataSource{} +// DataSourcePluginFactory exports a factory function compatible with GeCo data source plugin SDK. +var DataSourcePluginFactory sdk.DataSourcePluginFactory = i2b2datasource.NewI2b2DataSource diff --git a/go.mod b/go.mod index cb86375..45e600b 100644 --- a/go.mod +++ b/go.mod @@ -6,74 +6,14 @@ replace github.com/ldsec/geco => ./third_party/geco require ( github.com/ldsec/geco v0.0.2-0.20211215101444-5aa7d423be0d - github.com/mitchellh/mapstructure v1.4.3 + github.com/lib/pq v1.9.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 ) require ( - github.com/Nerzal/gocloak/v9 v9.0.3 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.34.28 // indirect - github.com/creasty/defaults v1.5.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect - github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect - github.com/go-openapi/analysis v0.20.1 // indirect - github.com/go-openapi/errors v0.20.1 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect - github.com/go-openapi/loads v0.20.3 // indirect - github.com/go-openapi/runtime v0.20.0 // indirect - github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/strfmt v0.20.3 // indirect - github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-openapi/validate v0.20.3 // indirect - github.com/go-playground/locales v0.13.0 // indirect - github.com/go-playground/universal-translator v0.17.0 // indirect - github.com/go-playground/validator v9.31.0+incompatible // indirect - github.com/go-resty/resty/v2 v2.3.0 // indirect - github.com/go-stack/stack v1.8.1 // indirect - github.com/goccy/go-json v0.7.10 // indirect - github.com/google/uuid v1.1.1 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/ldsec/lattigo/v2 v2.0.0 // indirect - github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 // indirect - github.com/leodido/go-urn v1.2.1 // indirect - github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect - github.com/lestrrat-go/blackmagic v1.0.0 // indirect - github.com/lestrrat-go/httpcc v1.0.0 // indirect - github.com/lestrrat-go/iter v1.0.1 // indirect - github.com/lestrrat-go/jwx v1.2.9 // indirect - github.com/lestrrat-go/option v1.0.0 // indirect - github.com/lib/pq v1.9.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/montanaflynn/stats v0.6.3 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/philippgille/gokv v0.6.0 // indirect - github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 // indirect - github.com/philippgille/gokv/file v0.6.0 // indirect - github.com/philippgille/gokv/s3 v0.6.0 // indirect - github.com/philippgille/gokv/syncmap v0.6.0 // indirect - github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/segmentio/ksuid v1.0.3 // indirect - go.dedis.ch/fixbuf v1.0.3 // indirect - go.dedis.ch/kyber/v3 v3.0.13 // indirect - go.dedis.ch/onet/v3 v3.2.3 // indirect - go.mongodb.org/mongo-driver v1.7.3 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - gonum.org/v1/gonum v0.7.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 23d7fba..2802571 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Cryptolens/cryptolens-golang v0.0.0-20210322162237-d3ae4c72721f/go.mod h1:zVXpFHn6wrg4t0PA7g2XT46hlspli2/g7vC9HRXvfwA= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/Nerzal/gocloak/v9 v9.0.3 h1:45Jiw+H2TIrSsMboO4t6bUozMAzxeQDcwIWvnJ/KJsU= github.com/Nerzal/gocloak/v9 v9.0.3/go.mod h1:AmXlpCliq31R1r/18ei7/ML5mfJDmJ1pTBoFU79DDZg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -22,10 +19,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -42,19 +37,14 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM= github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 h1:d/cVoZOrJPJHKH1NdeUjyVAWKp4OpOT+Q+6T1sH7jeU= github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d h1:1iy2qD6JEhHKKhUOA9IWs7mjco7lnw2qx8FsRI2wirE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -85,7 +75,6 @@ github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= -github.com/go-openapi/analysis v0.20.1 h1:zdVbw8yoD4SWZeq+cWdGgquaB0W4VrsJvDJHJND/Ktc= github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= @@ -95,20 +84,17 @@ github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8= github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -120,7 +106,6 @@ github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hs github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= -github.com/go-openapi/loads v0.20.3 h1:VnuSSPx0bbSmSLUwltC6ss45tWyWzfvIeAeCk73B6N4= github.com/go-openapi/loads v0.20.3/go.mod h1:r3u+N8rngPey6DHjYj9G4Wf61heNZjTQX2UjdIvUbn0= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= @@ -128,7 +113,6 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/runtime v0.20.0 h1:DEV4oYH28MqakaabtbxH0cjvlzFegi/15kfUVCfiZW0= github.com/go-openapi/runtime v0.20.0/go.mod h1:2WnLRxMiOUWNN0UZskSkxW0+WXdfB1KmqRKCFH+ZWYk= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -140,7 +124,6 @@ github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFu github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -152,7 +135,6 @@ github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= -github.com/go-openapi/strfmt v0.20.3 h1:YVG4ZgPZ00km/lRHrIf7c6cKL5/4FAUtG2T9RxWAgDY= github.com/go-openapi/strfmt v0.20.3/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -163,7 +145,6 @@ github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -173,21 +154,14 @@ github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0 github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= -github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= -github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So= github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= @@ -216,7 +190,6 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec= github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -230,17 +203,14 @@ github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c/go.mod h1:cHMBumiwaaRxRQ6NT8sU3zQSkXbYaPjbBcXa8UgTzAE= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -255,13 +225,10 @@ github.com/interpose/middleware v0.0.0-20150216143757-05ed56ed52fa/go.mod h1:eMb github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -285,23 +252,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/ldsec/lattigo/v2 v2.0.0 h1:P3rzbXFrc45swwl0u/zVCfcSDuIjTpIGueEDVQQzgEY= github.com/ldsec/lattigo/v2 v2.0.0/go.mod h1:Iu8ol3XIWyF54Ka5bBBWNnNR9kYtymmuYo1Y6Hxzr+U= -github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 h1:RCds6LS6RVHUI/vw64tgVoc/v0qtvtOygSBS2YEo/Bk= github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4/go.mod h1:/uvhNx7QKG/gNJh5pWYSNWhbO4xffJ6SC4oV91yftp4= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= -github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4= github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= -github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc= github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= -github.com/lestrrat-go/iter v1.0.1 h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A= github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= -github.com/lestrrat-go/jwx v1.2.9 h1:kS8kLI4oaBYJJ6u6rpbPI0tDYVCqo0P5u8vv1zoQ49U= github.com/lestrrat-go/jwx v1.2.9/go.mod h1:25DcLbNWArPA/Ew5CcBmewl32cJKxOk5cbepBsIJFzw= -github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -313,7 +271,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= @@ -328,18 +285,13 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk= github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -348,24 +300,16 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/philippgille/gokv v0.0.0-20191001201555-5ac9a20de634/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= github.com/philippgille/gokv v0.5.1-0.20191011213304-eb77f15b9c61/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= -github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK2c= github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= -github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI= github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= -github.com/philippgille/gokv/file v0.6.0 h1:ySYotRmkwaJLDkNSdT7Q0iDQzKHhSdq+ornlBXWgKzI= github.com/philippgille/gokv/file v0.6.0/go.mod h1:L5ulK3F64mxW+8OvYFGE5bowupGO73JdQBh4qE2bgEw= -github.com/philippgille/gokv/s3 v0.6.0 h1:AdQv7H7P73TdHW1h0i6fFgK7uKTwYhYx66vmsnvRoIo= github.com/philippgille/gokv/s3 v0.6.0/go.mod h1:VX24mhIyCpdir7WrhUo1yJGuw6aSbNIuDgjUJGVhqKA= -github.com/philippgille/gokv/syncmap v0.6.0 h1:2eWC2J6mTyUsl687WuGoYPIiyqFiTBZU7hSKPlr0mK4= github.com/philippgille/gokv/syncmap v0.6.0/go.mod h1:ZekkiO1XY9XbjRv9iunxA6+POW9Tw/QHsLO9xAHEaxo= -github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= -github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o= github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= github.com/phyber/negroni-gzip v1.0.0/go.mod h1:poOYjiFVKpeib8SnUpOgfQGStKNGLKsM8l09lOTNeyw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -384,7 +328,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -414,7 +357,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -433,14 +375,11 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= -go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= go.dedis.ch/kyber/v3 v3.0.12/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= -go.dedis.ch/kyber/v3 v3.0.13 h1:s5Lm8p2/CsTMueQHCN24gPpZ4couBBeKU7r2Yl6r32o= go.dedis.ch/kyber/v3 v3.0.13/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= -go.dedis.ch/onet/v3 v3.2.3 h1:Ps05qwaT/In9WgoJxJimKW7FBpkGAxj+AkDIhe+HdAs= go.dedis.ch/onet/v3 v3.2.3/go.mod h1:rKEuXJJiH44kj1VwlgmjT9PnqVItC5jos1W4/t1TMXQ= go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= @@ -455,7 +394,6 @@ go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= -go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -476,11 +414,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -511,7 +447,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -554,7 +489,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -576,12 +510,9 @@ golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -596,7 +527,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/satori/go.uuid.v1 v1.2.0/go.mod h1:kjjdhYBBaa5W5DYP+OcVG3fRM6VWu14hqDYST4Zvw+E= @@ -608,7 +538,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/plugin_test.go b/internal/plugin_test.go index cef5a45..c560c7e 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -4,9 +4,7 @@ import ( "plugin" "testing" - "github.com/ldsec/geco-i2b2-data-source/pkg" - "github.com/ldsec/geco/pkg/common/configuration" - "github.com/ldsec/geco/pkg/datamanager" + "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -15,14 +13,14 @@ func TestPlugin(t *testing.T) { p, err := plugin.Open("../build/geco-i2b2-data-source.so") require.NoError(t, err) - dsSymbol, err := p.Lookup("DataSourcePlugin") + dsSymbol, err := p.Lookup("DataSourcePluginFactory") require.NoError(t, err) - ds, ok := dsSymbol.(*pkg.DataSource) - require.True(t, ok) + pluginFactory, ok := dsSymbol.(*sdk.DataSourcePluginFactory) + t.Logf("%T", pluginFactory) + t.Logf("%T", dsSymbol) - dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) - require.NoError(t, err) + require.True(t, ok) config := make(map[string]string) config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" @@ -34,12 +32,11 @@ func TestPlugin(t *testing.T) { config["i2b2.api.ont-max-elements"] = "200" logrus.StandardLogger().SetLevel(logrus.DebugLevel) - err = (*ds).Init(dm, logrus.StandardLogger(), config) + ds, err := (*pluginFactory)(logrus.StandardLogger(), config) require.NoError(t, err) - params := make(map[string]interface{}) - params["path"] = "/" - params["operation"] = "children" - _, err = (*ds).Query("test", "searchConcept", params, nil) + params := `{"path": "/", "operation": "children"}` + res, _, err := ds.Query("testUser", "searchConcept", []byte(params)) require.NoError(t, err) + t.Logf("result: %v", string(res)) } diff --git a/pkg/geco-data-source.go b/pkg/geco-data-source.go deleted file mode 100644 index b6f18ad..0000000 --- a/pkg/geco-data-source.go +++ /dev/null @@ -1,19 +0,0 @@ -package pkg - -import ( - "github.com/ldsec/geco/pkg/datamanager" - "github.com/sirupsen/logrus" -) - -// todo: move this interface to geco once finalized, e.g. sdk - -// DataSource defines a GeCo data source plugin. The plugin must export a variable named DataSourcePlugin of the -// type DataSource to be compatible. -type DataSource interface { - - // Init the data source with the provided configuration. - Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) error - - // Query data source with a specific operation. - Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) -} diff --git a/pkg/i2b2datasource/dataobjects.go b/pkg/i2b2datasource/dataobjects.go deleted file mode 100644 index 6a48e3f..0000000 --- a/pkg/i2b2datasource/dataobjects.go +++ /dev/null @@ -1,42 +0,0 @@ -package i2b2datasource - -import ( - "fmt" - - "github.com/ldsec/geco/pkg/datamanager" - gecomodels "github.com/ldsec/geco/pkg/models" -) - -// todo: geco data manager needs to allow data objects that are int and int vectors, it is now temporarily stored as float! - -// storeIntValue stores an integer value in the data manager. -func (ds I2b2DataSource) storeIntValue(value uint64, doSharedID string) error { - - do := datamanager.NewFloatVector([]float64{float64(value)}) // todo: need to be different type of data object - - doID, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false) - if err != nil { - return fmt.Errorf("adding data object: %v", err) - } - ds.logger.Infof("added data object to data manager (%v)", doID) - - return nil -} - -// storeIntVector stores a vector of integers in the data manager. -func (ds I2b2DataSource) storeIntVector(values []uint64, doSharedID string) error { - - var floats []float64 - for _, value := range values { - floats = append(floats, float64(value)) - } - do := datamanager.NewFloatVector(floats) // todo: need to be different type of data object - - doID, err := ds.dm.AddDataObjectWithSharedID(do, gecomodels.DataObjectSharedID(doSharedID), false) - if err != nil { - return fmt.Errorf("adding data object: %v", err) - } - ds.logger.Infof("added data object to data manager (%v)", doID) - - return nil -} diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go index aff2cba..871bb63 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/i2b2datasource/datasource.go @@ -1,62 +1,30 @@ package i2b2datasource import ( + "encoding/json" "fmt" "time" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api" i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" - "github.com/ldsec/geco/pkg/datamanager" - "github.com/mitchellh/mapstructure" + gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" ) -// Operation is an operation of the data source supported by I2b2DataSource.Query. -type Operation string +// compile-time check that I2b2DataSource implements the interface sdk.DataSourcePlugin +var _ gecosdk.DataSourcePlugin = (*I2b2DataSource)(nil) -// Enumerated values for Operation. +// Names of result data objects. const ( - OperationSearchConcept Operation = "searchConcept" - OperationSearchModifier Operation = "searchModifier" - OperationExploreQuery Operation = "exploreQuery" - OperationGetCohorts Operation = "getCohorts" - OperationAddCohort Operation = "addCohort" - OperationDeleteCohort Operation = "deleteCohort" - OperationSurvivalQuery Operation = "survivalQuery" - OperationSearchOntology Operation = "searchOntology" + resultNameExploreQueryCount string = "count" + resultNameExploreQueryPatientList string = "patientList" ) -// Values of identifiers for data objects shared IDs. -const ( - sharedIDExploreQueryCount string = "count" - sharedIDExploreQueryPatientList string = "patientList" -) - -// I2b2DataSource is an i2b2 data source for GeCo. It implements the data source interface. -type I2b2DataSource struct { - - // init is true if the I2b2DataSource has been initialized. - init bool - - // dm is the GeCo data manager - dm *datamanager.DataManager - - // logger is the logger from GeCo - logger logrus.FieldLogger - - // i2b2Client is the i2b2 client - i2b2Client i2b2api.Client - - // i2b2OntMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 - i2b2OntMaxElements string -} - -// Init implements the data source interface Init function. -func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldLogger, config map[string]string) (err error) { - fmt.Println("called init") - - ds.dm = dm +// NewI2b2DataSource creates an i2b2 data source. +// Implements sdk.DataSourcePluginFactory. +func NewI2b2DataSource(logger logrus.FieldLogger, config map[string]string) (plugin gecosdk.DataSourcePlugin, err error) { + ds := new(I2b2DataSource) ds.logger = logger // todo: config keys @@ -76,7 +44,7 @@ func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldL if ci.WaitTime, err = time.ParseDuration(config["i2b2.api.wait-time"]); err != nil { err = fmt.Errorf("parsing i2b2 wait time: %v", err) logger.Error(err) - return err + return nil, err } ds.i2b2Client = i2b2api.Client{ @@ -85,64 +53,65 @@ func (ds *I2b2DataSource) Init(dm *datamanager.DataManager, logger logrus.FieldL } ds.i2b2OntMaxElements = config["i2b2.api.ont-max-elements"] - ds.init = true ds.logger.Infof("initialized i2b2 data source for %v", ci.HiveURL) - return nil + return ds, nil +} + +// I2b2DataSource is an i2b2 data source for GeCo. It implements the data source interface. +type I2b2DataSource struct { + + // logger is the logger from GeCo + logger logrus.FieldLogger + + // i2b2Client is the i2b2 client + i2b2Client i2b2api.Client + + // i2b2OntMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 + i2b2OntMaxElements string } // Query implements the data source interface Query function. -func (ds I2b2DataSource) Query(userID string, operation string, parameters map[string]interface{}, resultsSharedIds map[string]string) (results map[string]interface{}, err error) { - if !ds.init { - panic(fmt.Errorf("data source is not initialized")) - } +func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters []byte) (jsonResults []byte, resultDataObjects map[string]gecosdk.DataObject, err error) { ds.logger.Infof("executing operation %v for user %v", operation, userID) - ds.logger.Debugf("parameters: %+v", parameters) - ds.logger.Debugf("resultsSharedIds: %+v", resultsSharedIds) + ds.logger.Debugf("parameters: %v", string(jsonParameters)) - // todo: decoder might need squash param set - - results = make(map[string]interface{}) switch Operation(operation) { case OperationSearchConcept: decodedParams := &models.SearchConceptParameters{} - if err := mapstructure.Decode(parameters, decodedParams); err != nil { - return nil, ds.logError("decoding parameters", err) + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, ds.logError("decoding parameters", err) } else if searchResults, err := ds.SearchConcept(decodedParams); err != nil { - return nil, ds.logError("executing query", err) - } else if err := mapstructure.Decode(searchResults, &results); err != nil { - return nil, ds.logError("encoding results", err) + return nil, nil, ds.logError("executing query", err) + } else if jsonResults, err = json.Marshal(searchResults); err != nil { + return nil, nil, ds.logError("encoding results", err) } case OperationSearchModifier: decodedParams := &models.SearchModifierParameters{} - if err := mapstructure.Decode(parameters, decodedParams); err != nil { - return nil, ds.logError("decoding parameters", err) + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, ds.logError("decoding parameters", err) } else if searchResults, err := ds.SearchModifier(decodedParams); err != nil { - return nil, ds.logError("executing query", err) - } else if err := mapstructure.Decode(searchResults, &results); err != nil { - return nil, ds.logError("encoding results", err) + return nil, nil, ds.logError("executing query", err) + } else if jsonResults, err = json.Marshal(searchResults); err != nil { + return nil, nil, ds.logError("encoding results", err) } case OperationExploreQuery: - countSharedID, countOK := resultsSharedIds[sharedIDExploreQueryCount] - patientListSharedID, patientListOK := resultsSharedIds[sharedIDExploreQueryPatientList] - if !countOK || !patientListOK { - return nil, ds.logError("missing results shared ID", nil) - } - + var count int64 + var patientList []int64 decodedParams := &models.ExploreQueryParameters{} - if err := mapstructure.Decode(parameters, decodedParams); err != nil { - return nil, ds.logError("decoding parameters", err) - } else if count, patientList, err := ds.ExploreQuery(decodedParams); err != nil { - return nil, ds.logError("executing query", err) - } else if err := ds.storeIntValue(count, countSharedID); err != nil { - return nil, ds.logError("storing count", err) - } else if err := ds.storeIntVector(patientList, patientListSharedID); err != nil { - return nil, ds.logError("storing patient list", err) + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, ds.logError("decoding parameters", err) + } else if count, patientList, err = ds.ExploreQuery(decodedParams); err != nil { + return nil, nil, ds.logError("executing query", err) } + resultDataObjects = make(map[string]gecosdk.DataObject, 2) + resultDataObjects[resultNameExploreQueryCount] = gecosdk.DataObject{IntValue: &count} + resultDataObjects[resultNameExploreQueryPatientList] = gecosdk.DataObject{IntVector: patientList} + default: - return nil, ds.logError(fmt.Sprintf("unknown query requested (%v)", operation), nil) + return nil, nil, ds.logError(fmt.Sprintf("unknown query requested (%v)", operation), nil) } return } diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go index 828526b..0b8bb25 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/i2b2datasource/datasource_test.go @@ -3,18 +3,11 @@ package i2b2datasource import ( "testing" - "github.com/ldsec/geco/pkg/common/configuration" - "github.com/ldsec/geco/pkg/datamanager" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) -func getTestDataSource(t *testing.T) I2b2DataSource { - ds := I2b2DataSource{} - - dm, err := datamanager.NewDataManager(configuration.NewTestDataManagerConfig()) - require.NoError(t, err) - +func getTestDataSource(t *testing.T) *I2b2DataSource { config := make(map[string]string) config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" config["i2b2.api.domain"] = "i2b2demo" @@ -25,43 +18,26 @@ func getTestDataSource(t *testing.T) I2b2DataSource { config["i2b2.api.ont-max-elements"] = "200" logrus.StandardLogger().SetLevel(logrus.DebugLevel) - err = ds.Init(dm, logrus.StandardLogger(), config) + ds, err := NewI2b2DataSource(logrus.StandardLogger(), config) require.NoError(t, err) - return ds + return ds.(*I2b2DataSource) } -func TestDataManager(t *testing.T) { - +func TestQuery(t *testing.T) { ds := getTestDataSource(t) - - doId, err := ds.dm.AddDataObject(datamanager.NewFloatVector([]float64{0, 1, 2, 3, 4}), false) - require.NoError(t, err) - t.Logf("test data object ID is %v", doId) - - do, err := ds.dm.GetDataObject(doId) - require.NoError(t, err) - require.Equal(t, doId, do.ID) - require.Equal(t, datamanager.FloatVector, do.Type) - - data, err := do.FloatVector() + params := `{"path": "/", "operation": "children"}` + res, _, err := ds.Query("testUser", "searchConcept", []byte(params)) require.NoError(t, err) - require.InDeltaSlice(t, []float64{0, 1, 2, 3, 4}, data, 0.0001) + t.Logf("result: %v", string(res)) } -func TestQuery(t *testing.T) { +func TestQueryDataObject(t *testing.T) { ds := getTestDataSource(t) - params := make(map[string]interface{}) - params["path"] = "/" - params["operation"] = "children" - res, err := ds.Query("testUser", "searchConcept", params, nil) + params := `{"id": "0", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}` + res, do, err := ds.Query("testUser", "exploreQuery", []byte(params)) require.NoError(t, err) - t.Logf("result: %+v", res) - t.Logf("result: %T", res["SearchResults"]) - - // todo - //t.Logf(res["SearchResults"].([]interface{})[0].(string)) - // require.EqualValues(t, ) - - //res["searchResult"] - + require.EqualValues(t, 3, *do["count"].IntValue) + require.InDeltaSlice(t, []int64{1, 2, 3}, do["patientList"].IntVector, 0.001) + t.Logf("result: %v", string(res)) + t.Logf("do: %+v", do) } diff --git a/pkg/i2b2datasource/explore_query.go b/pkg/i2b2datasource/explore_query.go index cc3456f..38e491f 100644 --- a/pkg/i2b2datasource/explore_query.go +++ b/pkg/i2b2datasource/explore_query.go @@ -9,17 +9,17 @@ import ( ) // ExploreQuery makes an explore query, i.e. two i2b2 CRC queries, a PSM and a PDO query. -func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (patientCount uint64, patientList []uint64, err error) { +func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (patientCount int64, patientList []int64, err error) { if i2b2PatientCount, i2b2PatientSetID, err := ds.doExploreQuery(params); err != nil { return 0, nil, err } else if i2b2PatientIDs, err := ds.getPatientIDs(i2b2PatientSetID); err != nil { return 0, nil, err - } else if patientCount, err = strconv.ParseUint(i2b2PatientCount, 10, 64); err != nil { + } else if patientCount, err = strconv.ParseInt(i2b2PatientCount, 10, 64); err != nil { return 0, nil, fmt.Errorf("parsing patient count: %v", err) } else { for _, patientID := range i2b2PatientIDs { - parsedPatientID, err := strconv.ParseUint(patientID, 10, 64) + parsedPatientID, err := strconv.ParseInt(patientID, 10, 64) if err != nil { return 0, nil, fmt.Errorf("parsing patient ID: %v", err) } diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/i2b2datasource/explore_query_test.go index 1466746..124602c 100644 --- a/pkg/i2b2datasource/explore_query_test.go +++ b/pkg/i2b2datasource/explore_query_test.go @@ -24,7 +24,7 @@ func TestExploreQueryConcept(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 3, count) - require.Subset(t, []uint64{1, 2, 3}, patientList) + require.Subset(t, []int64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", @@ -40,7 +40,7 @@ func TestExploreQueryConcept(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 4, count) - require.Subset(t, []uint64{1, 2, 3, 4}, patientList) + require.Subset(t, []int64{1, 2, 3, 4}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "2", @@ -66,7 +66,7 @@ func TestExploreQueryConcept(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 3, count) - require.Subset(t, []uint64{1, 2, 3}, patientList) + require.Subset(t, []int64{1, 2, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "3", @@ -84,7 +84,7 @@ func TestExploreQueryConcept(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 4, count) - require.Subset(t, []uint64{1, 2, 3, 4}, patientList) + require.Subset(t, []int64{1, 2, 3, 4}, patientList) } func TestExploreQueryConceptValue(t *testing.T) { @@ -107,7 +107,7 @@ func TestExploreQueryConceptValue(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 1, count) - require.Subset(t, []uint64{1}, patientList) + require.Subset(t, []int64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", @@ -131,7 +131,7 @@ func TestExploreQueryConceptValue(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 2, count) - require.Subset(t, []uint64{1, 4}, patientList) + require.Subset(t, []int64{1, 4}, patientList) } func TestExploreQueryModifier(t *testing.T) { @@ -158,7 +158,7 @@ func TestExploreQueryModifier(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 2, count) - require.Subset(t, []uint64{1, 3}, patientList) + require.Subset(t, []int64{1, 3}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", @@ -190,7 +190,7 @@ func TestExploreQueryModifier(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 3, count) - require.Subset(t, []uint64{1, 2, 3}, patientList) + require.Subset(t, []int64{1, 2, 3}, patientList) } func TestExploreQueryModifierValue(t *testing.T) { @@ -220,7 +220,7 @@ func TestExploreQueryModifierValue(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 1, count) - require.Subset(t, []uint64{1}, patientList) + require.Subset(t, []int64{1}, patientList) count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", @@ -261,5 +261,5 @@ func TestExploreQueryModifierValue(t *testing.T) { require.NoError(t, err) t.Logf("results: count=%v, patientList=%v", count, patientList) require.EqualValues(t, 1, count) - require.Subset(t, []uint64{2}, patientList) + require.Subset(t, []int64{2}, patientList) } diff --git a/pkg/i2b2datasource/operations.go b/pkg/i2b2datasource/operations.go new file mode 100644 index 0000000..360dd17 --- /dev/null +++ b/pkg/i2b2datasource/operations.go @@ -0,0 +1,16 @@ +package i2b2datasource + +// Operation is an operation of the data source supported by I2b2DataSource.Query. +type Operation string + +// Enumerated values for Operation. +const ( + OperationSearchConcept Operation = "searchConcept" + OperationSearchModifier Operation = "searchModifier" + OperationExploreQuery Operation = "exploreQuery" + OperationGetCohorts Operation = "getCohorts" + OperationAddCohort Operation = "addCohort" + OperationDeleteCohort Operation = "deleteCohort" + OperationSurvivalQuery Operation = "survivalQuery" + OperationSearchOntology Operation = "searchOntology" +) From 5fac9eb15044b504adef296584cda36ac7e6c6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 17 Dec 2021 17:21:27 +0100 Subject: [PATCH 50/81] update to new SDK --- go.mod | 22 ++++++++++++++++++++++ go.sum | 25 +++++++++++++++++++++++++ internal/plugin_test.go | 2 +- pkg/i2b2datasource/datasource.go | 27 ++++++++++++++++++++------- pkg/i2b2datasource/datasource_test.go | 18 ++++++++++++++---- third_party/geco | 2 +- 6 files changed, 83 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 45e600b..5937e81 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,30 @@ require ( ) require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-openapi/analysis v0.20.1 // indirect + github.com/go-openapi/errors v0.20.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/loads v0.20.3 // indirect + github.com/go-openapi/runtime v0.20.0 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/strfmt v0.20.3 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-openapi/validate v0.20.3 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + go.mongodb.org/mongo-driver v1.7.3 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect + golang.org/x/text v0.3.7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 2802571..3e18f37 100644 --- a/go.sum +++ b/go.sum @@ -5,7 +5,9 @@ github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/Nerzal/gocloak/v9 v9.0.3/go.mod h1:AmXlpCliq31R1r/18ei7/ML5mfJDmJ1pTBoFU79DDZg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -19,6 +21,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= @@ -75,6 +78,7 @@ github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.20.1 h1:zdVbw8yoD4SWZeq+cWdGgquaB0W4VrsJvDJHJND/Ktc= github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= @@ -84,17 +88,20 @@ github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8= github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -106,6 +113,7 @@ github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hs github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= +github.com/go-openapi/loads v0.20.3 h1:VnuSSPx0bbSmSLUwltC6ss45tWyWzfvIeAeCk73B6N4= github.com/go-openapi/loads v0.20.3/go.mod h1:r3u+N8rngPey6DHjYj9G4Wf61heNZjTQX2UjdIvUbn0= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= @@ -113,6 +121,7 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.20.0 h1:DEV4oYH28MqakaabtbxH0cjvlzFegi/15kfUVCfiZW0= github.com/go-openapi/runtime v0.20.0/go.mod h1:2WnLRxMiOUWNN0UZskSkxW0+WXdfB1KmqRKCFH+ZWYk= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -124,6 +133,7 @@ github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFu github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -135,6 +145,7 @@ github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/strfmt v0.20.3 h1:YVG4ZgPZ00km/lRHrIf7c6cKL5/4FAUtG2T9RxWAgDY= github.com/go-openapi/strfmt v0.20.3/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -145,6 +156,7 @@ github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -154,6 +166,7 @@ github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0 github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= +github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -161,6 +174,7 @@ github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+Pugk github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -209,8 +223,10 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -229,6 +245,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -271,6 +288,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= @@ -284,6 +302,7 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= @@ -291,6 +310,7 @@ github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFW github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -357,6 +377,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -394,6 +415,7 @@ go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= +go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -447,6 +469,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -489,6 +512,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -538,6 +562,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/plugin_test.go b/internal/plugin_test.go index c560c7e..c11dce8 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -36,7 +36,7 @@ func TestPlugin(t *testing.T) { require.NoError(t, err) params := `{"path": "/", "operation": "children"}` - res, _, err := ds.Query("testUser", "searchConcept", []byte(params)) + res, _, err := ds.Query("testUser", "searchConcept", []byte(params), nil) require.NoError(t, err) t.Logf("result: %v", string(res)) } diff --git a/pkg/i2b2datasource/datasource.go b/pkg/i2b2datasource/datasource.go index 871bb63..2aa082f 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/i2b2datasource/datasource.go @@ -8,6 +8,7 @@ import ( "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api" i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + gecomodels "github.com/ldsec/geco/pkg/models" gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" ) @@ -15,10 +16,10 @@ import ( // compile-time check that I2b2DataSource implements the interface sdk.DataSourcePlugin var _ gecosdk.DataSourcePlugin = (*I2b2DataSource)(nil) -// Names of result data objects. +// Names of output data objects. const ( - resultNameExploreQueryCount string = "count" - resultNameExploreQueryPatientList string = "patientList" + outputNameExploreQueryCount gecosdk.OutputDataObjectName = "count" + outputNameExploreQueryPatientList gecosdk.OutputDataObjectName = "patientList" ) // NewI2b2DataSource creates an i2b2 data source. @@ -71,7 +72,7 @@ type I2b2DataSource struct { } // Query implements the data source interface Query function. -func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters []byte) (jsonResults []byte, resultDataObjects map[string]gecosdk.DataObject, err error) { +func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters []byte, outputDataObjectsSharedIDs map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (jsonResults []byte, outputDataObjects []gecosdk.DataObject, err error) { ds.logger.Infof("executing operation %v for user %v", operation, userID) ds.logger.Debugf("parameters: %v", string(jsonParameters)) @@ -97,6 +98,10 @@ func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters [ } case OperationExploreQuery: + if outputDataObjectsSharedIDs[outputNameExploreQueryCount] == "" || outputDataObjectsSharedIDs[outputNameExploreQueryPatientList] == "" { + return nil, nil, ds.logError("missing output data object name", nil) + } + var count int64 var patientList []int64 decodedParams := &models.ExploreQueryParameters{} @@ -106,9 +111,17 @@ func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters [ return nil, nil, ds.logError("executing query", err) } - resultDataObjects = make(map[string]gecosdk.DataObject, 2) - resultDataObjects[resultNameExploreQueryCount] = gecosdk.DataObject{IntValue: &count} - resultDataObjects[resultNameExploreQueryPatientList] = gecosdk.DataObject{IntVector: patientList} + outputDataObjects = []gecosdk.DataObject{ + { + OutputName: outputNameExploreQueryCount, + SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryCount], + IntValue: &count, + }, { + OutputName: outputNameExploreQueryPatientList, + SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryPatientList], + IntVector: patientList, + }, + } default: return nil, nil, ds.logError(fmt.Sprintf("unknown query requested (%v)", operation), nil) diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/i2b2datasource/datasource_test.go index 0b8bb25..7942e30 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/i2b2datasource/datasource_test.go @@ -3,6 +3,8 @@ package i2b2datasource import ( "testing" + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -26,7 +28,7 @@ func getTestDataSource(t *testing.T) *I2b2DataSource { func TestQuery(t *testing.T) { ds := getTestDataSource(t) params := `{"path": "/", "operation": "children"}` - res, _, err := ds.Query("testUser", "searchConcept", []byte(params)) + res, _, err := ds.Query("testUser", "searchConcept", []byte(params), nil) require.NoError(t, err) t.Logf("result: %v", string(res)) } @@ -34,10 +36,18 @@ func TestQuery(t *testing.T) { func TestQueryDataObject(t *testing.T) { ds := getTestDataSource(t) params := `{"id": "0", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}` - res, do, err := ds.Query("testUser", "exploreQuery", []byte(params)) + sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{outputNameExploreQueryCount: "countSharedID", outputNameExploreQueryPatientList: "patientListSharedID"} + res, do, err := ds.Query("testUser", "exploreQuery", []byte(params), sharedIDs) require.NoError(t, err) - require.EqualValues(t, 3, *do["count"].IntValue) - require.InDeltaSlice(t, []int64{1, 2, 3}, do["patientList"].IntVector, 0.001) + + require.EqualValues(t, 3, *do[0].IntValue) + require.EqualValues(t, "countSharedID", do[0].SharedID) + require.EqualValues(t, outputNameExploreQueryCount, do[0].OutputName) + + require.InDeltaSlice(t, []int64{1, 2, 3}, do[1].IntVector, 0.001) + require.EqualValues(t, "patientListSharedID", do[1].SharedID) + require.EqualValues(t, outputNameExploreQueryPatientList, do[1].OutputName) + t.Logf("result: %v", string(res)) t.Logf("do: %+v", do) } diff --git a/third_party/geco b/third_party/geco index 1fb6d97..35f2874 160000 --- a/third_party/geco +++ b/third_party/geco @@ -1 +1 @@ -Subproject commit 1fb6d975a45010b3de0032d51df9388ca4f2def8 +Subproject commit 35f2874b93b7975991f66f1832cac6ac0fbdf80f From 0e7ccd012fd61b7c9894aacf7783ae0ab76a4fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 17 Dec 2021 17:22:13 +0100 Subject: [PATCH 51/81] cleanup --- pkg/i2b2datasource/models/common.go | 25 ------------------------- pkg/i2b2datasource/models/search.go | 24 ------------------------ 2 files changed, 49 deletions(-) delete mode 100644 pkg/i2b2datasource/models/common.go diff --git a/pkg/i2b2datasource/models/common.go b/pkg/i2b2datasource/models/common.go deleted file mode 100644 index 7a33ab9..0000000 --- a/pkg/i2b2datasource/models/common.go +++ /dev/null @@ -1,25 +0,0 @@ -package models - -import "fmt" - -type parseError error - -func recoverParseError(retErr *error) { - if r := recover(); r != nil { - if parseErr, ok := r.(parseError); !ok { - panic(r) // panic was not caused by a parse error - } else { - *retErr = fmt.Errorf("parsing value: %v", parseErr) - } - } -} - -func getString(params map[string]interface{}, key string) string { - if stringVal, ok := params[key]; !ok { - panic(parseError(fmt.Errorf("key \"%v\" not found in params", key))) - } else if stringCast, ok := stringVal.(string); !ok { - panic(parseError(fmt.Errorf("key \"%v\" is not a string (%T)", key, stringVal))) - } else { - return stringCast - } -} diff --git a/pkg/i2b2datasource/models/search.go b/pkg/i2b2datasource/models/search.go index b5719aa..996aeca 100644 --- a/pkg/i2b2datasource/models/search.go +++ b/pkg/i2b2datasource/models/search.go @@ -4,36 +4,12 @@ import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" // --- parameters -// func NewSearchConceptParameters(params map[string]interface{}) (model SearchConceptParameters, retErr error) { -// defer recoverParseError(&retErr) -// -// return SearchConceptParameters{ -// Path: getString(params, "path"), -// Operation: getString(params, "operation"), -// }, nil -// } - // SearchConceptParameters is the parameter for the SearchConcept operation. type SearchConceptParameters struct { Path string Operation string // children | info | concept } -// func NewSearchModifierParameters(params map[string]interface{}) (model SearchModifierParameters, retErr error) { -// defer recoverParseError(&retErr) -// -// return SearchModifierParameters{ -// SearchConceptParameters: SearchConceptParameters{ -// Path: getString(params, "path"), -// Operation: getString(params, "operation"), -// }, -// AppliedPath: getString(params, "appliedPath"), -// AppliedConcept: getString(params, "appliedConcept"), -// }, nil -// } - -// todo: maybe squash model together if OK with serializing etc. - // SearchModifierParameters is the parameter for the SearchModifier operation. type SearchModifierParameters struct { SearchConceptParameters From bb0f53f48efc1f873c862f0b62a4c8ba6828596f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 21 Dec 2021 16:39:15 +0100 Subject: [PATCH 52/81] implement database package --- go.mod | 1 + go.sum | 2 + pkg/i2b2datasource/database/common.go | 47 ++++++ pkg/i2b2datasource/database/ddl.go | 80 ++++++++++ pkg/i2b2datasource/database/ddl_test.go | 65 ++++++++ pkg/i2b2datasource/database/explore_query.go | 114 ++++++++++++++ .../database/explore_query_test.go | 68 ++++++++ pkg/i2b2datasource/database/models.go | 22 +++ pkg/i2b2datasource/database/saved_cohort.go | 146 ++++++++++++++++++ .../database/saved_cohort_test.go | 94 +++++++++++ 10 files changed, 639 insertions(+) create mode 100644 pkg/i2b2datasource/database/common.go create mode 100644 pkg/i2b2datasource/database/ddl.go create mode 100644 pkg/i2b2datasource/database/ddl_test.go create mode 100644 pkg/i2b2datasource/database/explore_query.go create mode 100644 pkg/i2b2datasource/database/explore_query_test.go create mode 100644 pkg/i2b2datasource/database/models.go create mode 100644 pkg/i2b2datasource/database/saved_cohort.go create mode 100644 pkg/i2b2datasource/database/saved_cohort_test.go diff --git a/go.mod b/go.mod index 5937e81..6837b49 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-openapi/swag v0.19.15 // indirect github.com/go-openapi/validate v0.20.3 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect diff --git a/go.sum b/go.sum index 3e18f37..7a91416 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= diff --git a/pkg/i2b2datasource/database/common.go b/pkg/i2b2datasource/database/common.go new file mode 100644 index 0000000..04bdca4 --- /dev/null +++ b/pkg/i2b2datasource/database/common.go @@ -0,0 +1,47 @@ +package database + +import ( + "database/sql" + "fmt" + + // Registering postgres driver + _ "github.com/lib/pq" + "github.com/sirupsen/logrus" +) + +// NewPostgresDatabase initializes a connection to a postgres database, and loads its structure if not present. +func NewPostgresDatabase(logger logrus.FieldLogger, host, port, databaseName, schemaName, userLogin, userPassword string) (db *PostgresDatabase, err error) { + connectionString := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s search_path=%s sslmode=disable", host, port, userLogin, userPassword, databaseName, schemaName) + + logger.Infof("initializing connection to postgres database %v", host) + logger.Debugf("postgres connection string is %v", connectionString) + + dbHandle, err := sql.Open("postgres", connectionString) + if err != nil { + return nil, fmt.Errorf("initializing connection to postgres database: %v", err) + } + + db = &PostgresDatabase{ + logger: logger, + handle: dbHandle, + } + + return db, db.loadDdl(schemaName, userLogin) +} + +// PostgresDatabase is TBD +type PostgresDatabase struct { + logger logrus.FieldLogger + + handle *sql.DB +} + +func closeRows(rows *sql.Rows, logger logrus.FieldLogger) { + if rows.Next() { + logger.Warnf("leftover rows available") + } + + if err := rows.Close(); err != nil { + logger.Errorf("closing result rows: %v", err) + } +} diff --git a/pkg/i2b2datasource/database/ddl.go b/pkg/i2b2datasource/database/ddl.go new file mode 100644 index 0000000..f77c741 --- /dev/null +++ b/pkg/i2b2datasource/database/ddl.go @@ -0,0 +1,80 @@ +package database + +import "fmt" + +const ddlLoaded = ` +SELECT EXISTS ( + SELECT 1 FROM pg_namespace WHERE nspname = $1 +); +` + +const createSchemaFunction = ` +CREATE OR REPLACE FUNCTION public.create_schema(schema_name NAME, user_name NAME) +RETURNS BOOL AS ' + BEGIN + EXECUTE ''CREATE SCHEMA '' || schema_name; + EXECUTE ''GRANT ALL ON SCHEMA '' || schema_name || '' TO '' || user_name; + EXECUTE ''GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA '' || schema_name || '' TO '' || user_name; + RETURN true; + END; +' LANGUAGE PLPGSQL; +` + +const ddlStatement = ` +BEGIN; + CREATE TYPE query_status AS ENUM ('requested', 'running', 'success', 'error'); + + CREATE TABLE IF NOT EXISTS explore_query( + id uuid NOT NULL, + create_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + user_id character varying(255) NOT NULL, + status query_status NOT NULL, + definition TEXT NOT NULL, + + result_i2b2_patient_set_id integer, + result_geco_shared_id_count uuid, + result_geco_shared_id_patient_list uuid, + + PRIMARY KEY (id) + ); + + CREATE TABLE IF NOT EXISTS saved_cohort( + name character varying(255) NOT NULL, + create_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + explore_query_id uuid NOT NULL, + + CONSTRAINT saved_cohort_pkey PRIMARY KEY (name, explore_query_id), + CONSTRAINT saved_cohort_fkey_explore_query FOREIGN KEY (explore_query_id) REFERENCES explore_query(id) + ); +COMMIT; +` + +// loadDdl loads the data structure of the database. +func (db PostgresDatabase) loadDdl(schemaName, userLogin string) (err error) { + + // check if DDL was loaded + var isDdlLoaded bool + if rows, err := db.handle.Query(ddlLoaded, schemaName); err != nil { + return fmt.Errorf("querying ddlLoaded: %v", err) + } else if !rows.Next() { + return fmt.Errorf("no available rows in ddlLoaded: %v", err) + } else if err = rows.Scan(&isDdlLoaded); err != nil { + return fmt.Errorf("scanning rows of ddlLoaded: %v", err) + } else { + closeRows(rows, db.logger) + } + + // load DDL if not already loaded + if !isDdlLoaded { + if _, err = db.handle.Exec(createSchemaFunction); err != nil { + return fmt.Errorf("executing createSchemaFunction: %v", err) + } + if _, err = db.handle.Exec("SELECT public.create_schema($1::name, $2::name);", schemaName, userLogin); err != nil { + return fmt.Errorf("executing create_schema: %v", err) + } + if _, err = db.handle.Exec(ddlStatement); err != nil { + return fmt.Errorf("executing ddlStatement: %v", err) + } + } + return nil +} diff --git a/pkg/i2b2datasource/database/ddl_test.go b/pkg/i2b2datasource/database/ddl_test.go new file mode 100644 index 0000000..72cfece --- /dev/null +++ b/pkg/i2b2datasource/database/ddl_test.go @@ -0,0 +1,65 @@ +package database + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +const unitTestsSchemaName = "gecodatasourceunittest" +const testDataStatement = ` +BEGIN; + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('00000000-0000-0000-0000-000000000000', NOW(), 'testuser1', 'success', '{"query": {}}', + 0, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('11111111-1111-1111-1111-111111111111', NOW(), 'testuser1', 'success', '{"query": {}}', + 1, '11111111-1111-1111-1111-111111111112', '11111111-1111-1111-1111-111111111113'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('22222222-2222-2222-2222-222222222222', NOW(), 'testuser1', 'error', '{"query": {}}', + NULL, NULL, NULL); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('33333333-3333-3333-3333-333333333333', NOW(), 'testuser2', 'success', '{"query": {}}', + 2, '33333333-3333-3333-3333-333333333334', '33333333-3333-3333-3333-333333333335'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('44444444-4444-4444-4444-444444444444', NOW(), 'testuser2', 'success', '{"query": {}}', + 3, '44444444-4444-4444-4444-444444444445', '44444444-4444-4444-4444-444444444446'); + + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort1', NOW(), '11111111-1111-1111-1111-111111111111'); + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort3', NOW(), '33333333-3333-3333-3333-333333333333'); + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort4', NOW(), '44444444-4444-4444-4444-444444444444'); +COMMIT; +` + +func getDB(t *testing.T) *PostgresDatabase { + db, err := NewPostgresDatabase(logrus.StandardLogger(), + "localhost", "5432", "i2b2", unitTestsSchemaName, "postgres", "postgres") + require.NoError(t, err) + + return db +} + +func dbLoadTestData(t *testing.T, db *PostgresDatabase) { + _, err := db.handle.Exec(testDataStatement) + require.NoError(t, err) +} + +func dbCleanup(t *testing.T, db *PostgresDatabase) { + _, err := db.handle.Exec("DROP SCHEMA " + unitTestsSchemaName + " CASCADE;") + require.NoError(t, err) +} + +func TestDDL(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + dbCleanup(t, db) +} diff --git a/pkg/i2b2datasource/database/explore_query.go b/pkg/i2b2datasource/database/explore_query.go new file mode 100644 index 0000000..e37e577 --- /dev/null +++ b/pkg/i2b2datasource/database/explore_query.go @@ -0,0 +1,114 @@ +package database + +import ( + "database/sql" + "fmt" + + "github.com/google/uuid" +) + +// GetExploreQuery retrieves an explore query from the database. +// Returns nil (not an error) if query does not exist. +func (db PostgresDatabase) GetExploreQuery(id string) (query *ExploreQuery, err error) { + const getExploreQueryStatement = ` + SELECT + id, + create_date, + user_id, + status, + definition, + result_i2b2_patient_set_id, + result_geco_shared_id_count, + result_geco_shared_id_patient_list + + FROM explore_query + WHERE id = $1;` + + var rows *sql.Rows + if rows, err = db.handle.Query(getExploreQueryStatement, id); err != nil { + return nil, fmt.Errorf("querying getExploreQueryStatement: %v", err) + } + defer closeRows(rows, db.logger) + + if rows.Next() { + query = new(ExploreQuery) + + err = rows.Scan( + &query.ID, + &query.CreateDate, + &query.UserID, + &query.Status, + &query.Definition, + &query.ResultI2b2PatientSetID, + &query.ResultGecoSharedIDCount, + &query.ResultGecoSharedIDPatientList, + ) + if err != nil { + return nil, fmt.Errorf("scanning row of getExploreQueryStatement: %v", err) + } + } else { + db.logger.Warnf("explore query not found (ID: %v)", id) + } + return +} + +// AddExploreQuery adds an explore query with the requested status. +func (db PostgresDatabase) AddExploreQuery(userID, queryDefinition string) (queryID string, err error) { + const addExploreQueryStatement = ` + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ($1, NOW(), $2, 'requested', $3, NULL, NULL, NULL);` + + queryID = uuid.NewString() + if _, err = db.handle.Exec(addExploreQueryStatement, queryID, userID, queryDefinition); err != nil { + return "", fmt.Errorf("executing addExploreQueryStatement: %v", err) + } + db.logger.Infof("added explore query (ID: %v)", queryID) + return +} + +// SetExploreQueryRunning sets the running status to an explore query. +func (db PostgresDatabase) SetExploreQueryRunning(userID, queryID string) error { + return db.setExploreQueryStatus("running", userID, queryID) +} + +// SetExploreQueryError sets the error status to an explore query. +func (db PostgresDatabase) SetExploreQueryError(userID, queryID string) error { + return db.setExploreQueryStatus("error", userID, queryID) +} + +// SetExploreQuerySuccess sets the success status to an explore query and stores its results. +func (db PostgresDatabase) SetExploreQuerySuccess(userID, queryID, i2b2PatientSetID, gecoSharedIDCount, gecoSharedIDPatientList string) error { + const setExploreQuerySuccessStatement = ` + UPDATE explore_query SET + result_i2b2_patient_set_id = $3, + result_geco_shared_id_count = $4, + result_geco_shared_id_patient_list = $5 + + WHERE user_id = $1 AND id = $2;` + + if err := db.setExploreQueryStatus("success", userID, queryID); err != nil { + return err + } else if _, err := db.handle.Exec(setExploreQuerySuccessStatement, userID, queryID, i2b2PatientSetID, gecoSharedIDCount, gecoSharedIDPatientList); err != nil { + return fmt.Errorf("executing setExploreQuerySuccessStatement: %v", err) + } + + db.logger.Infof("updated explore query results (ID: %v)", queryID) + return nil +} + +// setExploreQueryStatus sets the status of an explore query. +func (db PostgresDatabase) setExploreQueryStatus(status, userID, queryID string) (err error) { + const setExploreQueryStatusStatement = ` + UPDATE explore_query SET status = $1 WHERE user_id = $2 AND id = $3;` + + if res, err := db.handle.Exec(setExploreQueryStatusStatement, status, userID, queryID); err != nil { + return fmt.Errorf("executing setExploreQueryStatusStatement: %v", err) + } else if rowsAffected, err := res.RowsAffected(); err != nil { + return fmt.Errorf("database error: %v", err) + } else if rowsAffected == 0 { + return fmt.Errorf("explore query does not exist") + } + db.logger.Infof("updated explore query status to %v (ID: %v)", status, queryID) + return +} diff --git a/pkg/i2b2datasource/database/explore_query_test.go b/pkg/i2b2datasource/database/explore_query_test.go new file mode 100644 index 0000000..e12db31 --- /dev/null +++ b/pkg/i2b2datasource/database/explore_query_test.go @@ -0,0 +1,68 @@ +package database + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetExploreQuery(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + query1, err := db.GetExploreQuery("11111111-1111-1111-1111-111111111111") + require.NoError(t, err) + require.EqualValues(t, "testuser1", query1.UserID) + t.Logf("%+v", query1) + + query3, err := db.GetExploreQuery("33333333-3333-3333-3333-333333333333") + require.NoError(t, err) + require.EqualValues(t, "testuser2", query3.UserID) + t.Logf("%+v", query3) + + queryNotFound, err := db.GetExploreQuery("11111111-1111-9999-1111-111111111111") + require.NoError(t, err) + require.Nil(t, queryNotFound) +} + +func TestExploreQuery(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + queryID, err := db.AddExploreQuery("testUser5", "{}") + require.NoError(t, err) + require.NotEmpty(t, queryID) + t.Logf("%v", queryID) + + query, err := db.GetExploreQuery(queryID) + require.NoError(t, err) + require.EqualValues(t, "testUser5", query.UserID) + require.EqualValues(t, queryID, query.ID) + require.EqualValues(t, "requested", query.Status) + t.Logf("%+v", query) + + err = db.SetExploreQueryRunning("testUser5", queryID) + require.NoError(t, err) + query, err = db.GetExploreQuery(queryID) + require.NoError(t, err) + require.EqualValues(t, "running", query.Status) + + err = db.SetExploreQueryError("testUser5", queryID) + require.NoError(t, err) + query, err = db.GetExploreQuery(queryID) + require.NoError(t, err) + require.EqualValues(t, "error", query.Status) + + err = db.SetExploreQuerySuccess("testUser5", queryID, "7", "31133333-3333-3333-3333-333333333333", "33333333-3333-3333-3333-333333333223") + require.NoError(t, err) + query, err = db.GetExploreQuery(queryID) + require.NoError(t, err) + require.EqualValues(t, "success", query.Status) + require.EqualValues(t, 7, query.ResultI2b2PatientSetID.Int64) + + err = db.SetExploreQueryRunning("asd", queryID) + require.Error(t, err) + t.Logf("%v", err) +} diff --git a/pkg/i2b2datasource/database/models.go b/pkg/i2b2datasource/database/models.go new file mode 100644 index 0000000..33a69d9 --- /dev/null +++ b/pkg/i2b2datasource/database/models.go @@ -0,0 +1,22 @@ +package database + +import "database/sql" + +// SavedCohort is a cohort such as stored in the database. +type SavedCohort struct { + Name string + CreateDate string + ExploreQuery ExploreQuery +} + +// ExploreQuery is an explore query such as stored in the database. +type ExploreQuery struct { + ID string + CreateDate string + UserID string + Status string + Definition string + ResultI2b2PatientSetID sql.NullInt64 + ResultGecoSharedIDCount sql.NullString + ResultGecoSharedIDPatientList sql.NullString +} diff --git a/pkg/i2b2datasource/database/saved_cohort.go b/pkg/i2b2datasource/database/saved_cohort.go new file mode 100644 index 0000000..2a31fba --- /dev/null +++ b/pkg/i2b2datasource/database/saved_cohort.go @@ -0,0 +1,146 @@ +package database + +import ( + "database/sql" + "fmt" +) + +// GetCohort retrieves a saved cohort from the database. +// Returns nil (not an error) if cohort does not exist. +func (db PostgresDatabase) GetCohort(cohortName string, exploreQueryID string) (cohort *SavedCohort, err error) { + const getCohortStatement = ` + SELECT + saved_cohort.name AS cohort_name, + saved_cohort.create_date AS cohort_create_date, + saved_cohort.explore_query_id AS explore_query_id, + explore_query.create_date AS explore_query_create_date, + explore_query.user_id AS explore_query_user_id, + explore_query.status AS explore_query_status, + explore_query.definition AS explore_query_definition, + explore_query.result_i2b2_patient_set_id AS result_i2b2_patient_set_id, + explore_query.result_geco_shared_id_count AS result_geco_shared_id_count, + explore_query.result_geco_shared_id_patient_list AS result_geco_shared_id_patient_list + + FROM explore_query + INNER JOIN saved_cohort ON explore_query.id = saved_cohort.explore_query_id + + WHERE saved_cohort.name = $1 AND saved_cohort.explore_query_id = $2;` + + var rows *sql.Rows + if rows, err = db.handle.Query(getCohortStatement, cohortName, exploreQueryID); err != nil { + return nil, fmt.Errorf("querying getCohortStatement: %v", err) + } + defer closeRows(rows, db.logger) + + if rows.Next() { + cohort = new(SavedCohort) + + err = rows.Scan( + &cohort.Name, + &cohort.CreateDate, + &cohort.ExploreQuery.ID, + &cohort.ExploreQuery.CreateDate, + &cohort.ExploreQuery.UserID, + &cohort.ExploreQuery.Status, + &cohort.ExploreQuery.Definition, + &cohort.ExploreQuery.ResultI2b2PatientSetID, + &cohort.ExploreQuery.ResultGecoSharedIDCount, + &cohort.ExploreQuery.ResultGecoSharedIDPatientList, + ) + if err != nil { + return nil, fmt.Errorf("scanning row of getCohortStatement: %v", err) + } + } else { + db.logger.Warnf("cohort not found (name: %v, explore query ID: %v)", cohortName, exploreQueryID) + } + return +} + +// GetCohorts retrieves the saved cohorts of a user from the database. +func (db PostgresDatabase) GetCohorts(userID string) (cohorts []*SavedCohort, err error) { + const getCohortsStatement = ` + SELECT + saved_cohort.name AS cohort_name, + saved_cohort.create_date AS cohort_create_date, + saved_cohort.explore_query_id AS explore_query_id, + explore_query.create_date AS explore_query_create_date, + explore_query.user_id AS explore_query_user_id, + explore_query.status AS explore_query_status, + explore_query.definition AS explore_query_definition, + explore_query.result_i2b2_patient_set_id AS result_i2b2_patient_set_id, + explore_query.result_geco_shared_id_count AS result_geco_shared_id_count, + explore_query.result_geco_shared_id_patient_list AS result_geco_shared_id_patient_list + + FROM explore_query + INNER JOIN saved_cohort ON explore_query.id = saved_cohort.explore_query_id + + WHERE explore_query.user_id = $1 AND explore_query.status = 'success' + ORDER BY saved_cohort.create_date DESC;` + + var rows *sql.Rows + if rows, err = db.handle.Query(getCohortsStatement, userID); err != nil { + return nil, fmt.Errorf("querying getCohortsStatement: %v", err) + } + defer closeRows(rows, db.logger) + + for rows.Next() { + c := new(SavedCohort) + + err = rows.Scan( + &c.Name, + &c.CreateDate, + &c.ExploreQuery.ID, + &c.ExploreQuery.CreateDate, + &c.ExploreQuery.UserID, + &c.ExploreQuery.Status, + &c.ExploreQuery.Definition, + &c.ExploreQuery.ResultI2b2PatientSetID, + &c.ExploreQuery.ResultGecoSharedIDCount, + &c.ExploreQuery.ResultGecoSharedIDPatientList, + ) + if err != nil { + return nil, fmt.Errorf("scanning row of getCohortsStatement: %v", err) + } + cohorts = append(cohorts, c) + } + db.logger.Infof("retrieved %v cohorts for user %v", len(cohorts), userID) + return +} + +// AddCohort adds a saved cohort for a user. Returns an error (due to foreign key violation) if the explore query does not exist. +func (db PostgresDatabase) AddCohort(userID, cohortName, exploreQueryID string) (err error) { + const addCohortStatement = ` + INSERT INTO saved_cohort(name, create_date, explore_query_id) + SELECT $2, NOW(), id FROM explore_query WHERE id = $3 AND user_id = $1;` + + if res, err := db.handle.Exec(addCohortStatement, userID, cohortName, exploreQueryID); err != nil { + return fmt.Errorf("executing addCohortStatement: %v", err) + } else if rowsAffected, err := res.RowsAffected(); err != nil { + return fmt.Errorf("database error: %v", err) + } else if rowsAffected == 0 { + return fmt.Errorf("explore query does not exist") + } + db.logger.Infof("added cohort (name: %v, explore query ID: %v)", cohortName, exploreQueryID) + return +} + +// DeleteCohort deletes a saved cohort for a user. +func (db PostgresDatabase) DeleteCohort(userID, cohortName, exploreQueryID string) (err error) { + const deleteCohortStatement = ` + DELETE FROM saved_cohort + USING explore_query + WHERE + explore_query.user_id = $1 AND + saved_cohort.name = $2 AND + saved_cohort.explore_query_id = $3 AND explore_query.id = $3;` + + if res, err := db.handle.Exec(deleteCohortStatement, userID, cohortName, exploreQueryID); err != nil { + return fmt.Errorf("executing deleteCohortStatement: %v", err) + } else if rowsAffected, err := res.RowsAffected(); err != nil { + return fmt.Errorf("database error: %v", err) + } else if rowsAffected == 0 { + return fmt.Errorf("cohort does not exist") + } + db.logger.Infof("deleted cohort (name: %v, explore query ID: %v)", cohortName, exploreQueryID) + return +} diff --git a/pkg/i2b2datasource/database/saved_cohort_test.go b/pkg/i2b2datasource/database/saved_cohort_test.go new file mode 100644 index 0000000..d183635 --- /dev/null +++ b/pkg/i2b2datasource/database/saved_cohort_test.go @@ -0,0 +1,94 @@ +package database + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetCohort(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + cohort1, err := db.GetCohort("cohort1", "11111111-1111-1111-1111-111111111111") + require.NoError(t, err) + require.EqualValues(t, "cohort1", cohort1.Name) + require.EqualValues(t, "testuser1", cohort1.ExploreQuery.UserID) + t.Logf("%+v", cohort1) + + cohort3, err := db.GetCohort("cohort3", "33333333-3333-3333-3333-333333333333") + require.NoError(t, err) + require.EqualValues(t, "cohort3", cohort3.Name) + require.EqualValues(t, "testuser2", cohort3.ExploreQuery.UserID) + t.Logf("%+v", cohort3) + + cohortNotFound, err := db.GetCohort("cohort1", "11111111-1111-9999-1111-111111111111") + require.NoError(t, err) + require.Nil(t, cohortNotFound) +} + +func TestGetCohorts(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + cohorts1, err := db.GetCohorts("testuser1") + require.NoError(t, err) + require.EqualValues(t, 1, len(cohorts1)) + require.EqualValues(t, "cohort1", cohorts1[0].Name) + + cohorts2, err := db.GetCohorts("testuser2") + require.NoError(t, err) + require.EqualValues(t, 2, len(cohorts2)) + require.EqualValues(t, "cohort3", cohorts2[0].Name) + require.EqualValues(t, "cohort4", cohorts2[1].Name) +} + +func TestAddCohort(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + err := db.AddCohort("notvalid", "cohort0", "00000000-0000-0000-0000-000000000000") + require.Error(t, err) + t.Logf("%v", err) + + err = db.AddCohort("testuser1", "cohort0", "00000000-0000-0000-0000-000000000000") + require.NoError(t, err) + + cohort0, err := db.GetCohort("cohort0", "00000000-0000-0000-0000-000000000000") + require.NoError(t, err) + require.EqualValues(t, "cohort0", cohort0.Name) + require.EqualValues(t, "testuser1", cohort0.ExploreQuery.UserID) + t.Logf("%+v", cohort0) + + err = db.AddCohort("testuser1", "cohort0", "00000000-0000-0000-9999-000000000000") + require.Error(t, err) + t.Logf("%v", err) +} + +func TestDeleteCohort(t *testing.T) { + db := getDB(t) + dbLoadTestData(t, db) + defer dbCleanup(t, db) + + cohortFound, err := db.GetCohort("cohort1", "11111111-1111-1111-1111-111111111111") + require.NoError(t, err) + require.EqualValues(t, "cohort1", cohortFound.Name) + + err = db.DeleteCohort("testuser2", "cohort1", "11111111-1111-1111-1111-111111111111") + require.Error(t, err) + t.Logf("%v", err) + + err = db.DeleteCohort("testuser1", "cohort1", "11111111-1111-1111-1111-111111111111") + require.NoError(t, err) + + cohortNotFound, err := db.GetCohort("cohort1", "11111111-1111-1111-1111-111111111111") + require.NoError(t, err) + require.Nil(t, cohortNotFound) + + err = db.DeleteCohort("testuser1", "cohort2", "11111111-1111-1111-1111-111111111111") + require.Error(t, err) + t.Logf("%v", err) +} From 3bd0d3723f29f5d22db5d5499cd6b751ae7df432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 12:08:14 +0100 Subject: [PATCH 53/81] update package names --- .../database/common.go | 0 .../database/ddl.go | 0 .../database/ddl_test.go | 0 .../database/explore_query.go | 9 +- .../database/explore_query_test.go | 7 +- .../database/models.go | 0 .../database/saved_cohort.go | 0 .../database/saved_cohort_test.go | 0 .../datasource.go | 100 +++++++----------- .../datasource_operations.go} | 12 +++ .../datasource_test.go | 11 ++ .../explore_query_test.go | 20 ++-- .../models/explore_query.go | 24 ++--- .../models/search.go | 2 +- pkg/{i2b2datasource => datasource}/search.go | 57 +++++++--- .../search_test.go | 0 pkg/{i2b2api => i2b2client}/client.go | 0 pkg/{i2b2api => i2b2client}/crc_client.go | 0 pkg/{i2b2api => i2b2client}/models/common.go | 0 pkg/{i2b2api => i2b2client}/models/crc_pdo.go | 0 pkg/{i2b2api => i2b2client}/models/crc_psm.go | 0 pkg/{i2b2api => i2b2client}/models/ont.go | 0 pkg/{i2b2api => i2b2client}/models/util.go | 0 pkg/{i2b2api => i2b2client}/ont_client.go | 0 24 files changed, 136 insertions(+), 106 deletions(-) rename pkg/{i2b2datasource => datasource}/database/common.go (100%) rename pkg/{i2b2datasource => datasource}/database/ddl.go (100%) rename pkg/{i2b2datasource => datasource}/database/ddl_test.go (100%) rename pkg/{i2b2datasource => datasource}/database/explore_query.go (92%) rename pkg/{i2b2datasource => datasource}/database/explore_query_test.go (87%) rename pkg/{i2b2datasource => datasource}/database/models.go (100%) rename pkg/{i2b2datasource => datasource}/database/saved_cohort.go (100%) rename pkg/{i2b2datasource => datasource}/database/saved_cohort_test.go (100%) rename pkg/{i2b2datasource => datasource}/datasource.go (50%) rename pkg/{i2b2datasource/operations.go => datasource/datasource_operations.go} (56%) rename pkg/{i2b2datasource => datasource}/datasource_test.go (87%) rename pkg/{i2b2datasource => datasource}/explore_query_test.go (88%) rename pkg/{i2b2datasource => datasource}/models/explore_query.go (63%) rename pkg/{i2b2datasource => datasource}/models/search.go (99%) rename pkg/{i2b2datasource => datasource}/search.go (51%) rename pkg/{i2b2datasource => datasource}/search_test.go (100%) rename pkg/{i2b2api => i2b2client}/client.go (100%) rename pkg/{i2b2api => i2b2client}/crc_client.go (100%) rename pkg/{i2b2api => i2b2client}/models/common.go (100%) rename pkg/{i2b2api => i2b2client}/models/crc_pdo.go (100%) rename pkg/{i2b2api => i2b2client}/models/crc_psm.go (100%) rename pkg/{i2b2api => i2b2client}/models/ont.go (100%) rename pkg/{i2b2api => i2b2client}/models/util.go (100%) rename pkg/{i2b2api => i2b2client}/ont_client.go (100%) diff --git a/pkg/i2b2datasource/database/common.go b/pkg/datasource/database/common.go similarity index 100% rename from pkg/i2b2datasource/database/common.go rename to pkg/datasource/database/common.go diff --git a/pkg/i2b2datasource/database/ddl.go b/pkg/datasource/database/ddl.go similarity index 100% rename from pkg/i2b2datasource/database/ddl.go rename to pkg/datasource/database/ddl.go diff --git a/pkg/i2b2datasource/database/ddl_test.go b/pkg/datasource/database/ddl_test.go similarity index 100% rename from pkg/i2b2datasource/database/ddl_test.go rename to pkg/datasource/database/ddl_test.go diff --git a/pkg/i2b2datasource/database/explore_query.go b/pkg/datasource/database/explore_query.go similarity index 92% rename from pkg/i2b2datasource/database/explore_query.go rename to pkg/datasource/database/explore_query.go index e37e577..4667e81 100644 --- a/pkg/i2b2datasource/database/explore_query.go +++ b/pkg/datasource/database/explore_query.go @@ -3,8 +3,6 @@ package database import ( "database/sql" "fmt" - - "github.com/google/uuid" ) // GetExploreQuery retrieves an explore query from the database. @@ -53,15 +51,14 @@ func (db PostgresDatabase) GetExploreQuery(id string) (query *ExploreQuery, err } // AddExploreQuery adds an explore query with the requested status. -func (db PostgresDatabase) AddExploreQuery(userID, queryDefinition string) (queryID string, err error) { +func (db PostgresDatabase) AddExploreQuery(userID, queryID, queryDefinition string) (err error) { const addExploreQueryStatement = ` INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES ($1, NOW(), $2, 'requested', $3, NULL, NULL, NULL);` - queryID = uuid.NewString() if _, err = db.handle.Exec(addExploreQueryStatement, queryID, userID, queryDefinition); err != nil { - return "", fmt.Errorf("executing addExploreQueryStatement: %v", err) + return fmt.Errorf("executing addExploreQueryStatement: %v", err) } db.logger.Infof("added explore query (ID: %v)", queryID) return @@ -78,7 +75,7 @@ func (db PostgresDatabase) SetExploreQueryError(userID, queryID string) error { } // SetExploreQuerySuccess sets the success status to an explore query and stores its results. -func (db PostgresDatabase) SetExploreQuerySuccess(userID, queryID, i2b2PatientSetID, gecoSharedIDCount, gecoSharedIDPatientList string) error { +func (db PostgresDatabase) SetExploreQuerySuccess(userID, queryID string, i2b2PatientSetID int64, gecoSharedIDCount, gecoSharedIDPatientList string) error { const setExploreQuerySuccessStatement = ` UPDATE explore_query SET result_i2b2_patient_set_id = $3, diff --git a/pkg/i2b2datasource/database/explore_query_test.go b/pkg/datasource/database/explore_query_test.go similarity index 87% rename from pkg/i2b2datasource/database/explore_query_test.go rename to pkg/datasource/database/explore_query_test.go index e12db31..1ce9d1c 100644 --- a/pkg/i2b2datasource/database/explore_query_test.go +++ b/pkg/datasource/database/explore_query_test.go @@ -31,10 +31,9 @@ func TestExploreQuery(t *testing.T) { dbLoadTestData(t, db) defer dbCleanup(t, db) - queryID, err := db.AddExploreQuery("testUser5", "{}") + queryID := "11111111-7777-9999-1111-111111111111" + err := db.AddExploreQuery("testUser5",queryID, "{}") require.NoError(t, err) - require.NotEmpty(t, queryID) - t.Logf("%v", queryID) query, err := db.GetExploreQuery(queryID) require.NoError(t, err) @@ -55,7 +54,7 @@ func TestExploreQuery(t *testing.T) { require.NoError(t, err) require.EqualValues(t, "error", query.Status) - err = db.SetExploreQuerySuccess("testUser5", queryID, "7", "31133333-3333-3333-3333-333333333333", "33333333-3333-3333-3333-333333333223") + err = db.SetExploreQuerySuccess("testUser5", queryID, 7, "31133333-3333-3333-3333-333333333333", "33333333-3333-3333-3333-333333333223") require.NoError(t, err) query, err = db.GetExploreQuery(queryID) require.NoError(t, err) diff --git a/pkg/i2b2datasource/database/models.go b/pkg/datasource/database/models.go similarity index 100% rename from pkg/i2b2datasource/database/models.go rename to pkg/datasource/database/models.go diff --git a/pkg/i2b2datasource/database/saved_cohort.go b/pkg/datasource/database/saved_cohort.go similarity index 100% rename from pkg/i2b2datasource/database/saved_cohort.go rename to pkg/datasource/database/saved_cohort.go diff --git a/pkg/i2b2datasource/database/saved_cohort_test.go b/pkg/datasource/database/saved_cohort_test.go similarity index 100% rename from pkg/i2b2datasource/database/saved_cohort_test.go rename to pkg/datasource/database/saved_cohort_test.go diff --git a/pkg/i2b2datasource/datasource.go b/pkg/datasource/datasource.go similarity index 50% rename from pkg/i2b2datasource/datasource.go rename to pkg/datasource/datasource.go index 2aa082f..72efdd9 100644 --- a/pkg/i2b2datasource/datasource.go +++ b/pkg/datasource/datasource.go @@ -1,13 +1,12 @@ package i2b2datasource import ( - "encoding/json" "fmt" "time" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api" - i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client" + i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/database" gecomodels "github.com/ldsec/geco/pkg/models" gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" @@ -22,19 +21,23 @@ const ( outputNameExploreQueryPatientList gecosdk.OutputDataObjectName = "patientList" ) -// NewI2b2DataSource creates an i2b2 data source. -// Implements sdk.DataSourcePluginFactory. +// NewI2b2DataSource creates an i2b2 data source. Implements sdk.DataSourcePluginFactory. +// Configuration keys: +// - I2b2: i2b2.api.url, i2b2.api.domain, i2b2.api.username, i2b2.api.password, i2b2.api.project, i2b2.api.wait-time, i2b2.api.ont-max-elements +// - Database: db.host, db.port, db.db-name, db.schema-name, db.user, db.password func NewI2b2DataSource(logger logrus.FieldLogger, config map[string]string) (plugin gecosdk.DataSourcePlugin, err error) { ds := new(I2b2DataSource) ds.logger = logger - // todo: config keys - // i2b2.api.username - // i2b2.db.xxx - // datasource.db.xxx + // initialize database connection + ds.db, err = database.NewPostgresDatabase(logger, config["db.host"], config["db.port"], + config["db.db-name"], config["db.schema-name"], config["db.user"], config["db.password"]) + if err != nil { + return nil, ds.logError("initializing database connection", err) + } // parse i2b2 API connection info and initialize i2b2 client - ci := i2b2apimodels.ConnectionInfo{ + ci := i2b2clientmodels.ConnectionInfo{ HiveURL: config["i2b2.api.url"], Domain: config["i2b2.api.domain"], Username: config["i2b2.api.username"], @@ -43,12 +46,10 @@ func NewI2b2DataSource(logger logrus.FieldLogger, config map[string]string) (plu } if ci.WaitTime, err = time.ParseDuration(config["i2b2.api.wait-time"]); err != nil { - err = fmt.Errorf("parsing i2b2 wait time: %v", err) - logger.Error(err) - return nil, err + return nil, ds.logError("parsing i2b2 wait time", err) } - ds.i2b2Client = i2b2api.Client{ + ds.i2b2Client = i2b2client.Client{ Logger: logger, Ci: ci, } @@ -64,8 +65,11 @@ type I2b2DataSource struct { // logger is the logger from GeCo logger logrus.FieldLogger + // db is the database handler of the data source + db *database.PostgresDatabase + // i2b2Client is the i2b2 client - i2b2Client i2b2api.Client + i2b2Client i2b2client.Client // i2b2OntMaxElements is the configuration for the maximum number of ontology elements to return from i2b2 i2b2OntMaxElements string @@ -76,61 +80,39 @@ func (ds I2b2DataSource) Query(userID string, operation string, jsonParameters [ ds.logger.Infof("executing operation %v for user %v", operation, userID) ds.logger.Debugf("parameters: %v", string(jsonParameters)) + var handler OperationHandler switch Operation(operation) { case OperationSearchConcept: - decodedParams := &models.SearchConceptParameters{} - if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { - return nil, nil, ds.logError("decoding parameters", err) - } else if searchResults, err := ds.SearchConcept(decodedParams); err != nil { - return nil, nil, ds.logError("executing query", err) - } else if jsonResults, err = json.Marshal(searchResults); err != nil { - return nil, nil, ds.logError("encoding results", err) - } - + handler = ds.SearchConceptHandler case OperationSearchModifier: - decodedParams := &models.SearchModifierParameters{} - if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { - return nil, nil, ds.logError("decoding parameters", err) - } else if searchResults, err := ds.SearchModifier(decodedParams); err != nil { - return nil, nil, ds.logError("executing query", err) - } else if jsonResults, err = json.Marshal(searchResults); err != nil { - return nil, nil, ds.logError("encoding results", err) - } - + handler = ds.SearchModifierHandler case OperationExploreQuery: - if outputDataObjectsSharedIDs[outputNameExploreQueryCount] == "" || outputDataObjectsSharedIDs[outputNameExploreQueryPatientList] == "" { - return nil, nil, ds.logError("missing output data object name", nil) - } - - var count int64 - var patientList []int64 - decodedParams := &models.ExploreQueryParameters{} - if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { - return nil, nil, ds.logError("decoding parameters", err) - } else if count, patientList, err = ds.ExploreQuery(decodedParams); err != nil { - return nil, nil, ds.logError("executing query", err) - } - - outputDataObjects = []gecosdk.DataObject{ - { - OutputName: outputNameExploreQueryCount, - SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryCount], - IntValue: &count, - }, { - OutputName: outputNameExploreQueryPatientList, - SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryPatientList], - IntVector: patientList, - }, - } + handler = ds.ExploreQueryHandler + case OperationGetCohorts: + handler = ds.GetCohortsHandler + case OperationAddCohort: + handler = ds.AddCohortHandler + case OperationDeleteCohort: + handler = ds.DeleteCohortHandler + case OperationSurvivalQuery: + return nil, nil, ds.logError("operation survivalQuery not implemented", nil) // todo + case OperationSearchOntology: + return nil, nil, ds.logError("operation searchOntology not implemented", nil) // todo default: return nil, nil, ds.logError(fmt.Sprintf("unknown query requested (%v)", operation), nil) } + + if jsonResults, outputDataObjects, err = handler(userID, jsonParameters, outputDataObjectsSharedIDs); err != nil { + return nil, nil, ds.logError(fmt.Sprintf("executing operation %v", operation), err) + } + + ds.logger.Infof("successfully executed operation %v for user %v", operation, userID) + ds.logger.Debugf("results: %v", string(jsonResults)) return } // logError creates and logs an error. -// todo: exists in GeCo, can be exposed by SDK code func (ds I2b2DataSource) logError(errMsg string, causedBy error) (err error) { if causedBy == nil { err = fmt.Errorf("%v", errMsg) diff --git a/pkg/i2b2datasource/operations.go b/pkg/datasource/datasource_operations.go similarity index 56% rename from pkg/i2b2datasource/operations.go rename to pkg/datasource/datasource_operations.go index 360dd17..f4a529f 100644 --- a/pkg/i2b2datasource/operations.go +++ b/pkg/datasource/datasource_operations.go @@ -1,5 +1,10 @@ package i2b2datasource +import ( + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" +) + // Operation is an operation of the data source supported by I2b2DataSource.Query. type Operation string @@ -14,3 +19,10 @@ const ( OperationSurvivalQuery Operation = "survivalQuery" OperationSearchOntology Operation = "searchOntology" ) + +// OperationHandler is a handler function for an operation of the data source supported by I2b2DataSource.Query. +type OperationHandler func( + userID string, jsonParameters []byte, outputDataObjectsSharedIDs map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID, + ) ( + jsonResults []byte, outputDataObjects []gecosdk.DataObject, err error, + ) diff --git a/pkg/i2b2datasource/datasource_test.go b/pkg/datasource/datasource_test.go similarity index 87% rename from pkg/i2b2datasource/datasource_test.go rename to pkg/datasource/datasource_test.go index 7942e30..82590b2 100644 --- a/pkg/i2b2datasource/datasource_test.go +++ b/pkg/datasource/datasource_test.go @@ -19,6 +19,13 @@ func getTestDataSource(t *testing.T) *I2b2DataSource { config["i2b2.api.wait-time"] = "10s" config["i2b2.api.ont-max-elements"] = "200" + config["db.host"] = "localhost" + config["db.port"] = "5432" + config["db.db-name"] = "i2b2" + config["db.schema-name"] = "gecodatasource" + config["db.user"] = "postgres" + config["db.password"] = "postgres" + logrus.StandardLogger().SetLevel(logrus.DebugLevel) ds, err := NewI2b2DataSource(logrus.StandardLogger(), config) require.NoError(t, err) @@ -51,3 +58,7 @@ func TestQueryDataObject(t *testing.T) { t.Logf("result: %v", string(res)) t.Logf("do: %+v", do) } + +func TestWorkflow(t *testing.T) { + // todo: impl. workflow full +} diff --git a/pkg/i2b2datasource/explore_query_test.go b/pkg/datasource/explore_query_test.go similarity index 88% rename from pkg/i2b2datasource/explore_query_test.go rename to pkg/datasource/explore_query_test.go index 124602c..158f3b9 100644 --- a/pkg/i2b2datasource/explore_query_test.go +++ b/pkg/datasource/explore_query_test.go @@ -10,7 +10,7 @@ import ( func TestExploreQueryConcept(t *testing.T) { ds := getTestDataSource(t) - count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -26,7 +26,7 @@ func TestExploreQueryConcept(t *testing.T) { require.EqualValues(t, 3, count) require.Subset(t, []int64{1, 2, 3}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -42,7 +42,7 @@ func TestExploreQueryConcept(t *testing.T) { require.EqualValues(t, 4, count) require.Subset(t, []int64{1, 2, 3, 4}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "2", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -68,7 +68,7 @@ func TestExploreQueryConcept(t *testing.T) { require.EqualValues(t, 3, count) require.Subset(t, []int64{1, 2, 3}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "3", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -90,7 +90,7 @@ func TestExploreQueryConcept(t *testing.T) { func TestExploreQueryConceptValue(t *testing.T) { ds := getTestDataSource(t) - count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -109,7 +109,7 @@ func TestExploreQueryConceptValue(t *testing.T) { require.EqualValues(t, 1, count) require.Subset(t, []int64{1}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -137,7 +137,7 @@ func TestExploreQueryConceptValue(t *testing.T) { func TestExploreQueryModifier(t *testing.T) { ds := getTestDataSource(t) - count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -160,7 +160,7 @@ func TestExploreQueryModifier(t *testing.T) { require.EqualValues(t, 2, count) require.Subset(t, []int64{1, 3}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -196,7 +196,7 @@ func TestExploreQueryModifier(t *testing.T) { func TestExploreQueryModifierValue(t *testing.T) { ds := getTestDataSource(t) - count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ @@ -222,7 +222,7 @@ func TestExploreQueryModifierValue(t *testing.T) { require.EqualValues(t, 1, count) require.Subset(t, []int64{1}, patientList) - count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ + _, count, patientList, err = ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "1", Definition: models.ExploreQueryDefinition{ Panels: []models.Panel{{ diff --git a/pkg/i2b2datasource/models/explore_query.go b/pkg/datasource/models/explore_query.go similarity index 63% rename from pkg/i2b2datasource/models/explore_query.go rename to pkg/datasource/models/explore_query.go index 079236f..472eb0a 100644 --- a/pkg/i2b2datasource/models/explore_query.go +++ b/pkg/datasource/models/explore_query.go @@ -1,6 +1,6 @@ package models -import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" +import i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" // --- parameters @@ -37,17 +37,17 @@ type ConceptItem struct { } // ToI2b2APIModel converts this query definition in the i2b2 API format. -func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2apimodels.Panel, i2b2ApiTiming i2b2apimodels.Timing) { - i2b2ApiTiming = i2b2apimodels.Timing(d.Timing) +func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2clientmodels.Panel, i2b2ApiTiming i2b2clientmodels.Timing) { + i2b2ApiTiming = i2b2clientmodels.Timing(d.Timing) for panelIdx, panel := range d.Panels { - var i2b2ApiItems []i2b2apimodels.Item + var i2b2ApiItems []i2b2clientmodels.Item for _, item := range panel.ConceptItems { - i2b2ApiItem := i2b2apimodels.Item{ItemKey: i2b2apimodels.ConvertPathToI2b2Format(item.QueryTerm)} + i2b2ApiItem := i2b2clientmodels.Item{ItemKey: i2b2clientmodels.ConvertPathToI2b2Format(item.QueryTerm)} if item.Operator != "" && item.Modifier.Key == "" { - i2b2ApiItem.ConstrainByValue = &i2b2apimodels.ConstrainByValue{ + i2b2ApiItem.ConstrainByValue = &i2b2clientmodels.ConstrainByValue{ ValueType: item.Type, ValueOperator: item.Operator, ValueConstraint: item.Value, @@ -55,12 +55,12 @@ func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2apimodels. } if item.Modifier.Key != "" { - i2b2ApiItem.ConstrainByModifier = &i2b2apimodels.ConstrainByModifier{ - ModifierKey: i2b2apimodels.ConvertPathToI2b2Format(item.Modifier.Key), - AppliedPath: i2b2apimodels.ConvertAppliedPathToI2b2Format(item.Modifier.AppliedPath), + i2b2ApiItem.ConstrainByModifier = &i2b2clientmodels.ConstrainByModifier{ + ModifierKey: i2b2clientmodels.ConvertPathToI2b2Format(item.Modifier.Key), + AppliedPath: i2b2clientmodels.ConvertAppliedPathToI2b2Format(item.Modifier.AppliedPath), } if item.Operator != "" { - i2b2ApiItem.ConstrainByModifier.ConstrainByValue = &i2b2apimodels.ConstrainByValue{ + i2b2ApiItem.ConstrainByModifier.ConstrainByValue = &i2b2clientmodels.ConstrainByValue{ ValueType: item.Type, ValueOperator: item.Operator, ValueConstraint: item.Value, @@ -71,11 +71,11 @@ func (d ExploreQueryDefinition) ToI2b2APIModel() (i2b2ApiPanels []i2b2apimodels. } for _, cohort := range panel.CohortItems { - i2b2ApiItems = append(i2b2ApiItems, i2b2apimodels.Item{ItemKey: cohort}) + i2b2ApiItems = append(i2b2ApiItems, i2b2clientmodels.Item{ItemKey: cohort}) } i2b2ApiPanels = append(i2b2ApiPanels, - i2b2apimodels.NewPanel(panelIdx, panel.Not, i2b2apimodels.Timing(panel.Timing), i2b2ApiItems), + i2b2clientmodels.NewPanel(panelIdx, panel.Not, i2b2clientmodels.Timing(panel.Timing), i2b2ApiItems), ) } return diff --git a/pkg/i2b2datasource/models/search.go b/pkg/datasource/models/search.go similarity index 99% rename from pkg/i2b2datasource/models/search.go rename to pkg/datasource/models/search.go index 996aeca..6e75f52 100644 --- a/pkg/i2b2datasource/models/search.go +++ b/pkg/datasource/models/search.go @@ -1,6 +1,6 @@ package models -import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" +import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" // --- parameters diff --git a/pkg/i2b2datasource/search.go b/pkg/datasource/search.go similarity index 51% rename from pkg/i2b2datasource/search.go rename to pkg/datasource/search.go index bd5406e..7160949 100644 --- a/pkg/i2b2datasource/search.go +++ b/pkg/datasource/search.go @@ -1,20 +1,36 @@ package i2b2datasource import ( + "encoding/json" "fmt" "strings" - i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" ) +// SearchConceptHandler is the OperationHandler for the searchConcept Operation. +func (ds I2b2DataSource) SearchConceptHandler(_ string, jsonParameters []byte, _ map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (jsonResults []byte, _ []gecosdk.DataObject, err error) { + decodedParams := &models.SearchConceptParameters{} + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } else if searchResults, err := ds.SearchConcept(decodedParams); err != nil { + return nil, nil, fmt.Errorf("executing query: %v", err) + } else if jsonResults, err = json.Marshal(searchResults); err != nil { + return nil, nil, fmt.Errorf("encoding results: %v", err) + } + return +} + // SearchConcept retrieves the info about or the children of the concept identified by path. func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) (*models.SearchResults, error) { // make the appropriate request to i2b2 path := strings.TrimSpace(params.Path) - i2b2FormatPath := i2b2apimodels.ConvertPathToI2b2Format(path) - var resp *i2b2apimodels.OntRespConceptsMessageBody + i2b2FormatPath := i2b2clientmodels.ConvertPathToI2b2Format(path) + var resp *i2b2clientmodels.OntRespConceptsMessageBody var err error if len(path) == 0 { @@ -27,21 +43,21 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } - req := i2b2apimodels.NewOntReqGetTermInfoMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + req := i2b2clientmodels.NewOntReqGetTermInfoMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) if resp, err = ds.i2b2Client.OntGetTermInfo(&req); err != nil { return nil, fmt.Errorf("requesting term info: %v", err) } case "children": if path == "/" { - req := i2b2apimodels.NewOntReqGetCategoriesMessageBody() + req := i2b2clientmodels.NewOntReqGetCategoriesMessageBody() if resp, err = ds.i2b2Client.OntGetCategories(&req); err != nil { return nil, fmt.Errorf("requesting categories: %v", err) } break } - req := i2b2apimodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) + req := i2b2clientmodels.NewOntReqGetChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath) if resp, err = ds.i2b2Client.OntGetChildren(&req); err != nil { return nil, fmt.Errorf("requesting children: %v", err) } @@ -58,13 +74,26 @@ func (ds I2b2DataSource) SearchConcept(params *models.SearchConceptParameters) ( return &models.SearchResults{SearchResults: searchResults}, nil } +// SearchModifierHandler is the OperationHandler for the searchModifier Operation. +func (ds I2b2DataSource) SearchModifierHandler(_ string, jsonParameters []byte, _ map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (jsonResults []byte, _ []gecosdk.DataObject, err error) { + decodedParams := &models.SearchModifierParameters{} + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } else if searchResults, err := ds.SearchModifier(decodedParams); err != nil { + return nil, nil, fmt.Errorf("executing query: %v", err) + } else if jsonResults, err = json.Marshal(searchResults); err != nil { + return nil, nil, fmt.Errorf("encoding results: %v", err) + } + return +} + // SearchModifier retrieves the info about or the children of the modifier identified by path. func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) (*models.SearchResults, error) { // make the appropriate request to i2b2 path := strings.TrimSpace(params.Path) - i2b2FormatPath := i2b2apimodels.ConvertPathToI2b2Format(path) - var resp *i2b2apimodels.OntRespModifiersMessageBody + i2b2FormatPath := i2b2clientmodels.ConvertPathToI2b2Format(path) + var resp *i2b2clientmodels.OntRespModifiersMessageBody var err error if len(path) == 0 { @@ -77,7 +106,7 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } - req := i2b2apimodels.NewOntReqGetModifiersMessageBody(i2b2FormatPath) + req := i2b2clientmodels.NewOntReqGetModifiersMessageBody(i2b2FormatPath) if resp, err = ds.i2b2Client.OntGetModifiers(&req); err != nil { return nil, fmt.Errorf("requesting modifiers of concept: %v", err) } @@ -87,8 +116,8 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } - i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) - req := i2b2apimodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) + i2b2FormatAppliedPath := i2b2clientmodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) + req := i2b2clientmodels.NewOntReqGetModifierInfoMessageBody(i2b2FormatPath, i2b2FormatAppliedPath) if resp, err = ds.i2b2Client.OntGetModifierInfo(&req); err != nil { return nil, fmt.Errorf("requesting info of modifier: %v", err) } @@ -98,9 +127,9 @@ func (ds I2b2DataSource) SearchModifier(params *models.SearchModifierParameters) return &models.SearchResults{SearchResults: make([]models.SearchResult, 0)}, nil } - i2b2FormatAppliedPath := i2b2apimodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) - i2b2FormatAppliedConcept := i2b2apimodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) - req := i2b2apimodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) + i2b2FormatAppliedPath := i2b2clientmodels.ConvertAppliedPathToI2b2Format(strings.TrimSpace(params.AppliedPath)) + i2b2FormatAppliedConcept := i2b2clientmodels.ConvertPathToI2b2Format(strings.TrimSpace(params.AppliedConcept)) + req := i2b2clientmodels.NewOntReqGetModifierChildrenMessageBody(ds.i2b2OntMaxElements, i2b2FormatPath, i2b2FormatAppliedPath, i2b2FormatAppliedConcept) if resp, err = ds.i2b2Client.OntGetModifierChildren(&req); err != nil { return nil, fmt.Errorf("requesting children of modifier: %v", err) } diff --git a/pkg/i2b2datasource/search_test.go b/pkg/datasource/search_test.go similarity index 100% rename from pkg/i2b2datasource/search_test.go rename to pkg/datasource/search_test.go diff --git a/pkg/i2b2api/client.go b/pkg/i2b2client/client.go similarity index 100% rename from pkg/i2b2api/client.go rename to pkg/i2b2client/client.go diff --git a/pkg/i2b2api/crc_client.go b/pkg/i2b2client/crc_client.go similarity index 100% rename from pkg/i2b2api/crc_client.go rename to pkg/i2b2client/crc_client.go diff --git a/pkg/i2b2api/models/common.go b/pkg/i2b2client/models/common.go similarity index 100% rename from pkg/i2b2api/models/common.go rename to pkg/i2b2client/models/common.go diff --git a/pkg/i2b2api/models/crc_pdo.go b/pkg/i2b2client/models/crc_pdo.go similarity index 100% rename from pkg/i2b2api/models/crc_pdo.go rename to pkg/i2b2client/models/crc_pdo.go diff --git a/pkg/i2b2api/models/crc_psm.go b/pkg/i2b2client/models/crc_psm.go similarity index 100% rename from pkg/i2b2api/models/crc_psm.go rename to pkg/i2b2client/models/crc_psm.go diff --git a/pkg/i2b2api/models/ont.go b/pkg/i2b2client/models/ont.go similarity index 100% rename from pkg/i2b2api/models/ont.go rename to pkg/i2b2client/models/ont.go diff --git a/pkg/i2b2api/models/util.go b/pkg/i2b2client/models/util.go similarity index 100% rename from pkg/i2b2api/models/util.go rename to pkg/i2b2client/models/util.go diff --git a/pkg/i2b2api/ont_client.go b/pkg/i2b2client/ont_client.go similarity index 100% rename from pkg/i2b2api/ont_client.go rename to pkg/i2b2client/ont_client.go From 22d948a9ddd2c1da85c4eaa12066100f2303edca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 12:09:09 +0100 Subject: [PATCH 54/81] formatting --- pkg/datasource/database/explore_query_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/datasource/database/explore_query_test.go b/pkg/datasource/database/explore_query_test.go index 1ce9d1c..d112531 100644 --- a/pkg/datasource/database/explore_query_test.go +++ b/pkg/datasource/database/explore_query_test.go @@ -32,7 +32,7 @@ func TestExploreQuery(t *testing.T) { defer dbCleanup(t, db) queryID := "11111111-7777-9999-1111-111111111111" - err := db.AddExploreQuery("testUser5",queryID, "{}") + err := db.AddExploreQuery("testUser5", queryID, "{}") require.NoError(t, err) query, err := db.GetExploreQuery(queryID) From 09a38fce358bc24c823f53fcf8b6ff0c79ae0a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 12:09:47 +0100 Subject: [PATCH 55/81] update i2b2client package name --- pkg/datasource/models/search.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/datasource/models/search.go b/pkg/datasource/models/search.go index 6e75f52..3791be2 100644 --- a/pkg/datasource/models/search.go +++ b/pkg/datasource/models/search.go @@ -1,6 +1,6 @@ package models -import i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" +import i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" // --- parameters @@ -25,12 +25,12 @@ type SearchResults struct { } // NewSearchResultFromI2b2Concept creates a new SearchResult from an i2b2 concept. -func NewSearchResultFromI2b2Concept(concept i2b2apimodels.Concept) SearchResult { +func NewSearchResultFromI2b2Concept(concept i2b2clientmodels.Concept) SearchResult { parsed := SearchResult{ Name: concept.Name, DisplayName: concept.Name, Code: concept.Basecode, - Path: i2b2apimodels.ConvertPathFromI2b2Format(concept.Key), + Path: i2b2clientmodels.ConvertPathFromI2b2Format(concept.Key), AppliedPath: "@", Comment: concept.Comment, } @@ -76,9 +76,9 @@ func NewSearchResultFromI2b2Concept(concept i2b2apimodels.Concept) SearchResult } // NewSearchResultFromI2b2Modifier creates a new SearchResult from an i2b2 modifier. -func NewSearchResultFromI2b2Modifier(modifier i2b2apimodels.Modifier) SearchResult { +func NewSearchResultFromI2b2Modifier(modifier i2b2clientmodels.Modifier) SearchResult { res := NewSearchResultFromI2b2Concept(modifier.Concept) - res.AppliedPath = i2b2apimodels.ConvertPathFromI2b2Format(modifier.AppliedPath) + res.AppliedPath = i2b2clientmodels.ConvertPathFromI2b2Format(modifier.AppliedPath) return res } From 2d2f8c31eec6aaa52697cd8efcb406837e6d25a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 12:10:45 +0100 Subject: [PATCH 56/81] update i2b2api>i2b2client package name --- pkg/i2b2client/client.go | 4 ++-- pkg/i2b2client/crc_client.go | 4 ++-- pkg/i2b2client/ont_client.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/i2b2client/client.go b/pkg/i2b2client/client.go index 172ff4c..4816618 100644 --- a/pkg/i2b2client/client.go +++ b/pkg/i2b2client/client.go @@ -1,4 +1,4 @@ -package i2b2api +package i2b2client import ( "bytes" @@ -7,7 +7,7 @@ import ( "io/ioutil" "net/http" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" "github.com/sirupsen/logrus" ) diff --git a/pkg/i2b2client/crc_client.go b/pkg/i2b2client/crc_client.go index d547a34..7062084 100644 --- a/pkg/i2b2client/crc_client.go +++ b/pkg/i2b2client/crc_client.go @@ -1,9 +1,9 @@ -package i2b2api +package i2b2client import ( "fmt" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" ) // CrcPsmReqFromQueryDef makes an i2b2 API request to /QueryToolService/request for CRC PSM from query definition. diff --git a/pkg/i2b2client/ont_client.go b/pkg/i2b2client/ont_client.go index 0b40391..62b23ef 100644 --- a/pkg/i2b2client/ont_client.go +++ b/pkg/i2b2client/ont_client.go @@ -1,9 +1,9 @@ -package i2b2api +package i2b2client import ( "fmt" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" ) // OntGetCategories makes an i2b2 API request to /OntologyService/getCategories. From 6f0a0c83982525e655cb8f39cab115260cae018b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 12:13:50 +0100 Subject: [PATCH 57/81] package names updates and explore query modifications --- .../data-source-plugin.go | 4 +- pkg/datasource/datasource.go | 4 +- pkg/datasource/datasource_operations.go | 10 +- pkg/datasource/datasource_test.go | 2 +- pkg/datasource/explore_query.go | 149 ++++++++++++++++++ pkg/datasource/explore_query_test.go | 4 +- pkg/datasource/search.go | 4 +- pkg/datasource/search_test.go | 4 +- pkg/i2b2datasource/explore_query.go | 97 ------------ 9 files changed, 165 insertions(+), 113 deletions(-) create mode 100644 pkg/datasource/explore_query.go delete mode 100644 pkg/i2b2datasource/explore_query.go diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go index 34e17ce..e47eafa 100644 --- a/cmd/geco-i2b2-data-source/data-source-plugin.go +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -1,9 +1,9 @@ package main import ( - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource" "github.com/ldsec/geco/pkg/sdk" ) // DataSourcePluginFactory exports a factory function compatible with GeCo data source plugin SDK. -var DataSourcePluginFactory sdk.DataSourcePluginFactory = i2b2datasource.NewI2b2DataSource +var DataSourcePluginFactory sdk.DataSourcePluginFactory = datasource.NewI2b2DataSource diff --git a/pkg/datasource/datasource.go b/pkg/datasource/datasource.go index 72efdd9..0a12f67 100644 --- a/pkg/datasource/datasource.go +++ b/pkg/datasource/datasource.go @@ -1,12 +1,12 @@ -package i2b2datasource +package datasource import ( "fmt" "time" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/database" "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client" i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/database" gecomodels "github.com/ldsec/geco/pkg/models" gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" diff --git a/pkg/datasource/datasource_operations.go b/pkg/datasource/datasource_operations.go index f4a529f..bad36fe 100644 --- a/pkg/datasource/datasource_operations.go +++ b/pkg/datasource/datasource_operations.go @@ -1,4 +1,4 @@ -package i2b2datasource +package datasource import ( gecomodels "github.com/ldsec/geco/pkg/models" @@ -22,7 +22,7 @@ const ( // OperationHandler is a handler function for an operation of the data source supported by I2b2DataSource.Query. type OperationHandler func( - userID string, jsonParameters []byte, outputDataObjectsSharedIDs map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID, - ) ( - jsonResults []byte, outputDataObjects []gecosdk.DataObject, err error, - ) + userID string, jsonParameters []byte, outputDataObjectsSharedIDs map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID, +) ( + jsonResults []byte, outputDataObjects []gecosdk.DataObject, err error, +) diff --git a/pkg/datasource/datasource_test.go b/pkg/datasource/datasource_test.go index 82590b2..bca6495 100644 --- a/pkg/datasource/datasource_test.go +++ b/pkg/datasource/datasource_test.go @@ -1,4 +1,4 @@ -package i2b2datasource +package datasource import ( "testing" diff --git a/pkg/datasource/explore_query.go b/pkg/datasource/explore_query.go new file mode 100644 index 0000000..c46df22 --- /dev/null +++ b/pkg/datasource/explore_query.go @@ -0,0 +1,149 @@ +package datasource + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" + i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" +) + +// ExploreQueryHandler is the OperationHandler for the exploreQuery Operation. +func (ds I2b2DataSource) ExploreQueryHandler(userID string, jsonParameters []byte, outputDataObjectsSharedIDs map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (jsonResults []byte, outputDataObjects []gecosdk.DataObject, err error) { + + // parse parameters + decodedParams := &models.ExploreQueryParameters{} + if outputDataObjectsSharedIDs[outputNameExploreQueryCount] == "" || outputDataObjectsSharedIDs[outputNameExploreQueryPatientList] == "" { + return nil, nil, fmt.Errorf("missing output data object name") + } else if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } + + // register query in DB + if queryDef, err := json.Marshal(decodedParams.Definition); err != nil { + return nil, nil, fmt.Errorf("marshalling query definition: %v", err) + } else if err := ds.db.AddExploreQuery(userID, decodedParams.ID, string(queryDef)); err != nil { + return nil, nil, fmt.Errorf("registering explore query: %v", err) + } else if err := ds.db.SetExploreQueryRunning(userID, decodedParams.ID); err != nil { + return nil, nil, fmt.Errorf("updating explore query status: %v", err) + } + + // run query and update status in DB + i2b2PatientSetID, count, patientList, err := ds.ExploreQuery(decodedParams) + if err != nil { + return nil, nil, fmt.Errorf("executing query: %v", err) + } else if err := ds.db.SetExploreQuerySuccess( + userID, decodedParams.ID, i2b2PatientSetID, + string(outputDataObjectsSharedIDs[outputNameExploreQueryCount]), + string(outputDataObjectsSharedIDs[outputNameExploreQueryPatientList]), + ); err != nil { + return nil, nil, fmt.Errorf("updating explore query status: %v", err) + } + + // wrap results in data objects + outputDataObjects = []gecosdk.DataObject{ + { + OutputName: outputNameExploreQueryCount, + SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryCount], + IntValue: &count, + }, { + OutputName: outputNameExploreQueryPatientList, + SharedID: outputDataObjectsSharedIDs[outputNameExploreQueryPatientList], + IntVector: patientList, + }, + } + return +} + +// ExploreQuery makes an explore query, i.e. two i2b2 CRC queries, a PSM and a PDO query. +func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (patientSetID int64, patientCount int64, patientList []int64, err error) { + + if i2b2PatientCount, i2b2PatientSetID, err := ds.doCrcPsmQuery(params); err != nil { + return -1, -1, nil, err + } else if patientCount, err = strconv.ParseInt(i2b2PatientCount, 10, 64); err != nil { + return -1, -1, nil, fmt.Errorf("parsing patient count: %v", err) + } else if patientSetID, err = strconv.ParseInt(i2b2PatientSetID, 10, 64); err != nil { + return -1, -1, nil, fmt.Errorf("parsing patient set ID: %v", err) + } else if i2b2PatientIDs, err := ds.getPatientIDs(i2b2PatientSetID); err != nil { + return -1, -1, nil, err + } else { + // parse patient IDs + for _, patientID := range i2b2PatientIDs { + parsedPatientID, err := strconv.ParseInt(patientID, 10, 64) + if err != nil { + return -1, -1, nil, fmt.Errorf("parsing patient ID: %v", err) + } + patientList = append(patientList, parsedPatientID) + } + } + return +} + +// doCrcPsmQuery requests a PSM query to the i2b2 CRC and parse its results. +func (ds I2b2DataSource) doCrcPsmQuery(params *models.ExploreQueryParameters) (patientCount, patientSetID string, err error) { + + // build query + panels, timing := params.Definition.ToI2b2APIModel() + psmReq := i2b2clientmodels.NewCrcPsmReqFromQueryDef( + ds.i2b2Client.Ci, + params.ID, + panels, + timing, + []i2b2clientmodels.ResultOutputName{i2b2clientmodels.ResultOutputPatientSet, i2b2clientmodels.ResultOutputCount}, + ) + + // request query + var psmResp *i2b2clientmodels.CrcPsmRespMessageBody + if psmResp, err = ds.i2b2Client.CrcPsmReqFromQueryDef(&psmReq); err != nil { + return "", "", fmt.Errorf("requesting PSM query: %v", err) + } + + // extract results from result instances + for i, qri := range psmResp.Response.QueryResultInstances { + + // check status + if err := qri.CheckStatus(); err != nil { + return "", "", fmt.Errorf("found error in query result instance %v: %v", i, err) + } + + // extract result + switch qri.QueryResultType.Name { + case string(i2b2clientmodels.ResultOutputPatientSet): + if patientSetID != "" { + ds.logger.Warnf("unexpected additional patient set result in i2b2 CRC PSM response (previous: %v)", patientSetID) + } + patientSetID = qri.ResultInstanceID + + case string(i2b2clientmodels.ResultOutputCount): + if patientCount != "" { + ds.logger.Warnf("unexpected additional patient count result in i2b2 CRC PSM response (previous: %v)", patientCount) + } + patientCount = qri.SetSize + + default: + ds.logger.Warnf("unexpected result in i2b2 CRC PSM response: %v", qri.QueryResultType.Name) + } + } + + if patientCount == "" || patientSetID == "" { + return "", "", fmt.Errorf("missing result from i2b2 PSM response: patientCount=%v, patientSetID=%v", patientCount, patientSetID) + } + return +} + +// getPatientIDs retrieves the list of patient IDs from the i2b2 CRC using a patient set ID. +func (ds I2b2DataSource) getPatientIDs(patientSetID string) (patientIDs []string, err error) { + pdoReq := i2b2clientmodels.NewCrcPdoReqFromInputList(patientSetID) + var pdoResp *i2b2clientmodels.CrcPdoRespMessageBody + if pdoResp, err = ds.i2b2Client.CrcPdoReqFromInputList(&pdoReq); err != nil { + return nil, fmt.Errorf("requesting PDO query: %v", err) + } + + for _, patient := range pdoResp.Response.PatientData.PatientSet.Patient { + patientIDs = append(patientIDs, patient.PatientID) + } + return +} diff --git a/pkg/datasource/explore_query_test.go b/pkg/datasource/explore_query_test.go index 158f3b9..a40c53f 100644 --- a/pkg/datasource/explore_query_test.go +++ b/pkg/datasource/explore_query_test.go @@ -1,9 +1,9 @@ -package i2b2datasource +package datasource import ( "testing" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" "github.com/stretchr/testify/require" ) diff --git a/pkg/datasource/search.go b/pkg/datasource/search.go index 7160949..6528f75 100644 --- a/pkg/datasource/search.go +++ b/pkg/datasource/search.go @@ -1,12 +1,12 @@ -package i2b2datasource +package datasource import ( "encoding/json" "fmt" "strings" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/models" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" gecomodels "github.com/ldsec/geco/pkg/models" gecosdk "github.com/ldsec/geco/pkg/sdk" ) diff --git a/pkg/datasource/search_test.go b/pkg/datasource/search_test.go index b909385..36cd44e 100644 --- a/pkg/datasource/search_test.go +++ b/pkg/datasource/search_test.go @@ -1,9 +1,9 @@ -package i2b2datasource +package datasource import ( "testing" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" "github.com/stretchr/testify/require" ) diff --git a/pkg/i2b2datasource/explore_query.go b/pkg/i2b2datasource/explore_query.go deleted file mode 100644 index 38e491f..0000000 --- a/pkg/i2b2datasource/explore_query.go +++ /dev/null @@ -1,97 +0,0 @@ -package i2b2datasource - -import ( - "fmt" - "strconv" - - i2b2apimodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2api/models" - "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2datasource/models" -) - -// ExploreQuery makes an explore query, i.e. two i2b2 CRC queries, a PSM and a PDO query. -func (ds I2b2DataSource) ExploreQuery(params *models.ExploreQueryParameters) (patientCount int64, patientList []int64, err error) { - - if i2b2PatientCount, i2b2PatientSetID, err := ds.doExploreQuery(params); err != nil { - return 0, nil, err - } else if i2b2PatientIDs, err := ds.getPatientIDs(i2b2PatientSetID); err != nil { - return 0, nil, err - } else if patientCount, err = strconv.ParseInt(i2b2PatientCount, 10, 64); err != nil { - return 0, nil, fmt.Errorf("parsing patient count: %v", err) - } else { - for _, patientID := range i2b2PatientIDs { - parsedPatientID, err := strconv.ParseInt(patientID, 10, 64) - if err != nil { - return 0, nil, fmt.Errorf("parsing patient ID: %v", err) - } - patientList = append(patientList, parsedPatientID) - } - } - - return -} - -// doExploreQuery requests an explore query to the i2b2 CRC and parse its results. -func (ds I2b2DataSource) doExploreQuery(params *models.ExploreQueryParameters) (patientCount, patientSetID string, err error) { - - // build query - panels, timing := params.Definition.ToI2b2APIModel() - psmReq := i2b2apimodels.NewCrcPsmReqFromQueryDef( - ds.i2b2Client.Ci, - params.ID, - panels, - timing, - []i2b2apimodels.ResultOutputName{i2b2apimodels.ResultOutputPatientSet, i2b2apimodels.ResultOutputCount}, - ) - - // request query - var psmResp *i2b2apimodels.CrcPsmRespMessageBody - if psmResp, err = ds.i2b2Client.CrcPsmReqFromQueryDef(&psmReq); err != nil { - return "", "", fmt.Errorf("requesting PSM query: %v", err) - } - - // extract results from result instances - for i, qri := range psmResp.Response.QueryResultInstances { - - // check status - if err := qri.CheckStatus(); err != nil { - return "", "", fmt.Errorf("found error in query result instance %v: %v", i, err) - } - - // extract result - switch qri.QueryResultType.Name { - case string(i2b2apimodels.ResultOutputPatientSet): - if patientSetID != "" { - ds.logger.Warnf("unexpected additional patient set result in i2b2 CRC PSM response (previous: %v)", patientSetID) - } - patientSetID = qri.ResultInstanceID - - case string(i2b2apimodels.ResultOutputCount): - if patientCount != "" { - ds.logger.Warnf("unexpected additional patient count result in i2b2 CRC PSM response (previous: %v)", patientCount) - } - patientCount = qri.SetSize - - default: - ds.logger.Warnf("unexpected result in i2b2 CRC PSM response: %v", qri.QueryResultType.Name) - } - } - - if patientCount == "" || patientSetID == "" { - return "", "", fmt.Errorf("missing result from i2b2 PSM response: patientCount=%v, patientSetID=%v", patientCount, patientSetID) - } - return -} - -// getPatientIDs retrieves the list of patient IDs from the i2b2 CRC using a patient set ID. -func (ds I2b2DataSource) getPatientIDs(patientSetID string) (patientIDs []string, err error) { - pdoReq := i2b2apimodels.NewCrcPdoReqFromInputList(patientSetID) - var pdoResp *i2b2apimodels.CrcPdoRespMessageBody - if pdoResp, err = ds.i2b2Client.CrcPdoReqFromInputList(&pdoReq); err != nil { - return nil, fmt.Errorf("requesting PDO query: %v", err) - } - - for _, patient := range pdoResp.Response.PatientData.PatientSet.Patient { - patientIDs = append(patientIDs, patient.PatientID) - } - return -} From ec16ad0347e3d1eb0ea91db02c7b94d4d2c07409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 15:53:08 +0100 Subject: [PATCH 58/81] refactor, fix tests; add full test workflow --- internal/plugin_test.go | 7 + pkg/datasource/database/ddl_test.go | 52 ++----- pkg/datasource/database/explore_query_test.go | 6 +- pkg/datasource/database/saved_cohort_test.go | 20 +-- pkg/datasource/datasource_test.go | 143 ++++++++++++++++-- pkg/datasource/explore_query_test.go | 38 ++++- pkg/datasource/search_test.go | 21 ++- 7 files changed, 208 insertions(+), 79 deletions(-) diff --git a/internal/plugin_test.go b/internal/plugin_test.go index c11dce8..71823e4 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -31,6 +31,13 @@ func TestPlugin(t *testing.T) { config["i2b2.api.wait-time"] = "10s" config["i2b2.api.ont-max-elements"] = "200" + config["db.host"] = "localhost" + config["db.port"] = "5432" + config["db.db-name"] = "i2b2" + config["db.schema-name"] = "gecodatasourceplugintest" + config["db.user"] = "postgres" + config["db.password"] = "postgres" + logrus.StandardLogger().SetLevel(logrus.DebugLevel) ds, err := (*pluginFactory)(logrus.StandardLogger(), config) require.NoError(t, err) diff --git a/pkg/datasource/database/ddl_test.go b/pkg/datasource/database/ddl_test.go index 72cfece..dd4deed 100644 --- a/pkg/datasource/database/ddl_test.go +++ b/pkg/datasource/database/ddl_test.go @@ -7,59 +7,25 @@ import ( "github.com/stretchr/testify/require" ) -const unitTestsSchemaName = "gecodatasourceunittest" -const testDataStatement = ` -BEGIN; - INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, - result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES - ('00000000-0000-0000-0000-000000000000', NOW(), 'testuser1', 'success', '{"query": {}}', - 0, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002'); - INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, - result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES - ('11111111-1111-1111-1111-111111111111', NOW(), 'testuser1', 'success', '{"query": {}}', - 1, '11111111-1111-1111-1111-111111111112', '11111111-1111-1111-1111-111111111113'); - INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, - result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES - ('22222222-2222-2222-2222-222222222222', NOW(), 'testuser1', 'error', '{"query": {}}', - NULL, NULL, NULL); - INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, - result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES - ('33333333-3333-3333-3333-333333333333', NOW(), 'testuser2', 'success', '{"query": {}}', - 2, '33333333-3333-3333-3333-333333333334', '33333333-3333-3333-3333-333333333335'); - INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, - result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES - ('44444444-4444-4444-4444-444444444444', NOW(), 'testuser2', 'success', '{"query": {}}', - 3, '44444444-4444-4444-4444-444444444445', '44444444-4444-4444-4444-444444444446'); +func getDB(t *testing.T) *PostgresDatabase { + log := logrus.StandardLogger() + log.SetLevel(logrus.DebugLevel) - INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES - ('cohort1', NOW(), '11111111-1111-1111-1111-111111111111'); - INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES - ('cohort3', NOW(), '33333333-3333-3333-3333-333333333333'); - INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES - ('cohort4', NOW(), '44444444-4444-4444-4444-444444444444'); -COMMIT; -` + db, err := NewPostgresDatabase(log, "localhost", "5432", "i2b2", TestSchemaName, "postgres", "postgres") + require.NoError(t, err) -func getDB(t *testing.T) *PostgresDatabase { - db, err := NewPostgresDatabase(logrus.StandardLogger(), - "localhost", "5432", "i2b2", unitTestsSchemaName, "postgres", "postgres") + err = db.TestLoadData() require.NoError(t, err) return db } -func dbLoadTestData(t *testing.T, db *PostgresDatabase) { - _, err := db.handle.Exec(testDataStatement) - require.NoError(t, err) -} - -func dbCleanup(t *testing.T, db *PostgresDatabase) { - _, err := db.handle.Exec("DROP SCHEMA " + unitTestsSchemaName + " CASCADE;") +func dbCleanUp(t *testing.T, db *PostgresDatabase) { + err := db.TestCleanUp() require.NoError(t, err) } func TestDDL(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - dbCleanup(t, db) + dbCleanUp(t, db) } diff --git a/pkg/datasource/database/explore_query_test.go b/pkg/datasource/database/explore_query_test.go index d112531..6ffafce 100644 --- a/pkg/datasource/database/explore_query_test.go +++ b/pkg/datasource/database/explore_query_test.go @@ -8,8 +8,7 @@ import ( func TestGetExploreQuery(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) query1, err := db.GetExploreQuery("11111111-1111-1111-1111-111111111111") require.NoError(t, err) @@ -28,8 +27,7 @@ func TestGetExploreQuery(t *testing.T) { func TestExploreQuery(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) queryID := "11111111-7777-9999-1111-111111111111" err := db.AddExploreQuery("testUser5", queryID, "{}") diff --git a/pkg/datasource/database/saved_cohort_test.go b/pkg/datasource/database/saved_cohort_test.go index d183635..7045f7d 100644 --- a/pkg/datasource/database/saved_cohort_test.go +++ b/pkg/datasource/database/saved_cohort_test.go @@ -8,8 +8,7 @@ import ( func TestGetCohort(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) cohort1, err := db.GetCohort("cohort1", "11111111-1111-1111-1111-111111111111") require.NoError(t, err) @@ -30,25 +29,23 @@ func TestGetCohort(t *testing.T) { func TestGetCohorts(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) - cohorts1, err := db.GetCohorts("testuser1") + cohorts1, err := db.GetCohorts("testuser1", 5) require.NoError(t, err) require.EqualValues(t, 1, len(cohorts1)) require.EqualValues(t, "cohort1", cohorts1[0].Name) - cohorts2, err := db.GetCohorts("testuser2") + cohorts2, err := db.GetCohorts("testuser2", 5) require.NoError(t, err) require.EqualValues(t, 2, len(cohorts2)) - require.EqualValues(t, "cohort3", cohorts2[0].Name) - require.EqualValues(t, "cohort4", cohorts2[1].Name) + require.EqualValues(t, "cohort4", cohorts2[0].Name) + require.EqualValues(t, "cohort3", cohorts2[1].Name) } func TestAddCohort(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) err := db.AddCohort("notvalid", "cohort0", "00000000-0000-0000-0000-000000000000") require.Error(t, err) @@ -70,8 +67,7 @@ func TestAddCohort(t *testing.T) { func TestDeleteCohort(t *testing.T) { db := getDB(t) - dbLoadTestData(t, db) - defer dbCleanup(t, db) + defer dbCleanUp(t, db) cohortFound, err := db.GetCohort("cohort1", "11111111-1111-1111-1111-111111111111") require.NoError(t, err) diff --git a/pkg/datasource/datasource_test.go b/pkg/datasource/datasource_test.go index bca6495..a44188f 100644 --- a/pkg/datasource/datasource_test.go +++ b/pkg/datasource/datasource_test.go @@ -1,15 +1,17 @@ package datasource import ( + "fmt" "testing" + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/database" gecomodels "github.com/ldsec/geco/pkg/models" gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) -func getTestDataSource(t *testing.T) *I2b2DataSource { +func getDataSource(t *testing.T) *I2b2DataSource { config := make(map[string]string) config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" config["i2b2.api.domain"] = "i2b2demo" @@ -22,18 +24,29 @@ func getTestDataSource(t *testing.T) *I2b2DataSource { config["db.host"] = "localhost" config["db.port"] = "5432" config["db.db-name"] = "i2b2" - config["db.schema-name"] = "gecodatasource" + config["db.schema-name"] = database.TestSchemaName config["db.user"] = "postgres" config["db.password"] = "postgres" logrus.StandardLogger().SetLevel(logrus.DebugLevel) ds, err := NewI2b2DataSource(logrus.StandardLogger(), config) require.NoError(t, err) + + err = ds.(*I2b2DataSource).db.TestLoadData() + require.NoError(t, err) + return ds.(*I2b2DataSource) } +func dataSourceCleanUp(t *testing.T, ds *I2b2DataSource) { + err := ds.db.TestCleanUp() + require.NoError(t, err) +} + func TestQuery(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + params := `{"path": "/", "operation": "children"}` res, _, err := ds.Query("testUser", "searchConcept", []byte(params), nil) require.NoError(t, err) @@ -41,18 +54,23 @@ func TestQuery(t *testing.T) { } func TestQueryDataObject(t *testing.T) { - ds := getTestDataSource(t) - params := `{"id": "0", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}` - sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{outputNameExploreQueryCount: "countSharedID", outputNameExploreQueryPatientList: "patientListSharedID"} + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + params := `{"id": "99999999-9999-1122-0000-999999999999", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}` + sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{ + outputNameExploreQueryCount: "99999999-9999-9999-1111-999999999999", + outputNameExploreQueryPatientList: "99999999-9999-9999-0000-999999999999", + } res, do, err := ds.Query("testUser", "exploreQuery", []byte(params), sharedIDs) require.NoError(t, err) require.EqualValues(t, 3, *do[0].IntValue) - require.EqualValues(t, "countSharedID", do[0].SharedID) + require.EqualValues(t, "99999999-9999-9999-1111-999999999999", do[0].SharedID) require.EqualValues(t, outputNameExploreQueryCount, do[0].OutputName) require.InDeltaSlice(t, []int64{1, 2, 3}, do[1].IntVector, 0.001) - require.EqualValues(t, "patientListSharedID", do[1].SharedID) + require.EqualValues(t, "99999999-9999-9999-0000-999999999999", do[1].SharedID) require.EqualValues(t, outputNameExploreQueryPatientList, do[1].OutputName) t.Logf("result: %v", string(res)) @@ -60,5 +78,112 @@ func TestQueryDataObject(t *testing.T) { } func TestWorkflow(t *testing.T) { - // todo: impl. workflow full + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + user := "testUser" + + // search the ontology + params := `{"path": "/", "operation": "children"}` + res, do, err := ds.Query(user, "searchConcept", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.Contains(t, string(res), "Test Ontology") + + params = `{"path": "/TEST/test/", "operation": "children"}` + res, do, err = ds.Query(user, "searchConcept", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.Contains(t, string(res), "Concept 1") + + params = `{"path": "/TEST/test/1/", "operation": "concept"}` + res, do, err = ds.Query(user, "searchModifier", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.Contains(t, string(res), "Modifier 1") + + params = `{"path": "/TEST/modifiers/", "operation": "children", "appliedPath": "/test/%", "appliedConcept": "/TEST/test/1/"}` + res, do, err = ds.Query(user, "searchModifier", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.Contains(t, string(res), "Modifier 1") + + // execute query + queryID := "99999999-9999-9999-9999-999999999999" + params = fmt.Sprintf(`{ + "id": "%v", + "definition": { + "panels": [ + { + "conceptItems": [ + { + "queryTerm": "/TEST/test/2/", + "operator": "LIKE[contains]", + "value": "cd", + "type": "TEXT", + "modifier": { + "key": "/TEST/modifiers/2text/", + "appliedPath": "/test/2/" + } + } + ] + },{ + "conceptItems": [ + { + "queryTerm": "/TEST/test/1/", + "operator": "EQ", + "value": "10", + "type": "NUMBER" + },{ + "queryTerm": "/TEST/test/3/", + "operator": "EQ", + "value": "20", + "type": "NUMBER" + } + ] + } + ] + } + }`, queryID) + sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{ + outputNameExploreQueryCount: "99999999-9999-9999-1111-999999999999", + outputNameExploreQueryPatientList: "99999999-9999-9999-0000-999999999999", + } + res, do, err = ds.Query(user, "exploreQuery", []byte(params), sharedIDs) + require.NoError(t, err) + require.EqualValues(t, 2, len(do)) + for i := range do { + if do[i].OutputName == outputNameExploreQueryCount { + require.EqualValues(t, 1, *do[i].IntValue) + } else if do[i].OutputName == outputNameExploreQueryPatientList { + require.Subset(t, do[i].IntVector, []int64{1}) + } else { + require.Fail(t, "unexpected output name encountered") + } + } + + // save cohort + params = fmt.Sprintf(`{"name": "mycohort", "exploreQueryID": "%s"}`, queryID) + res, do, err = ds.Query(user, "addCohort", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.EqualValues(t, "", string(res)) + + params = `{}` + res, do, err = ds.Query(user, "getCohorts", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.Contains(t, string(res), "mycohort") + + params = fmt.Sprintf(`{"name": "mycohort", "exploreQueryID": "%s"}`, queryID) + res, do, err = ds.Query(user, "deleteCohort", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.EqualValues(t, "", string(res)) + + params = `{"limit": 7}` + res, do, err = ds.Query(user, "getCohorts", []byte(params), nil) + require.NoError(t, err) + require.Empty(t, do) + require.NotContains(t, string(res), "mycohort") } diff --git a/pkg/datasource/explore_query_test.go b/pkg/datasource/explore_query_test.go index a40c53f..4e0ff79 100644 --- a/pkg/datasource/explore_query_test.go +++ b/pkg/datasource/explore_query_test.go @@ -1,14 +1,18 @@ package datasource import ( + "fmt" "testing" "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/stretchr/testify/require" ) func TestExploreQueryConcept(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", @@ -88,7 +92,8 @@ func TestExploreQueryConcept(t *testing.T) { } func TestExploreQueryConceptValue(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", @@ -135,7 +140,8 @@ func TestExploreQueryConceptValue(t *testing.T) { } func TestExploreQueryModifier(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", @@ -194,7 +200,8 @@ func TestExploreQueryModifier(t *testing.T) { } func TestExploreQueryModifierValue(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, count, patientList, err := ds.ExploreQuery(&models.ExploreQueryParameters{ ID: "0", @@ -263,3 +270,26 @@ func TestExploreQueryModifierValue(t *testing.T) { require.EqualValues(t, 1, count) require.Subset(t, []int64{2}, patientList) } + +func TestExploreQueryDatabase(t *testing.T) { + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + queryID := "44444444-7777-4444-4444-444444444442" + countSharedID := "44444444-7777-8888-4444-444444444444" + patientListSharedID := "44444444-7777-4444-7121-444444444444" + + params := fmt.Sprintf(`{"id": "%v", "definition": {"panels": [{"conceptItems": [{"queryTerm": "/TEST/test/1/"}]}]}}`, queryID) + sharedIDs := map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID{ + outputNameExploreQueryCount: gecomodels.DataObjectSharedID(countSharedID), + outputNameExploreQueryPatientList: gecomodels.DataObjectSharedID(patientListSharedID), + } + _, _, err := ds.Query("testUser", "exploreQuery", []byte(params), sharedIDs) + require.NoError(t, err) + + query, err := ds.db.GetExploreQuery(queryID) + require.NoError(t, err) + require.EqualValues(t, "success", query.Status) + require.EqualValues(t, countSharedID, query.ResultGecoSharedIDCount.String) + require.EqualValues(t, patientListSharedID, query.ResultGecoSharedIDPatientList.String) +} diff --git a/pkg/datasource/search_test.go b/pkg/datasource/search_test.go index 36cd44e..ea8ad1f 100644 --- a/pkg/datasource/search_test.go +++ b/pkg/datasource/search_test.go @@ -8,7 +8,8 @@ import ( ) func TestSearchConceptInfo(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "/", @@ -37,7 +38,8 @@ func TestSearchConceptInfo(t *testing.T) { } func TestSearchConceptChildren(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) searchResults, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "/", @@ -67,7 +69,8 @@ func TestSearchConceptChildren(t *testing.T) { } func TestSearchConceptError(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, err := ds.SearchConcept(&models.SearchConceptParameters{ Path: "", @@ -92,7 +95,8 @@ func TestSearchConceptError(t *testing.T) { } func TestSearchModifierConcept(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -123,7 +127,8 @@ func TestSearchModifierConcept(t *testing.T) { } func TestSearchModifierInfo(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -151,7 +156,8 @@ func TestSearchModifierInfo(t *testing.T) { } func TestSearchModifierChildren(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) searchResults, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ @@ -192,7 +198,8 @@ func TestSearchModifierChildren(t *testing.T) { } func TestSearchModifierError(t *testing.T) { - ds := getTestDataSource(t) + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) _, err := ds.SearchModifier(&models.SearchModifierParameters{ SearchConceptParameters: models.SearchConceptParameters{ From b2685af9db844bebf5edea939fa757de84582ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 15:54:17 +0100 Subject: [PATCH 59/81] add loading of test data in DB --- pkg/datasource/database/common.go | 2 +- pkg/datasource/database/ddl.go | 31 +++++++++------ pkg/datasource/database/test_data.go | 59 ++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 pkg/datasource/database/test_data.go diff --git a/pkg/datasource/database/common.go b/pkg/datasource/database/common.go index 04bdca4..283b15d 100644 --- a/pkg/datasource/database/common.go +++ b/pkg/datasource/database/common.go @@ -29,7 +29,7 @@ func NewPostgresDatabase(logger logrus.FieldLogger, host, port, databaseName, sc return db, db.loadDdl(schemaName, userLogin) } -// PostgresDatabase is TBD +// PostgresDatabase wraps the Postgres database of the data source. type PostgresDatabase struct { logger logrus.FieldLogger diff --git a/pkg/datasource/database/ddl.go b/pkg/datasource/database/ddl.go index f77c741..a1772ce 100644 --- a/pkg/datasource/database/ddl.go +++ b/pkg/datasource/database/ddl.go @@ -2,14 +2,10 @@ package database import "fmt" -const ddlLoaded = ` -SELECT EXISTS ( - SELECT 1 FROM pg_namespace WHERE nspname = $1 -); -` +const ddlLoaded = `SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = $1);` -const createSchemaFunction = ` -CREATE OR REPLACE FUNCTION public.create_schema(schema_name NAME, user_name NAME) +const createDeleteSchemaFunctions = ` +CREATE OR REPLACE FUNCTION public.create_gecoi2b2datasource_schema(schema_name NAME, user_name NAME) RETURNS BOOL AS ' BEGIN EXECUTE ''CREATE SCHEMA '' || schema_name; @@ -18,8 +14,20 @@ RETURNS BOOL AS ' RETURN true; END; ' LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION public.delete_gecoi2b2datasource_schema(schema_name NAME) +RETURNS BOOL AS ' + BEGIN + EXECUTE ''DROP SCHEMA '' || schema_name || '' CASCADE''; + RETURN true; + END; +' LANGUAGE PLPGSQL; ` +const createSchemaStatement = `SELECT public.create_gecoi2b2datasource_schema($1::name, $2::name);` + +const deleteSchemaStatement = `SELECT public.delete_gecoi2b2datasource_schema($1::name);` + const ddlStatement = ` BEGIN; CREATE TYPE query_status AS ENUM ('requested', 'running', 'success', 'error'); @@ -66,11 +74,12 @@ func (db PostgresDatabase) loadDdl(schemaName, userLogin string) (err error) { // load DDL if not already loaded if !isDdlLoaded { - if _, err = db.handle.Exec(createSchemaFunction); err != nil { - return fmt.Errorf("executing createSchemaFunction: %v", err) + db.logger.Infof("database structure does not exist, loading DDL") + if _, err = db.handle.Exec(createDeleteSchemaFunctions); err != nil { + return fmt.Errorf("executing createDeleteSchemaFunctions: %v", err) } - if _, err = db.handle.Exec("SELECT public.create_schema($1::name, $2::name);", schemaName, userLogin); err != nil { - return fmt.Errorf("executing create_schema: %v", err) + if _, err = db.handle.Exec(createSchemaStatement, schemaName, userLogin); err != nil { + return fmt.Errorf("executing createSchemaStatement: %v", err) } if _, err = db.handle.Exec(ddlStatement); err != nil { return fmt.Errorf("executing ddlStatement: %v", err) diff --git a/pkg/datasource/database/test_data.go b/pkg/datasource/database/test_data.go new file mode 100644 index 0000000..4afeab4 --- /dev/null +++ b/pkg/datasource/database/test_data.go @@ -0,0 +1,59 @@ +package database + +import ( + "fmt" +) + +// TestSchemaName defines the database schema name used for tests. +const TestSchemaName = "gecodatasourcetest" +const loadTestDataStatement = ` +BEGIN; + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('00000000-0000-0000-0000-000000000000', NOW(), 'testuser1', 'success', '{"query": {}}', + 0, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('11111111-1111-1111-1111-111111111111', NOW(), 'testuser1', 'success', '{"query": {}}', + 1, '11111111-1111-1111-1111-111111111112', '11111111-1111-1111-1111-111111111113'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('22222222-2222-2222-2222-222222222222', NOW(), 'testuser1', 'error', '{"query": {}}', + NULL, NULL, NULL); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('33333333-3333-3333-3333-333333333333', NOW(), 'testuser2', 'success', '{"query": {}}', + 2, '33333333-3333-3333-3333-333333333334', '33333333-3333-3333-3333-333333333335'); + INSERT INTO explore_query(id, create_date, user_id, status, definition, result_i2b2_patient_set_id, + result_geco_shared_id_count, result_geco_shared_id_patient_list) VALUES + ('44444444-4444-4444-4444-444444444444', NOW(), 'testuser2', 'success', '{"query": {}}', + 3, '44444444-4444-4444-4444-444444444445', '44444444-4444-4444-4444-444444444446'); + + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort1', '2021-12-20T13:47:24.015216Z', '11111111-1111-1111-1111-111111111111'); + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort3', '2021-12-21T13:47:24.015216Z', '33333333-3333-3333-3333-333333333333'); + INSERT INTO saved_cohort(name, create_date, explore_query_id) VALUES + ('cohort4', '2021-12-22T13:47:24.015216Z', '44444444-4444-4444-4444-444444444444'); +COMMIT; +` + +// TestLoadData loads test data into the database. For usage with tests. +func (db PostgresDatabase) TestLoadData() error { + db.logger.Warnf("loading test data") + + if _, err := db.handle.Exec(loadTestDataStatement); err != nil { + return fmt.Errorf("executing loadTestDataStatement: %v", err) + } + return nil +} + +// TestCleanUp deletes ALL test data and structure of the data source. Use with caution! +func (db PostgresDatabase) TestCleanUp() error { + db.logger.Warnf("deleting all test data from the database schema") + + if _, err := db.handle.Exec(deleteSchemaStatement, TestSchemaName); err != nil { + return fmt.Errorf("executing deleteSchemaStatement: %v", err) + } + return nil +} From 491650cc8097e3699b4a88b68ce5a9e26c9d91e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 15:54:37 +0100 Subject: [PATCH 60/81] add limit to get cohorts --- pkg/datasource/database/saved_cohort.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/datasource/database/saved_cohort.go b/pkg/datasource/database/saved_cohort.go index 2a31fba..61a705e 100644 --- a/pkg/datasource/database/saved_cohort.go +++ b/pkg/datasource/database/saved_cohort.go @@ -57,7 +57,7 @@ func (db PostgresDatabase) GetCohort(cohortName string, exploreQueryID string) ( } // GetCohorts retrieves the saved cohorts of a user from the database. -func (db PostgresDatabase) GetCohorts(userID string) (cohorts []*SavedCohort, err error) { +func (db PostgresDatabase) GetCohorts(userID string, limit int) (cohorts []SavedCohort, err error) { const getCohortsStatement = ` SELECT saved_cohort.name AS cohort_name, @@ -75,17 +75,17 @@ func (db PostgresDatabase) GetCohorts(userID string) (cohorts []*SavedCohort, er INNER JOIN saved_cohort ON explore_query.id = saved_cohort.explore_query_id WHERE explore_query.user_id = $1 AND explore_query.status = 'success' - ORDER BY saved_cohort.create_date DESC;` + ORDER BY saved_cohort.create_date DESC + LIMIT $2;` var rows *sql.Rows - if rows, err = db.handle.Query(getCohortsStatement, userID); err != nil { + if rows, err = db.handle.Query(getCohortsStatement, userID, limit); err != nil { return nil, fmt.Errorf("querying getCohortsStatement: %v", err) } defer closeRows(rows, db.logger) for rows.Next() { - c := new(SavedCohort) - + c := SavedCohort{} err = rows.Scan( &c.Name, &c.CreateDate, @@ -107,7 +107,8 @@ func (db PostgresDatabase) GetCohorts(userID string) (cohorts []*SavedCohort, er return } -// AddCohort adds a saved cohort for a user. Returns an error (due to foreign key violation) if the explore query does not exist. +// AddCohort adds a saved cohort for a user. Returns an error (due to foreign key violation) if the explore query does +// not exist, or if the provided user does not match. func (db PostgresDatabase) AddCohort(userID, cohortName, exploreQueryID string) (err error) { const addCohortStatement = ` INSERT INTO saved_cohort(name, create_date, explore_query_id) From 7cf8c92c71a1b1bb304fe5ea9a64a8d46569042d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 15:55:06 +0100 Subject: [PATCH 61/81] add implementation for cohorts --- pkg/datasource/cohorts.go | 82 ++++++++++++++++++++++++++++++++ pkg/datasource/cohorts_test.go | 80 +++++++++++++++++++++++++++++++ pkg/datasource/models/cohorts.go | 58 ++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 pkg/datasource/cohorts.go create mode 100644 pkg/datasource/cohorts_test.go create mode 100644 pkg/datasource/models/cohorts.go diff --git a/pkg/datasource/cohorts.go b/pkg/datasource/cohorts.go new file mode 100644 index 0000000..f60ca2a --- /dev/null +++ b/pkg/datasource/cohorts.go @@ -0,0 +1,82 @@ +package datasource + +import ( + "encoding/json" + "fmt" + + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" + gecomodels "github.com/ldsec/geco/pkg/models" + gecosdk "github.com/ldsec/geco/pkg/sdk" +) + +// GetCohortsHandler is the OperationHandler for the getCohorts Operation. +func (ds I2b2DataSource) GetCohortsHandler(userID string, jsonParameters []byte, _ map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (jsonResults []byte, _ []gecosdk.DataObject, err error) { + decodedParams := &models.GetCohortsParameters{} + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } else if cohortResults, err := ds.GetCohorts(userID, decodedParams); err != nil { + return nil, nil, fmt.Errorf("getting cohorts: %v", err) + } else if jsonResults, err = json.Marshal(cohortResults); err != nil { + return nil, nil, fmt.Errorf("encoding results: %v", err) + } + return +} + +// GetCohorts retrieves the list of cohorts of the user. +func (ds I2b2DataSource) GetCohorts(userID string, params *models.GetCohortsParameters) (results *models.CohortResults, err error) { + limit := 10 + if params.Limit > 0 { + limit = params.Limit + } + + dbCohorts, err := ds.db.GetCohorts(userID, limit) + if err != nil { + return nil, fmt.Errorf("retrieving cohorts from database: %v", err) + } + + results = new(models.CohortResults) + results.Cohorts = make([]models.Cohort, 0, len(dbCohorts)) + for _, dbCohort := range dbCohorts { + results.Cohorts = append(results.Cohorts, models.NewCohortFromDbModel(dbCohort)) + } + + return +} + +// AddCohortHandler is the OperationHandler for the addCohort Operation. +func (ds I2b2DataSource) AddCohortHandler(userID string, jsonParameters []byte, _ map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (_ []byte, _ []gecosdk.DataObject, err error) { + decodedParams := &models.AddDeleteCohortParameters{} + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } else if err := ds.AddCohort(userID, decodedParams); err != nil { + return nil, nil, fmt.Errorf("adding cohort: %v", err) + } + return +} + +// AddCohort adds a cohort. +func (ds I2b2DataSource) AddCohort(userID string, params *models.AddDeleteCohortParameters) error { + if err := ds.db.AddCohort(userID, params.Name, params.ExploreQueryID); err != nil { + return fmt.Errorf("adding cohort to database: %v", err) + } + return nil +} + +// DeleteCohortHandler is the OperationHandler for the deleteCohort Operation. +func (ds I2b2DataSource) DeleteCohortHandler(userID string, jsonParameters []byte, _ map[gecosdk.OutputDataObjectName]gecomodels.DataObjectSharedID) (_ []byte, _ []gecosdk.DataObject, err error) { + decodedParams := &models.AddDeleteCohortParameters{} + if err = json.Unmarshal(jsonParameters, decodedParams); err != nil { + return nil, nil, fmt.Errorf("decoding parameters: %v", err) + } else if err := ds.DeleteCohort(userID, decodedParams); err != nil { + return nil, nil, fmt.Errorf("deleting cohort: %v", err) + } + return +} + +// DeleteCohort deletes a cohort. +func (ds I2b2DataSource) DeleteCohort(userID string, params *models.AddDeleteCohortParameters) error { + if err := ds.db.DeleteCohort(userID, params.Name, params.ExploreQueryID); err != nil { + return fmt.Errorf("deleting cohort from database: %v", err) + } + return nil +} diff --git a/pkg/datasource/cohorts_test.go b/pkg/datasource/cohorts_test.go new file mode 100644 index 0000000..223b7f6 --- /dev/null +++ b/pkg/datasource/cohorts_test.go @@ -0,0 +1,80 @@ +package datasource + +import ( + "testing" + + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/models" + "github.com/stretchr/testify/require" +) + +func TestGetCohorts(t *testing.T) { + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + cohorts, err := ds.GetCohorts("testuser2", &models.GetCohortsParameters{}) + require.NoError(t, err) + require.EqualValues(t, 2, len(cohorts.Cohorts)) + require.EqualValues(t, "cohort4", cohorts.Cohorts[0].Name) + require.EqualValues(t, "cohort3", cohorts.Cohorts[1].Name) + + cohorts, err = ds.GetCohorts("testuser2", &models.GetCohortsParameters{Limit: 1}) + require.NoError(t, err) + require.EqualValues(t, 1, len(cohorts.Cohorts)) + require.EqualValues(t, "cohort4", cohorts.Cohorts[0].Name) +} + +func TestAddCohort(t *testing.T) { + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + err := ds.AddCohort("testuser1", &models.AddDeleteCohortParameters{ + Name: "cohort0", + ExploreQueryID: "00000000-0000-0000-0000-000000000000", + }) + require.NoError(t, err) + + cohorts, err := ds.GetCohorts("testuser1", &models.GetCohortsParameters{}) + require.NoError(t, err) + require.EqualValues(t, 2, len(cohorts.Cohorts)) + require.EqualValues(t, "cohort0", cohorts.Cohorts[0].Name) + require.EqualValues(t, "cohort1", cohorts.Cohorts[1].Name) + + err = ds.AddCohort("testuser2", &models.AddDeleteCohortParameters{ + Name: "cohort4bis", + ExploreQueryID: "44444444-4444-4444-4444-444444444444", + }) + require.NoError(t, err) + + cohorts, err = ds.GetCohorts("testuser2", &models.GetCohortsParameters{}) + require.NoError(t, err) + require.EqualValues(t, 3, len(cohorts.Cohorts)) + require.EqualValues(t, "cohort4bis", cohorts.Cohorts[0].Name) + require.EqualValues(t, "cohort4", cohorts.Cohorts[1].Name) + require.EqualValues(t, "cohort3", cohorts.Cohorts[2].Name) +} + +func TestDeleteCohort(t *testing.T) { + ds := getDataSource(t) + defer dataSourceCleanUp(t, ds) + + err := ds.DeleteCohort("testuser1", &models.AddDeleteCohortParameters{ + Name: "cohort1", + ExploreQueryID: "11111111-1111-1111-1111-111111111111", + }) + require.NoError(t, err) + + cohorts, err := ds.GetCohorts("testuser1", &models.GetCohortsParameters{}) + require.NoError(t, err) + require.EqualValues(t, 0, len(cohorts.Cohorts)) + + err = ds.DeleteCohort("testuser2", &models.AddDeleteCohortParameters{ + Name: "cohort4", + ExploreQueryID: "44444444-4444-4444-4444-444444444444", + }) + require.NoError(t, err) + + cohorts, err = ds.GetCohorts("testuser2", &models.GetCohortsParameters{}) + require.NoError(t, err) + require.EqualValues(t, 1, len(cohorts.Cohorts)) + require.EqualValues(t, "cohort3", cohorts.Cohorts[0].Name) +} diff --git a/pkg/datasource/models/cohorts.go b/pkg/datasource/models/cohorts.go new file mode 100644 index 0000000..90c6530 --- /dev/null +++ b/pkg/datasource/models/cohorts.go @@ -0,0 +1,58 @@ +package models + +import ( + "encoding/json" + + "github.com/ldsec/geco-i2b2-data-source/pkg/datasource/database" +) + +// --- parameters + +// GetCohortsParameters is the parameter for the GetCohorts operation. +type GetCohortsParameters struct { + Limit int +} + +// AddDeleteCohortParameters is the parameter for the AddCohort and DeleteCohort operations. +type AddDeleteCohortParameters struct { + Name string + ExploreQueryID string +} + +// --- results + +// CohortResults is the result of the GetCohorts operation. +type CohortResults struct { + Cohorts []Cohort +} + +// Cohort is a result part of the GetCohorts operation. +type Cohort struct { + Name string + CreationDate string + ExploreQuery struct { + ID string + CreationDate string + Status string + Definition ExploreQueryDefinition + ResultSharedIDs struct { + Count string + PatientList string + } + } +} + +// NewCohortFromDbModel creates a new Cohort from a database.SavedCohort. +func NewCohortFromDbModel(dbCohort database.SavedCohort) (cohort Cohort) { + cohort.Name = dbCohort.Name + cohort.CreationDate = dbCohort.CreateDate + + cohort.ExploreQuery.ID = dbCohort.ExploreQuery.ID + cohort.ExploreQuery.Status = dbCohort.ExploreQuery.Status + cohort.ExploreQuery.CreationDate = dbCohort.ExploreQuery.CreateDate + cohort.ExploreQuery.ResultSharedIDs.Count = dbCohort.ExploreQuery.ResultGecoSharedIDCount.String + cohort.ExploreQuery.ResultSharedIDs.PatientList = dbCohort.ExploreQuery.ResultGecoSharedIDPatientList.String + + _ = json.Unmarshal([]byte(dbCohort.ExploreQuery.Definition), &cohort.ExploreQuery.Definition) + return cohort +} From ddd7e923e5e710e26f497097327fad632481afef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 16:42:35 +0100 Subject: [PATCH 62/81] fix ci typo --- .github/workflows/ci-cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index fc2eb2c..7f15711 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -67,7 +67,7 @@ jobs: build-args: BUILD=docker load: true ssh: default - tags: ${{ steps.get_version.outputs.i2b2_docker_image }} + tags: ${{ steps.get_versions.outputs.i2b2_docker_image }} cache-from: type=gha,scope=buildkit cache-to: type=gha,scope=buildkit,mode=max @@ -102,4 +102,4 @@ jobs: # - name: Push GeCo Docker image # if: ${{ env.PUSH }} -# run: docker push ${{ steps.get_version.outputs.i2b2_docker_image }} +# run: docker push ${{ steps.get_versions.outputs.i2b2_dockerss_image }} From b43cdc99ac72004b83dc229a2f9bf6b93ae9221f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 17:33:08 +0100 Subject: [PATCH 63/81] update of geco version to use and expose DataSourceType --- .../data-source-plugin.go | 3 + go.mod | 37 +++++++++++ go.sum | 46 +++++++++++++- internal/plugin_test.go | 61 +++++++++++++++---- third_party/geco | 2 +- 5 files changed, 134 insertions(+), 15 deletions(-) diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go index e47eafa..5eab8f2 100644 --- a/cmd/geco-i2b2-data-source/data-source-plugin.go +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -5,5 +5,8 @@ import ( "github.com/ldsec/geco/pkg/sdk" ) +// DataSourceType is the type of the data source. +var DataSourceType sdk.DataSourceType = "i2b2-medco" + // DataSourcePluginFactory exports a factory function compatible with GeCo data source plugin SDK. var DataSourcePluginFactory sdk.DataSourcePluginFactory = datasource.NewI2b2DataSource diff --git a/go.mod b/go.mod index 6837b49..c89062f 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,16 @@ require ( ) require ( + github.com/Nerzal/gocloak/v9 v9.0.3 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/aws/aws-sdk-go v1.34.28 // indirect + github.com/creasty/defaults v1.5.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect + github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect github.com/go-openapi/analysis v0.20.1 // indirect github.com/go-openapi/errors v0.20.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -26,17 +32,48 @@ require ( github.com/go-openapi/strfmt v0.20.3 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/go-openapi/validate v0.20.3 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator v9.31.0+incompatible // indirect + github.com/go-resty/resty/v2 v2.3.0 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/goccy/go-json v0.7.10 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/ldsec/lattigo/v2 v2.0.0 // indirect + github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect + github.com/lestrrat-go/blackmagic v1.0.0 // indirect + github.com/lestrrat-go/httpcc v1.0.0 // indirect + github.com/lestrrat-go/iter v1.0.1 // indirect + github.com/lestrrat-go/jwx v1.2.9 // indirect + github.com/lestrrat-go/option v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/montanaflynn/stats v0.6.3 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/philippgille/gokv v0.6.0 // indirect + github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 // indirect + github.com/philippgille/gokv/file v0.6.0 // indirect + github.com/philippgille/gokv/s3 v0.6.0 // indirect + github.com/philippgille/gokv/syncmap v0.6.0 // indirect + github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/segmentio/ksuid v1.0.3 // indirect + go.dedis.ch/fixbuf v1.0.3 // indirect + go.dedis.ch/kyber/v3 v3.0.13 // indirect + go.dedis.ch/onet/v3 v3.2.3 // indirect go.mongodb.org/mongo-driver v1.7.3 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect golang.org/x/text v0.3.7 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gonum.org/v1/gonum v0.7.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 7a91416..0b08f42 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Cryptolens/cryptolens-golang v0.0.0-20210322162237-d3ae4c72721f/go.mod h1:zVXpFHn6wrg4t0PA7g2XT46hlspli2/g7vC9HRXvfwA= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Nerzal/gocloak/v9 v9.0.3 h1:45Jiw+H2TIrSsMboO4t6bUozMAzxeQDcwIWvnJ/KJsU= github.com/Nerzal/gocloak/v9 v9.0.3/go.mod h1:AmXlpCliq31R1r/18ei7/ML5mfJDmJ1pTBoFU79DDZg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -24,6 +25,7 @@ github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:W github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.25.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -40,14 +42,19 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM= github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920 h1:d/cVoZOrJPJHKH1NdeUjyVAWKp4OpOT+Q+6T1sH7jeU= github.com/daviddengcn/go-colortext v0.0.0-20180409174941-186a3d44e920/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d h1:1iy2qD6JEhHKKhUOA9IWs7mjco7lnw2qx8FsRI2wirE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -168,14 +175,19 @@ github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw= github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= +github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So= github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= @@ -204,6 +216,7 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/packr/v2 v2.8.0/go.mod h1:PDk2k3vGevNE3SwVyVRgQCCXETC9SaONCNSXT1Q8M1g= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec= github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -217,6 +230,7 @@ github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c/go.mod h1:cHMBumiwaaRxRQ6NT8sU3zQSkXbYaPjbBcXa8UgTzAE= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -226,7 +240,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -243,7 +256,9 @@ github.com/interpose/middleware v0.0.0-20150216143757-05ed56ed52fa/go.mod h1:eMb github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -271,14 +286,23 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ldsec/lattigo/v2 v2.0.0 h1:P3rzbXFrc45swwl0u/zVCfcSDuIjTpIGueEDVQQzgEY= github.com/ldsec/lattigo/v2 v2.0.0/go.mod h1:Iu8ol3XIWyF54Ka5bBBWNnNR9kYtymmuYo1Y6Hxzr+U= +github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 h1:gwm/c1H9vI+kuoIBBHqDU2zjhuZ3kyKlBxdTBt5LCug= github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4/go.mod h1:/uvhNx7QKG/gNJh5pWYSNWhbO4xffJ6SC4oV91yftp4= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4= github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= +github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc= github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= +github.com/lestrrat-go/iter v1.0.1 h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A= github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= +github.com/lestrrat-go/jwx v1.2.9 h1:kS8kLI4oaBYJJ6u6rpbPI0tDYVCqo0P5u8vv1zoQ49U= github.com/lestrrat-go/jwx v1.2.9/go.mod h1:25DcLbNWArPA/Ew5CcBmewl32cJKxOk5cbepBsIJFzw= +github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -308,12 +332,14 @@ github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7p github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk= github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -322,16 +348,24 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/philippgille/gokv v0.0.0-20191001201555-5ac9a20de634/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= github.com/philippgille/gokv v0.5.1-0.20191011213304-eb77f15b9c61/go.mod h1:OCoWPt+mbYuTO1FUVrQ2SxQU0oaaHBsn6lRhFX3JHOc= +github.com/philippgille/gokv v0.6.0 h1:fNEx/tSwV73nzlYd3iRYB8F+SEVJNNFzH1gsaT8SK2c= github.com/philippgille/gokv v0.6.0/go.mod h1:tjXRFw9xDHgxLS8WJdfYotKGWp8TWqu4RdXjMDG/XBo= +github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61 h1:IgQDuUPuEFVf22mBskeCLAtvd5c9XiiJG2UYud6eGHI= github.com/philippgille/gokv/encoding v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:SjxSrCoeYrYn85oTtroyG1ePY8aE72nvLQlw8IYwAN8= +github.com/philippgille/gokv/file v0.6.0 h1:ySYotRmkwaJLDkNSdT7Q0iDQzKHhSdq+ornlBXWgKzI= github.com/philippgille/gokv/file v0.6.0/go.mod h1:L5ulK3F64mxW+8OvYFGE5bowupGO73JdQBh4qE2bgEw= +github.com/philippgille/gokv/s3 v0.6.0 h1:AdQv7H7P73TdHW1h0i6fFgK7uKTwYhYx66vmsnvRoIo= github.com/philippgille/gokv/s3 v0.6.0/go.mod h1:VX24mhIyCpdir7WrhUo1yJGuw6aSbNIuDgjUJGVhqKA= +github.com/philippgille/gokv/syncmap v0.6.0 h1:2eWC2J6mTyUsl687WuGoYPIiyqFiTBZU7hSKPlr0mK4= github.com/philippgille/gokv/syncmap v0.6.0/go.mod h1:ZekkiO1XY9XbjRv9iunxA6+POW9Tw/QHsLO9xAHEaxo= +github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61 h1:4tVyBgfpK0NSqu7tNZTwYfC/pbyWUR2y+O7mxEg5BTQ= github.com/philippgille/gokv/test v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:EUc+s9ONc1+VOr9NUEd8S0YbGRrQd/gz/p+2tvwt12s= +github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61 h1:ril/jI0JgXNjPWwDkvcRxlZ09kgHXV2349xChjbsQ4o= github.com/philippgille/gokv/util v0.0.0-20191011213304-eb77f15b9c61/go.mod h1:2dBhsJgY/yVIkjY5V3AnDUxUbEPzT6uQ3LvoVT8TR20= github.com/phyber/negroni-gzip v1.0.0/go.mod h1:poOYjiFVKpeib8SnUpOgfQGStKNGLKsM8l09lOTNeyw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -350,6 +384,7 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -398,11 +433,14 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= go.dedis.ch/kyber/v3 v3.0.12/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= +go.dedis.ch/kyber/v3 v3.0.13 h1:s5Lm8p2/CsTMueQHCN24gPpZ4couBBeKU7r2Yl6r32o= go.dedis.ch/kyber/v3 v3.0.13/go.mod h1:kXy7p3STAurkADD+/aZcsznZGKVHEqbtmdIzvPfrs1U= +go.dedis.ch/onet/v3 v3.2.3 h1:Ps05qwaT/In9WgoJxJimKW7FBpkGAxj+AkDIhe+HdAs= go.dedis.ch/onet/v3 v3.2.3/go.mod h1:rKEuXJJiH44kj1VwlgmjT9PnqVItC5jos1W4/t1TMXQ= go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= @@ -438,9 +476,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -536,9 +576,12 @@ golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -553,6 +596,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/satori/go.uuid.v1 v1.2.0/go.mod h1:kjjdhYBBaa5W5DYP+OcVG3fRM6VWu14hqDYST4Zvw+E= diff --git a/internal/plugin_test.go b/internal/plugin_test.go index 71823e4..51b59a2 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -4,24 +4,16 @@ import ( "plugin" "testing" - "github.com/ldsec/geco/pkg/sdk" + gecoconf "github.com/ldsec/geco/pkg/common/configuration" + "github.com/ldsec/geco/pkg/datamanager" + gecosdk "github.com/ldsec/geco/pkg/sdk" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) -func TestPlugin(t *testing.T) { - p, err := plugin.Open("../build/geco-i2b2-data-source.so") - require.NoError(t, err) - - dsSymbol, err := p.Lookup("DataSourcePluginFactory") - require.NoError(t, err) - - pluginFactory, ok := dsSymbol.(*sdk.DataSourcePluginFactory) - t.Logf("%T", pluginFactory) - t.Logf("%T", dsSymbol) - - require.True(t, ok) +const pluginPath = "../build/geco-i2b2-data-source.so" +func getTestConfig() map[string]string { config := make(map[string]string) config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" config["i2b2.api.domain"] = "i2b2demo" @@ -37,6 +29,29 @@ func TestPlugin(t *testing.T) { config["db.schema-name"] = "gecodatasourceplugintest" config["db.user"] = "postgres" config["db.password"] = "postgres" + return config +} + +func TestPlugin(t *testing.T) { + p, err := plugin.Open(pluginPath) + require.NoError(t, err) + + dsTypeSymbol, err := p.Lookup("DataSourceType") + require.NoError(t, err) + + dsType, ok := dsTypeSymbol.(*gecosdk.DataSourceType) + require.True(t, ok) + require.EqualValues(t, "i2b2-medco", *dsType) + + dsSymbol, err := p.Lookup("DataSourcePluginFactory") + require.NoError(t, err) + + pluginFactory, ok := dsSymbol.(*gecosdk.DataSourcePluginFactory) + t.Logf("%T", pluginFactory) + t.Logf("%T", dsSymbol) + require.True(t, ok) + + config := getTestConfig() logrus.StandardLogger().SetLevel(logrus.DebugLevel) ds, err := (*pluginFactory)(logrus.StandardLogger(), config) @@ -47,3 +62,23 @@ func TestPlugin(t *testing.T) { require.NoError(t, err) t.Logf("result: %v", string(res)) } + +func TestPluginDataManager(t *testing.T) { + dm, err := datamanager.NewDataManager(gecoconf.NewTestDataManagerConfig()) + require.NoError(t, err) + + err = datamanager.LoadDataSourcePlugin(pluginPath) + require.NoError(t, err) + + config := getTestConfig() + ds, err := dm.NewDataSource("", "i2b2-medco-test", "test", "i2b2-medco", config, false) + require.NoError(t, err) + + params := `{"path": "/", "operation": "children"}` + res, err := ds.Query("testUser", "searchConcept", []byte(params), nil) + require.NoError(t, err) + t.Logf("result: %v", string(res)) + + dsFound, err := dm.GetDataSource(ds.UniqueID()) + require.Equal(t, &ds, dsFound) +} diff --git a/third_party/geco b/third_party/geco index 35f2874..eb7b2da 160000 --- a/third_party/geco +++ b/third_party/geco @@ -1 +1 @@ -Subproject commit 35f2874b93b7975991f66f1832cac6ac0fbdf80f +Subproject commit eb7b2dac35a08cca5ffba8eeec5da4be6e424fc4 From b5430be066a9cf2cb2aed9b544c231b8f5cdac93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 22 Dec 2021 17:55:11 +0100 Subject: [PATCH 64/81] update geco version --- third_party/geco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/geco b/third_party/geco index eb7b2da..8d169f2 160000 --- a/third_party/geco +++ b/third_party/geco @@ -1 +1 @@ -Subproject commit eb7b2dac35a08cca5ffba8eeec5da4be6e424fc4 +Subproject commit 8d169f2094b333379c09859492b636c96d6be85a From 788e9a38cb465fc1d636f876ca3247de4c1bcd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 23 Dec 2021 15:59:39 +0100 Subject: [PATCH 65/81] updated go.sum with LFS fix --- go.sum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 0b08f42..498c558 100644 --- a/go.sum +++ b/go.sum @@ -288,7 +288,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ldsec/lattigo/v2 v2.0.0 h1:P3rzbXFrc45swwl0u/zVCfcSDuIjTpIGueEDVQQzgEY= github.com/ldsec/lattigo/v2 v2.0.0/go.mod h1:Iu8ol3XIWyF54Ka5bBBWNnNR9kYtymmuYo1Y6Hxzr+U= -github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 h1:gwm/c1H9vI+kuoIBBHqDU2zjhuZ3kyKlBxdTBt5LCug= +github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4 h1:RCds6LS6RVHUI/vw64tgVoc/v0qtvtOygSBS2YEo/Bk= github.com/ldsec/spindle v1.0.1-0.20211101151719-eecd1e98a4e4/go.mod h1:/uvhNx7QKG/gNJh5pWYSNWhbO4xffJ6SC4oV91yftp4= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= From 8539c48e03c56ce79c6d139ce18998699473f3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 23 Dec 2021 17:20:17 +0100 Subject: [PATCH 66/81] update geco version --- third_party/geco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/geco b/third_party/geco index 8d169f2..69ce4b2 160000 --- a/third_party/geco +++ b/third_party/geco @@ -1 +1 @@ -Subproject commit 8d169f2094b333379c09859492b636c96d6be85a +Subproject commit 69ce4b2df48cd8b62d7c83da50df43bde37889bb From b3539e4ca827c9cf9992bbe0d6b55eb298490c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 23 Dec 2021 17:31:15 +0100 Subject: [PATCH 67/81] CI: enable push of i2b2 image --- .github/workflows/ci-cd.yml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 7f15711..e425833 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -1,5 +1,4 @@ name: Build and test GeCo i2b2 data source -# todo: i2b2 push when? is build fast enough? on: push: workflow_dispatch: @@ -7,8 +6,8 @@ jobs: ci-cd: name: Build and test GeCo i2b2 data source runs-on: ubuntu-latest -# env: -# PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} todo: also put if manual trigger + env: + PUSH: ${{ github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags') }} steps: - name: Set up Go @@ -92,14 +91,14 @@ jobs: if: always() run: make i2b2-docker-compose ARGS="logs" -# - name: Login to GitHub Container Registry -# if: ${{ env.PUSH }} -# uses: docker/login-action@v1 -# with: -# registry: ghcr.io -# username: ${{ secrets.LDS_PKG_PAT_USERNAME }} -# password: ${{ secrets.LDS_PKG_PAT }} - -# - name: Push GeCo Docker image -# if: ${{ env.PUSH }} -# run: docker push ${{ steps.get_versions.outputs.i2b2_dockerss_image }} + - name: Login to GitHub Container Registry + if: ${{ env.PUSH }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.LDS_PKG_PAT_USERNAME }} + password: ${{ secrets.LDS_PKG_PAT }} + + - name: Push i2b2 Docker image + if: ${{ env.PUSH }} + run: docker push ${{ steps.get_versions.outputs.i2b2_docker_image }} From 30f5170f5fa846d05ebd12e0289433071bb9addc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 23 Dec 2021 17:34:28 +0100 Subject: [PATCH 68/81] update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d7c8cc9..bb17b23 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,9 @@ make go-unit tests - `build/package/i2b2/`: i2b2 docker image definition - `cmd/geco-i2b2-data-source/`: go main package for the plugin - `pkg/`: exported go code - - `i2b2api/`: client for i2b2 HTTP XML API - - `i2b2datasource/`: definition of the i2b2 GeCo data source + - `i2b2client/`: client for i2b2 HTTP XML API + - `datasource/`: definition of the i2b2 GeCo data source + - `database/`: database wrapper for the datasource - `scripts/`: utility scripts - `test/i2b2/`: test files for the i2b2 docker image - `third_party/geco/`: git submodule for the GeCo source code From 01ac9e78e02961e0c74211918261cb22f9c55522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Thu, 23 Dec 2021 17:41:34 +0100 Subject: [PATCH 69/81] CI: disable i2b2 logs and fix push --- .github/workflows/ci-cd.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index e425833..527d535 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -87,12 +87,13 @@ jobs: if: always() run: make geco-docker-compose ARGS="logs" - - name: Show i2b2 deployment logs - if: always() - run: make i2b2-docker-compose ARGS="logs" + # disabled because takes ~1 minute + #- name: Show i2b2 deployment logs + # if: always() + # run: make i2b2-docker-compose ARGS="logs" - name: Login to GitHub Container Registry - if: ${{ env.PUSH }} + if: ${{ env.PUSH == 'true' }} uses: docker/login-action@v1 with: registry: ghcr.io @@ -100,5 +101,5 @@ jobs: password: ${{ secrets.LDS_PKG_PAT }} - name: Push i2b2 Docker image - if: ${{ env.PUSH }} + if: ${{ env.PUSH == 'true' }} run: docker push ${{ steps.get_versions.outputs.i2b2_docker_image }} From cc6ea3171e0322a70ca6fafa96f3d21cb553a638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 08:50:03 +0100 Subject: [PATCH 70/81] CI: show i2b2 deployment logs when CI fails --- .github/workflows/ci-cd.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 527d535..3a11015 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -87,10 +87,9 @@ jobs: if: always() run: make geco-docker-compose ARGS="logs" - # disabled because takes ~1 minute - #- name: Show i2b2 deployment logs - # if: always() - # run: make i2b2-docker-compose ARGS="logs" + - name: Show i2b2 deployment logs (only in case of failure) + if: failure() + run: make i2b2-docker-compose ARGS="logs" - name: Login to GitHub Container Registry if: ${{ env.PUSH == 'true' }} From 78fd25bc5db26ca4649998c64e21f19f26ae4a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 09:02:07 +0100 Subject: [PATCH 71/81] update i2b2 deployment port 8080>8081 --- deployments/i2b2.yml | 2 +- internal/plugin_test.go | 2 +- pkg/datasource/datasource_test.go | 2 +- test/i2b2/test_i2b2_docker.sh | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/i2b2.yml b/deployments/i2b2.yml index 5bb39ef..271c22b 100644 --- a/deployments/i2b2.yml +++ b/deployments/i2b2.yml @@ -5,7 +5,7 @@ services: build: context: ../build/package/i2b2 ports: - - "8080:8080" + - "8081:8080" environment: - I2B2_DB_HOST=postgresql - I2B2_DB_PORT=5432 diff --git a/internal/plugin_test.go b/internal/plugin_test.go index 51b59a2..36d0047 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -15,7 +15,7 @@ const pluginPath = "../build/geco-i2b2-data-source.so" func getTestConfig() map[string]string { config := make(map[string]string) - config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" + config["i2b2.api.url"] = "http://localhost:8081/i2b2/services" config["i2b2.api.domain"] = "i2b2demo" config["i2b2.api.username"] = "demo" config["i2b2.api.password"] = "changeme" diff --git a/pkg/datasource/datasource_test.go b/pkg/datasource/datasource_test.go index a44188f..f62d9de 100644 --- a/pkg/datasource/datasource_test.go +++ b/pkg/datasource/datasource_test.go @@ -13,7 +13,7 @@ import ( func getDataSource(t *testing.T) *I2b2DataSource { config := make(map[string]string) - config["i2b2.api.url"] = "http://localhost:8080/i2b2/services" + config["i2b2.api.url"] = "http://localhost:8081/i2b2/services" config["i2b2.api.domain"] = "i2b2demo" config["i2b2.api.username"] = "demo" config["i2b2.api.password"] = "changeme" diff --git a/test/i2b2/test_i2b2_docker.sh b/test/i2b2/test_i2b2_docker.sh index b0d7451..8e59164 100755 --- a/test/i2b2/test_i2b2_docker.sh +++ b/test/i2b2/test_i2b2_docker.sh @@ -7,14 +7,14 @@ function postXmlI2b2 { RESP_XML="$3" echo "HTTP POST $I2B2_ENDPOINT of $REQ_XML..." - curl -v --header "Content-Type:application/xml" -d "@$REQ_XML" -o "$RESP_XML" "http://localhost:8080/i2b2/services/$I2B2_ENDPOINT" > "$RESP_XML.log" 2>&1 + curl -v --header "Content-Type:application/xml" -d "@$REQ_XML" -o "$RESP_XML" "http://localhost:8081/i2b2/services/$I2B2_ENDPOINT" > "$RESP_XML.log" 2>&1 echo "Checking HTTP code..." grep "HTTP/1.1 200 OK" "$RESP_XML.log" } # wait for i2b2 to be up -until curl -v http://localhost:8080/i2b2/services/listServices; do +until curl -v http://localhost:8081/i2b2/services/listServices; do >&2 echo "Waiting for i2b2..." sleep 1 done From 0edeb6c0cb493b8684edc86fc6ec6901e76e2fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 09:02:44 +0100 Subject: [PATCH 72/81] Update README.md Co-authored-by: Francesco Marino <francesco.marino@epfl.ch> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb17b23..736e263 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Build Retrieve git submodule of GeCo: ```shell -make git submodule update --init --recursive +git submodule update --init --recursive ``` Generate GeCo Swagger files: From 65b6b4e2c1474d3d594bb860cdb02246f4aef9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 09:03:00 +0100 Subject: [PATCH 73/81] Update README.md Co-authored-by: Francesco Marino <francesco.marino@epfl.ch> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 736e263..8ccff9e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ make i2b2-test Run go unit tests i2b2 docker: ```shell -make go-unit tests +make go-unit-tests ``` ## Development From 5fa578775365829e65a26f1deefea73c725cf799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 11:21:58 +0100 Subject: [PATCH 74/81] add API documentation --- API.md | 238 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 3 + 2 files changed, 241 insertions(+) create mode 100644 API.md diff --git a/API.md b/API.md new file mode 100644 index 0000000..d57edf9 --- /dev/null +++ b/API.md @@ -0,0 +1,238 @@ +All the operations exposed by this data source plugin to the GeCo Data Manager +runtime are listed here, with their supported parameters, results, and output +data objects shared IDs. + +# searchConcept +Exploration of the concepts of the tree-like ontology. + +## Parameters +```json +{ + "Path": "/TEST/test", + "Operation": "children|info" +} +``` + +- `Path`: path to the requested concept +- `Operation`: + - `info`: request metadata about the concept itself + - `children`: request children of the concept + +## Results +```json +{ + "SearchResults": [{ + "Path": "/TEST/test", + "AppliedPath": "x", + "Name": "x", + "DisplayName": "x", + "Code": "x", + "Comment": "x", + "Type": "concept|concept_container|concept_folder|modifier|modifier_container|modifier_folder|genomic_annotation", + "Leaf": true, + "Metadata": { + "DataType": "PosInteger|Integer|Float|PosFloat|Enum|String", + "OkToUseValues": "Y", + "UnitValues": { + "NormalUnits": "x" + } + } + }] +} +``` + +- `SearchResults`: array of results, either concepts or modifiers + - `Path`: path to the modifier or concept + - `AppliedPath`: path(s) onto which the modifier applies (if a modifier) + - `Name`: name of the element + - `DisplayName`: nicely formatted name of the element + - `Code`: i2b2 two-elements code + - `Comment`: comment that can be used as tooltip + - `Type`: type of the element + - `concept`: concept + - `concept_container`: concept with children, queryable + - `concept_folder`: concept with children, not queryable + - `modifier`: modifier + - `modifier_container`: modifier with children, queryable + - `modifier_folder`: modifier with children, not queryable + - `genomic_annotations`: genomic annotation + - `Leaf`: true if element is a leaf, i.e. does not have children + - `Metadata`: some additional metadata + - `DataType`: detailed type of data + - `PosInteger`: positive integers + - `Integer`: integers + - `PosFloat`: positive floats + - `Float`: floats + - `Enum`: enumerated values (string) + - `String`: free text (string) + - `OkToUseValues`: is a "Y" if can use values, a message saying why otherwise + - `UnitValues`: metadata about unit of the value + - `NormalUnits`: specify the unit of the value + +# searchModifier +Exploration of the modifiers of the tree-like ontology. + +## Parameters +```json +{ + "Path": "/TEST/modifiers/", + "AppliedPath": "/test/%", + "AppliedConcept": "/TEST/test/1/", + "Operation": "concept|children|info" +} +``` + +- `Path`: path to the requested modifier or concept +- `AppliedPath`: path(s) onto which the modifier applies +- `AppliedConcept`: concept onto which the modifier applies +- `Operation`: + - `info`: request metadata about the modifier itself + - `children`: request children of the modifier + - `concept`: request modifiers of the requested concept + +## Results +See results of `searchConcept`. + +# exploreQuery +Retrieve patient IDs from i2b2 based on explore query terms. +[See i2b2 CRC API for more details.](https://www.i2b2.org/software/files/PDF/current/CRC_Messaging.pdf) + +## Parameters +```json +{ + "ID": "99999999-9999-9999-9999-999999999999", + "Definition": { + "Panels": [{ + "Not": false, + "Timing": "any|samevisit|sameinstancenum", + "CohortItems": ["cohortName0", "cohortName1"], + "ConceptItems": [{ + "QueryTerm": "/TEST/test/1/", + "Operator": "EQ|NE|GT|GE|LT||LE|BETWEEN|IN|LIKE[exact]|LIKE[begin]|LIKE[end]|LIKE[contains]", + "Value": "xxx", + "Type": "NUMBER|TEXT", + "Modifier": { + "AppliedPath": "/test/1/", + "Key": "/TEST/modifiers/1/" + } + }] + }], + "Timing": "any|samevisit|sameinstancenum" + } +} +``` + +- `ID`: ID of the query, must be an UUID +- `Definition`: definition of the explore query + - `Panels`: panels of the explore query (linked together by an AND) + - `Not`: true if the panel is inverted + - `Timing`: timing of the panel + - `any`: no constrain (default) + - `samevisit`: constrain to the same visit + - `sameinstancenum`: constrain to the same instance number + - `CohortItems`: array of cohort names if querying for cohorts (linked together by an OR) + - `ConceptItems`: array of concepts if querying for concepts (linked together by an OR) + - `QueryTerm`: path to the queried concept + - `Operator`: apply an operator to the queried concept + - `EQ`: equal (type=NUMBER) + - `NE`: not equal (type=NUMBER) + - `GT`: greater (type=NUMBER) + - `GE`: greater or equal (type=NUMBER) + - `LT`: less (type=NUMBER) + - `LE`: less or equal (type=NUMBER) + - `BETWEEN`: between values, value example: "100 and 200" (type=NUMBER) + - `IN`: value among set, value example: "('NEG','NEGATIVE')" (type=TEXT) + - `LIKE[exact]`: string is equal to (type=TEXT) + - `LIKE[begin]`: string begins with (type=TEXT) + - `LIKE[end]`: string ends with (type=TEXT) + - `LIKE[contains]`: string contains (type=TEXT) + - `Value`: value to use with operator + - `Type`: type of concept + - `NUMBER`: numeric type + - `TEXT`: string type + - `Modifier`: apply a modifier to the queried concept + - `AppliedPath`: path(s) onto which the modifier applies + - `Key`: path of the modifier + - `Timing`: timing of the query + - `any`: no constrain (default) + - `samevisit`: constrain to the same visit + - `sameinstancenum`: constrain to the same instance number + +## Output Data Objects Shared IDs +- `count`: integer containing the count of patients +- `patientList`: vector of integers containing the patient IDs + +# getCohorts +Retrieve the list of saved cohorts. + +## Parameters +```json +{ + "Limit": 10 +} +``` + +- `Limit`: max number of cohorts to retrieve + +## Results +```json +{ + "Cohorts": [{ + "Name": "Cohort 1", + "CreationDate": "xxx", + "ExploreQuery": { + "ID": "99999999-9999-9999-9999-999999999999", + "CreationDate": "xxx", + "Status": "running|success|error", + "Definition": {}, + "OutputDataObjectsSharedIDs": { + "Count": "xxx", + "PatientList": "xxx" + } + } + }] +} +``` + +- `Cohorts`: array of cohorts + - `Name`: name of the cohort + - `CreationDate`: date of the creation of the cohort + - `ExploreQuery`: the query tied to the cohort + - `ID`: identifier of the query + - `CreationDate`: date of the creation of the query + - `Status`: status of the query + - `running`: query is running + - `success`: query successfully ran + - `error`: query has errored + - `Definition`: definition of the query (see above for syntax) + - `OutputDataObjectsSharedIDs`: + - `Count`: data object shared ID of the count + - `PatientList`: data object shared ID of the patient list + +# addCohort +Add a cohort. + +## Parameters +```json +{ + "Name": "Cohort 1", + "ExploreQueryID": "99999999-9999-9999-9999-999999999999" +} +``` + +- `Name`: name of the cohort +- `ExploreQueryID`: query to associate to the cohort + +# deleteCohort +Delete a cohort. + +## Parameters +```json +{ +"Name": "Cohort 1", +"ExploreQueryID": "99999999-9999-9999-9999-999999999999" +} +``` + +- `Name`: name of the cohort +- `ExploreQueryID`: query associated to the cohort diff --git a/README.md b/README.md index 8ccff9e..8b8f32b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # GeCo i2b2 Data Source Plugin +## GeCo Data Source Plugin API Operations +[The operations are documented here.](API.md) + ## Build Retrieve git submodule of GeCo: ```shell From 25efc4bb44a43fff3732905c8aadb04317500afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 11:22:38 +0100 Subject: [PATCH 75/81] rename field of cohort result for consistency in naming --- pkg/datasource/models/cohorts.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/datasource/models/cohorts.go b/pkg/datasource/models/cohorts.go index 90c6530..0e446d2 100644 --- a/pkg/datasource/models/cohorts.go +++ b/pkg/datasource/models/cohorts.go @@ -31,11 +31,11 @@ type Cohort struct { Name string CreationDate string ExploreQuery struct { - ID string - CreationDate string - Status string - Definition ExploreQueryDefinition - ResultSharedIDs struct { + ID string + CreationDate string + Status string + Definition ExploreQueryDefinition + OutputDataObjectsSharedIDs struct { Count string PatientList string } @@ -50,8 +50,8 @@ func NewCohortFromDbModel(dbCohort database.SavedCohort) (cohort Cohort) { cohort.ExploreQuery.ID = dbCohort.ExploreQuery.ID cohort.ExploreQuery.Status = dbCohort.ExploreQuery.Status cohort.ExploreQuery.CreationDate = dbCohort.ExploreQuery.CreateDate - cohort.ExploreQuery.ResultSharedIDs.Count = dbCohort.ExploreQuery.ResultGecoSharedIDCount.String - cohort.ExploreQuery.ResultSharedIDs.PatientList = dbCohort.ExploreQuery.ResultGecoSharedIDPatientList.String + cohort.ExploreQuery.OutputDataObjectsSharedIDs.Count = dbCohort.ExploreQuery.ResultGecoSharedIDCount.String + cohort.ExploreQuery.OutputDataObjectsSharedIDs.PatientList = dbCohort.ExploreQuery.ResultGecoSharedIDPatientList.String _ = json.Unmarshal([]byte(dbCohort.ExploreQuery.Definition), &cohort.ExploreQuery.Definition) return cohort From c64ca03825e25a81129afaeab50970b4176433f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 11:23:20 +0100 Subject: [PATCH 76/81] update in-code doc of fields of search parameters --- pkg/datasource/models/search.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/datasource/models/search.go b/pkg/datasource/models/search.go index 3791be2..fe8b3a7 100644 --- a/pkg/datasource/models/search.go +++ b/pkg/datasource/models/search.go @@ -7,14 +7,14 @@ import i2b2clientmodels "github.com/ldsec/geco-i2b2-data-source/pkg/i2b2client/m // SearchConceptParameters is the parameter for the SearchConcept operation. type SearchConceptParameters struct { Path string - Operation string // children | info | concept + Operation string // children | info } // SearchModifierParameters is the parameter for the SearchModifier operation. type SearchModifierParameters struct { - SearchConceptParameters - AppliedPath string - AppliedConcept string + SearchConceptParameters // Operation: children | info | concept + AppliedPath string + AppliedConcept string } // --- results From 508940f993711f0c0e8321b8abe4a4824ebda560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Tue, 18 Jan 2022 11:37:01 +0100 Subject: [PATCH 77/81] add readmes for packages --- cmd/geco-i2b2-data-source/README.md | 4 ++++ internal/README.md | 4 ++++ pkg/datasource/README.md | 4 ++++ pkg/datasource/database/README.md | 3 +++ pkg/datasource/models/README.md | 4 ++++ pkg/i2b2client/README.md | 4 ++++ pkg/i2b2client/models/README.md | 4 ++++ 7 files changed, 27 insertions(+) create mode 100644 cmd/geco-i2b2-data-source/README.md create mode 100644 internal/README.md create mode 100644 pkg/datasource/README.md create mode 100644 pkg/datasource/database/README.md create mode 100644 pkg/datasource/models/README.md create mode 100644 pkg/i2b2client/README.md create mode 100644 pkg/i2b2client/models/README.md diff --git a/cmd/geco-i2b2-data-source/README.md b/cmd/geco-i2b2-data-source/README.md new file mode 100644 index 0000000..e1432c0 --- /dev/null +++ b/cmd/geco-i2b2-data-source/README.md @@ -0,0 +1,4 @@ +# Package cmd/geco-i2b2-data-source + +This package contains the exported fields of the plugin, conforming to both +the go plugin package and to the GeCo data source plugin interface. diff --git a/internal/README.md b/internal/README.md new file mode 100644 index 0000000..6373148 --- /dev/null +++ b/internal/README.md @@ -0,0 +1,4 @@ +# Package internal + +This internal package contains a test of the plugin in a compiled form that must +be separated from the rest of the sources. diff --git a/pkg/datasource/README.md b/pkg/datasource/README.md new file mode 100644 index 0000000..e1b695c --- /dev/null +++ b/pkg/datasource/README.md @@ -0,0 +1,4 @@ +# Package pkg/datasource + +This package exposes the i2b2 data source conforming to the GeCo data source plugin interface. +It implements the different operations of the i2b2 data source. diff --git a/pkg/datasource/database/README.md b/pkg/datasource/database/README.md new file mode 100644 index 0000000..483baf0 --- /dev/null +++ b/pkg/datasource/database/README.md @@ -0,0 +1,3 @@ +# Package pkg/datasource/database + +This package implements the database access of the data source to its postgresql database. diff --git a/pkg/datasource/models/README.md b/pkg/datasource/models/README.md new file mode 100644 index 0000000..d1191ca --- /dev/null +++ b/pkg/datasource/models/README.md @@ -0,0 +1,4 @@ +# Package pkg/datasource/models + +This package contains the models of the data source plugin. +It implements the JSON messages exchanged through the GeCo Data Manager. diff --git a/pkg/i2b2client/README.md b/pkg/i2b2client/README.md new file mode 100644 index 0000000..acffe94 --- /dev/null +++ b/pkg/i2b2client/README.md @@ -0,0 +1,4 @@ +# Package pkg/i2b2client + +This package exposes a client for the i2b2 XML API. +It implements a selection of requests for the ONT and CRC cells. diff --git a/pkg/i2b2client/models/README.md b/pkg/i2b2client/models/README.md new file mode 100644 index 0000000..9929f81 --- /dev/null +++ b/pkg/i2b2client/models/README.md @@ -0,0 +1,4 @@ +# Package pkg/i2b2client/models + +This package contains the models of the client for the i2b2 XML API. +It implements the XML messages exchanged with i2b2. From 4b13e5110be31da04ad139bff1e14f4e688e7751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Wed, 19 Jan 2022 19:21:12 +0100 Subject: [PATCH 78/81] update references medco>geco --- build/package/i2b2/sql/10-i2b2-modifications.sh | 2 +- cmd/geco-i2b2-data-source/data-source-plugin.go | 2 +- internal/plugin_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/package/i2b2/sql/10-i2b2-modifications.sh b/build/package/i2b2/sql/10-i2b2-modifications.sh index d3f0c1c..64841eb 100644 --- a/build/package/i2b2/sql/10-i2b2-modifications.sh +++ b/build/package/i2b2/sql/10-i2b2-modifications.sh @@ -1,7 +1,7 @@ #!/bin/bash set -Eeuo pipefail source "$(dirname "$0")/common.sh" -# set up common medco ontology +# update structure of default i2b2 DB psql $PSQL_PARAMS -d "$I2B2_DB_NAME" <<-EOSQL diff --git a/cmd/geco-i2b2-data-source/data-source-plugin.go b/cmd/geco-i2b2-data-source/data-source-plugin.go index 5eab8f2..6e3a966 100644 --- a/cmd/geco-i2b2-data-source/data-source-plugin.go +++ b/cmd/geco-i2b2-data-source/data-source-plugin.go @@ -6,7 +6,7 @@ import ( ) // DataSourceType is the type of the data source. -var DataSourceType sdk.DataSourceType = "i2b2-medco" +var DataSourceType sdk.DataSourceType = "i2b2-geco" // DataSourcePluginFactory exports a factory function compatible with GeCo data source plugin SDK. var DataSourcePluginFactory sdk.DataSourcePluginFactory = datasource.NewI2b2DataSource diff --git a/internal/plugin_test.go b/internal/plugin_test.go index 36d0047..9f7e6ee 100644 --- a/internal/plugin_test.go +++ b/internal/plugin_test.go @@ -41,7 +41,7 @@ func TestPlugin(t *testing.T) { dsType, ok := dsTypeSymbol.(*gecosdk.DataSourceType) require.True(t, ok) - require.EqualValues(t, "i2b2-medco", *dsType) + require.EqualValues(t, "i2b2-geco", *dsType) dsSymbol, err := p.Lookup("DataSourcePluginFactory") require.NoError(t, err) @@ -71,7 +71,7 @@ func TestPluginDataManager(t *testing.T) { require.NoError(t, err) config := getTestConfig() - ds, err := dm.NewDataSource("", "i2b2-medco-test", "test", "i2b2-medco", config, false) + ds, err := dm.NewDataSource("", "i2b2-geco-test", "test", "i2b2-geco", config, false) require.NoError(t, err) params := `{"path": "/", "operation": "children"}` From 5945adcefa3bc557ff5d9b20634eda365978eb8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 18 Feb 2022 12:48:04 +0100 Subject: [PATCH 79/81] add READMEs to i2b2 docker image --- build/package/i2b2/README.md | 17 +++++++ build/package/i2b2/patches/README.md | 8 ++++ build/package/i2b2/pre-init-scripts/README.md | 18 +++++++ build/package/i2b2/sql/README.md | 47 +++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 build/package/i2b2/README.md create mode 100644 build/package/i2b2/patches/README.md create mode 100644 build/package/i2b2/pre-init-scripts/README.md create mode 100644 build/package/i2b2/sql/README.md diff --git a/build/package/i2b2/README.md b/build/package/i2b2/README.md new file mode 100644 index 0000000..9b8ae3e --- /dev/null +++ b/build/package/i2b2/README.md @@ -0,0 +1,17 @@ +# I2b2 Docker Image + +This folder contains an i2b2 Docker image which loads its data structure at the first startup and with test data. +It is configurable through environment variables and easily customisable, either with patches to the source code or with +tweaks to the data loaded. + +## Source code organization +- `patches/`: (see README.md)[patches/README.md] +- `pre-init-scripts/`: (see README.md)[pre-init-scripts/README.md] +- `sql/`: (see README.md)[sql/README.md] +- `docker-entrypoint.sh`: entrypoint for the docker container, it waits for the + database to be available and then triggers the data loading if needed before starting i2b2 +- `Dockerfile`: the dockerfile defining the image +- `download-i2b2-sources.sh`: scripts used during the image build that downloads the i2b2 source code and its data definitions +- `I2b2PasswordHash.java`: a Java snippet that replicates the password hashing function of i2b2, allowing to set i2b2 + passwords directly in the database +- `install-i2b2.sh`: script that compiles and install i2b2 in the docker image at build time diff --git a/build/package/i2b2/patches/README.md b/build/package/i2b2/patches/README.md new file mode 100644 index 0000000..968e1af --- /dev/null +++ b/build/package/i2b2/patches/README.md @@ -0,0 +1,8 @@ +# Patches + +The `install-i2b2.sh` script will apply to the i2b2 source code all the patches contained in this folder before compiling i2b2. + +The patches must have the file extension `.diff` and be generated by git with `git diff -p` as they are applied with `git apply`. + +## `crc_pg_db_fix.diff` +This patch fixes an i2b2 bug in the CRC that is triggered only when a postgresql database is used. diff --git a/build/package/i2b2/pre-init-scripts/README.md b/build/package/i2b2/pre-init-scripts/README.md new file mode 100644 index 0000000..688160a --- /dev/null +++ b/build/package/i2b2/pre-init-scripts/README.md @@ -0,0 +1,18 @@ +# Pre-init scripts + +The `docker-entrypoint.sh` startup script of the Docker image will run every time all the scripts contained in this +folder before starting i2b2. +The scripts are executed in alphanumerical order, so a numbering in the filename should be used. + +The scripts must have the file extension `.sh` and handle failures correctly, e.g. by using `set -Eeuo pipefail`. + +## `05-wildfly-config.sh` +This script applies some configuration of the wildfly instance that runs i2b2. +It is used to set the password of the `admin` user of wildfly controlled by the environment variable `$WILDFLY_ADMIN_PASSWORD`. + +## `10-i2b2-config.sh` +This script applies some configuration of the i2b2 instance. +It is used to set the logging level of the axis2 instance running i2b2 controlled by the environment variable `$AXIS2_LOGLEVEL`. + +## `10-write-i2b2-datasources.sh` +This script applies the configuration of the i2b2 database credentials (i.e. the data sources). diff --git a/build/package/i2b2/sql/README.md b/build/package/i2b2/sql/README.md new file mode 100644 index 0000000..787fd5d --- /dev/null +++ b/build/package/i2b2/sql/README.md @@ -0,0 +1,47 @@ +# SQL scripts +The `docker-entrypoint.sh` startup script of the Docker image will run the scripts contained in this folder before +starting i2b2, in order to do the initial loading of the i2b2 data in the postgresql database. +This loading is performed only if the i2b2 database does not exist already, i.e. if the data was not previously loaded. +The scripts are executed in alphanumerical order, so a numbering in the filename should be used. + +The scripts must have the file extension `.sh` and handle failures correctly, e.g. by using `set -Eeuo pipefail`. + +## `common.sh` +This script provides to other scripts common functions, it is meant to be sourced with `source "$(dirname "$0")/common.sh"`. +It contains the function that initialises an i2b2 schema. + +## `05-i2b2-orig-data.sh` +This script loads the i2b2 untouched original data in the database as instructed in the i2b2 documentation. +Note that only the structure is loaded, i.e. the demo data is not loaded in order to save time. + +## `10-i2b2-modifications.sh` +This scripts does modifications to the i2b2 original data structure loaded in the database, it should be used in order +to persist some modifications of the i2b2 database structure when it is needed. +Currently, it increases the size of some fields and modify the type of some other fields. + +## `20-i2b2-configuration.sh` +This script configures i2b2 through the database. +For the hive configuration it adds the database lookups and sets some parameters of the CRC cell. +For the PM configuration it sets some parameters of the cell (notably the directory of the file repository), +sets the information about the project, and sets the passwords of the i2b2 users. + +## `50-stored-procedures.sh` +This scripts loads the stored procedures located in `50-stored-procedures/`. +The procedures must have the file extension `.plpgql` and contain SQL code loaded the procedure. + +### `50-stored-procedures/get_concept_codes.plpgsql` +A PL/pgSQL function that returns the concept codes for a given concept path and its descendants. + +### `50-stored-procedures/get_modifier_codes.plpgsql` +A PL/pgSQL function that returns the modifier codes for a given modifier path and its descendants, for a given applied +path. + +### `50-stored-procedures/get_ontology_elements.plpgsql` +A PL/pgSQL function that returns a set number of ontology elements whose paths contain the given search_string. + +### `50-stored-procedures/table_name.plpgsql` +A PL/pgSQL function that returns the table name for a given table code. + +## `90-test-data.sh` +This scripts loads a set of test data that is expected to exist after the data loaded and that is used for example for +unit or integration tests. From c78d02f94206b7e6f408347f37755a4973ed3dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 18 Feb 2022 12:54:35 +0100 Subject: [PATCH 80/81] add env variables of i2b2 docker image in readme --- build/package/i2b2/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/package/i2b2/README.md b/build/package/i2b2/README.md index 9b8ae3e..0655c06 100644 --- a/build/package/i2b2/README.md +++ b/build/package/i2b2/README.md @@ -1,5 +1,7 @@ # I2b2 Docker Image +I2b2 is a Java app that runs in the web services engine Axis2 which itself runs in the WildFly application server. + This folder contains an i2b2 Docker image which loads its data structure at the first startup and with test data. It is configurable through environment variables and easily customisable, either with patches to the source code or with tweaks to the data loaded. @@ -15,3 +17,11 @@ tweaks to the data loaded. - `I2b2PasswordHash.java`: a Java snippet that replicates the password hashing function of i2b2, allowing to set i2b2 passwords directly in the database - `install-i2b2.sh`: script that compiles and install i2b2 in the docker image at build time + +# Configuration through environment variables +- `I2B2_DB_XXX`: sets the database connection information, note that the database user configured must have the right to create databases +- `WILDFLY_ADMIN_PASSWORD`: sets the password of the wildfly admin user +- `I2B2_DOMAIN_NAME`: sets the i2b2 domain name to be used +- `I2B2_SERVICE_PASSWORD`: sets the i2b2 service user password, which is not used except by i2b2 itself +- `DEFAULT_USER_PASSWORD`: sets the password of the i2b2 users `demo` (standard user) and `i2b2` (admin user) +- `AXIS2_LOGLEVEL`: sets the logging level of axis2 From 0e51ba42c933f9bcf43e7a5297cc80ef556e9304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Misbach?= <mickael@misba.ch> Date: Fri, 18 Feb 2022 13:05:33 +0100 Subject: [PATCH 81/81] fix links --- build/package/i2b2/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/package/i2b2/README.md b/build/package/i2b2/README.md index 0655c06..163a7b0 100644 --- a/build/package/i2b2/README.md +++ b/build/package/i2b2/README.md @@ -7,9 +7,9 @@ It is configurable through environment variables and easily customisable, either tweaks to the data loaded. ## Source code organization -- `patches/`: (see README.md)[patches/README.md] -- `pre-init-scripts/`: (see README.md)[pre-init-scripts/README.md] -- `sql/`: (see README.md)[sql/README.md] +- `patches/`: [see README.md](patches/README.md) +- `pre-init-scripts/`: [see README.md](pre-init-scripts/README.md) +- `sql/`: [see README.md](sql/README.md) - `docker-entrypoint.sh`: entrypoint for the docker container, it waits for the database to be available and then triggers the data loading if needed before starting i2b2 - `Dockerfile`: the dockerfile defining the image