Search App v2

docker-compose.yml

@@ -356,16 +356,17 @@ services:
       - ./solr/core_travelq:/var/solr/data/core_travelq
       - ./solr/core_wrongdoing:/var/solr/data/core_wrongdoing
       # Persisting Django Solr Cores
-      - ./solr/core_od_search:/var/solr/data/core_od_search
-      - ./solr/core_bn_search:/var/solr/data/core_bn_search
-      - ./solr/core_ati_search:/var/solr/data/core_ati_search
-      - ./solr/core_ei_search:/var/solr/data/core_ei_search
-      - ./solr/core_ct_search:/var/solr/data/core_ct_search
-      - ./solr/core_gc_search:/var/solr/data/core_gc_search
-      - ./solr/core_ap_search:/var/solr/data/core_ap_search
-      - ./solr/core_sv_search:/var/solr/data/core_sv_search
-      - ./solr/core_qp_search:/var/solr/data/core_qp_search
-      - ./solr/core_sd_search:/var/solr/data/core_sd_search
+      - ./solr/search_adminaircraft:/var/solr/data/search_adminaircraft
+      - ./solr/search_briefingt:/var/solr/data/search_briefingt
+      - ./solr/search_contracts:/var/solr/data/search_contracts
+      - ./solr/search_ei:/var/solr/data/search_ei
+      - ./solr/search_grants:/var/solr/data/search_grants
+      - ./solr/search_nap5:/var/solr/data/search_nap5
+      - ./solr/search_opencanada:/var/solr/data/search_opencanada
+      - ./solr/search_qpnotes:/var/solr/data/search_qpnotes
+      - ./solr/search_suggesteddatasets:/var/solr/data/search_suggesteddatasets
+      - ./solr/search_travela:/var/solr/data/search_travela
+      - ./solr/search_travelq:/var/solr/data/search_travelq
       # Persisting Drupal Solr Cores
       - ./solr/drupal_content:/var/solr/data/drupal_content
       # Persisting CKAN Test Solr Cores
@@ -381,16 +382,17 @@ services:
       - ./solr/test__core_travelq:/var/solr/data/test__core_travelq
       - ./solr/test__core_wrongdoing:/var/solr/data/test__core_wrongdoing
       # Persisting Django Test Solr Cores
-      - ./solr/test__core_od_search:/var/solr/data/test__core_od_search
-      - ./solr/test__core_bn_search:/var/solr/data/test__core_bn_search
-      - ./solr/test__core_ati_search:/var/solr/data/test__core_ati_search
-      - ./solr/test__core_ei_search:/var/solr/data/test__core_ei_search
-      - ./solr/test__core_ct_search:/var/solr/data/test__core_ct_search
-      - ./solr/test__core_gc_search:/var/solr/data/test__core_gc_search
-      - ./solr/test__core_ap_search:/var/solr/data/test__core_ap_search
-      - ./solr/test__core_sv_search:/var/solr/data/test__core_sv_search
-      - ./solr/test__core_qp_search:/var/solr/data/test__core_qp_search
-      - ./solr/test__core_sd_search:/var/solr/data/test__core_sd_search
+      - ./solr/test__search_adminaircraft:/var/solr/data/test__search_adminaircraft
+      - ./solr/test__search_briefingt:/var/solr/data/test__search_briefingt
+      - ./solr/test__search_contracts:/var/solr/data/test__search_contracts
+      - ./solr/test__search_ei:/var/solr/data/test__search_ei
+      - ./solr/test__search_grants:/var/solr/data/test__search_grants
+      - ./solr/test__search_nap5:/var/solr/data/test__search_nap5
+      - ./solr/test__search_opencanada:/var/solr/data/test__search_opencanada
+      - ./solr/test__search_qpnotes:/var/solr/data/test__search_qpnotes
+      - ./solr/test__search_suggesteddatasets:/var/solr/data/test__search_suggesteddatasets
+      - ./solr/test__search_travela:/var/solr/data/test__search_travela
+      - ./solr/test__search_travelq:/var/solr/data/test__search_travelq
       # Persisting Drupal Test Solr Cores
       - ./solr/test__drupal_content:/var/solr/data/test__drupal_content
     environment:
@@ -496,10 +498,10 @@ services:
       PATH: '$PATH:/usr/local/lib'
       APP_ROOT: /var/ocs
       PY_VERSION: 3.7
-      WET_VERSION: v4.0.63
-      CDTS_VERSION: v4_0_28
+      FGP_VERSION: 3.3.7
+      GCWEB_DISTRO_VERSION: 13.4.0
+      WET_DISTRO_VERSION: 4.0.64.1
       CDTS_GCWEB_VERSION: v4_0_31
-      GCWEB_VERSION: v13.3.0
       PYTHONHTTPSVERIFY: 0
       PYTHONDONTWRITEBYTECODE: 1
       SSL_VERIFY: 0

docker/config/nginx/proxied_sites/search.open.local

@@ -30,7 +30,7 @@ server {
 
     location /static/ {
 
-        alias /var/ogproxy/django/src/oc-search/oc_search/static/;
+        alias /var/ogproxy/django/static/;
         autoindex off;
 
     }

docker/install/django/install-searches.sh

@@ -30,171 +30,22 @@ if [[ $installSearches_Django == "true" ]]; then
   wget https://raw.githubusercontent.com/open-data/ckanext-canada/master/ckanext/canada/tables/choices/minister.json
   wget https://raw.githubusercontent.com/open-data/ckanext-canada/master/ckanext/canada/schemas/prop.yaml
 
-  # download oc_searches repo from https://github.com/thriuin/oc_searches
-  printf "${SPACER}${Cyan}${INDENT}Downloading search files from https://github.com/thriuin/oc_searches to temporary directory${NC}${EOL}"
-  mkdir -p ${APP_ROOT}/django/tmp
-  cd ${APP_ROOT}/django/tmp
-  wget https://github.com/thriuin/oc_searches/archive/refs/heads/main.zip
-  unzip main.zip
-  tmp_dir="${APP_ROOT}/django/tmp/oc_searches-main"
-
-  search_types="adminaircraft briefingt contracts data dataprops experiment grants nap5 qpnotes travela travelq"
-
-  # copy plugin files
-  printf "${SPACER}${Cyan}${INDENT}Copying plugin files...${NC}${EOL}"
-  function copy_plugin_files() {
-
-    search_type="$1"
-    printf "${Cyan}${INDENT}${INDENT}Copying plugin files for ${search_type}${NC}${EOL}"
-
-    if [[ ! -f "${tmp_dir}/${search_type}/plugins/${search_type}.py" ]]; then
-
-      printf "${Yellow}${INDENT}${INDENT}${search_type} has no plugin file: SKIPPING${NC}${EOL}"
-
-    else
-
-      cp ${tmp_dir}/${search_type}/plugins/${search_type}.py ${APP_ROOT}/django/src/oc-search/search/plugins/${search_type}.py
-
-      if [[ $? -eq 0 ]]; then
-        printf "${Green}${INDENT}${INDENT}Copy plugins/${search_type}.py: OK${NC}${EOL}"
-      else
-        printf "${Red}${INDENT}${INDENT}Copy plugins/${search_type}.py: FAIL${NC}${EOL}"
-      fi
-
-    fi
-
-  }
-
-  for type in $search_types; do
-    copy_plugin_files $type
-  done
-  # END copy plugin files END
-
-  # copy custom snippet files
-  printf "${SPACER}${Cyan}${INDENT}Copying snippet files...${NC}${EOL}"
-  function copy_snippet_files() {
-
-    search_type="$1"
-    printf "${Cyan}${INDENT}${INDENT}Copying snippet files for ${search_type}${NC}${EOL}"
-
-    if [[ ! -d "${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type}" ]]; then
-
-      mkdir -p ${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type}
-
-    fi
-
-    cp ${tmp_dir}/${search_type}/snippets/* ${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type}/
-
-    if [[ $? -eq 0 ]]; then
-      printf "${Green}${INDENT}${INDENT}Copy ${search_type}/snippets: OK${NC}${EOL}"
-    else
-      printf "${Red}${INDENT}${INDENT}Copy ${search_type}/snippets: FAIL${NC}${EOL}"
-    fi
-
-  }
-
-  for type in $search_types; do
-    copy_snippet_files $type
-  done
-  # END copy custom snippet files END
-
-  # copy command files
-  printf "${SPACER}${Cyan}${INDENT}Copying command files...${NC}${EOL}"
-  function copy_command_files() {
-
-    search_type="$1"
-    printf "${Cyan}${INDENT}${INDENT}Copying command files for ${search_type}${NC}${EOL}"
-
-    if [[ ! -d "${tmp_dir}/${search_type}/commands" ]]; then
-
-      printf "${Yellow}${INDENT}${INDENT}${search_type} has no command files: SKIPPING${NC}${EOL}"
-
-    else
-
-      cp ${tmp_dir}/${search_type}/commands/* ${APP_ROOT}/django/src/oc-search/search/management/commands/
-
-      if [[ $? -eq 0 ]]; then
-        printf "${Green}${INDENT}${INDENT}Copy ${search_type}/commands: OK${NC}${EOL}"
-      else
-        printf "${Red}${INDENT}${INDENT}Copy ${search_type}/commands: FAIL${NC}${EOL}"
-      fi
-
-    fi
-
-  }
-
-  for type in $search_types; do
-    copy_command_files $type
-  done
-  # END copy command files END
-
-  # copy locale files
-  printf "${SPACER}${Cyan}${INDENT}Copying locale files...${NC}${EOL}"
-  function copy_locale_files() {
-
-    search_type="$1"
-    printf "${Cyan}${INDENT}${INDENT}Copying command files for ${search_type}${NC}${EOL}"
-
-    if [[ ! -f "${tmp_dir}/${search_type}/locale/${search_type}.po" ]]; then
-
-      printf "${Yellow}${INDENT}${INDENT}${search_type} has no locale files: SKIPPING${NC}${EOL}"
-
-    else
-
-      cp ${tmp_dir}/${search_type}/locale/${search_type}.po ${APP_ROOT}/django/src/oc-search/locale/fr/LC_MESSAGES/${search_type}.po
-
-      if [[ $? -eq 0 ]]; then
-        printf "${Green}${INDENT}${INDENT}Copy locale/${search_type}.po: OK${NC}${EOL}"
-      else
-        printf "${Red}${INDENT}${INDENT}Copy locale/${search_type}.po: FAIL${NC}${EOL}"
-      fi
-
-    fi
-
-  }
-
-  for type in $search_types; do
-    copy_locale_files $type
-  done
-  # END copy locale files END
-
-  # copy database files
-  printf "${SPACER}${Cyan}${INDENT}Copying database files...${NC}${EOL}"
-  function copy_database_files() {
-
-    search_type="$1"
-    printf "${Cyan}${INDENT}${INDENT}Copying database files for ${search_type}${NC}${EOL}"
-
-    if [[ ! -d "${tmp_dir}/${search_type}/db" ]]; then
-
-      printf "${Yellow}${INDENT}${INDENT}${search_type} has no database files: SKIPPING${NC}${EOL}"
-
-    else
-
-      cp ${tmp_dir}/${search_type}/db/* ${APP_ROOT}/django/src/oc-search/data/
-      if [[ $? -eq 0 ]]; then
-        printf "${Green}${INDENT}${INDENT}Copy ${search_type}/db: OK${NC}${EOL}"
-      else
-        printf "${Red}${INDENT}${INDENT}Copy ${search_type}/db: FAIL${NC}${EOL}"
-      fi
-
-    fi
-
-  }
-
-  for type in $search_types; do
-    copy_database_files $type
-  done
-  # END copy database files END
-
-  # cleanup tmp directory
-  printf "${SPACER}${Cyan}${INDENT}Cleaning up temporary directory${NC}${EOL}"
-  rm -rf ${APP_ROOT}/django/tmp
-  if [[ $? -eq 0 ]]; then
-    printf "${Green}${INDENT}Cleanup django/tmp: OK${NC}${EOL}"
+  # clone oc_searches repo from https://github.com/thriuin/oc_searches
+  printf "${SPACER}${Cyan}${INDENT}Pulling search files from https://github.com/thriuin/oc_searches${NC}${EOL}"
+  if [[ ! -d "${APP_ROOT}/django/_searches" ]]; then
+    mkdir -p ${APP_ROOT}/django/_searches
+  fi
+  if [[ ! -d "${APP_ROOT}/django/_searches/.git" ]]; then
+    git clone https://github.com/thriuin/oc_searches.git ${APP_ROOT}/django/_searches
   else
-    printf "${Red}${INDENT}Cleanup django/tmp: FAIL${NC}${EOL}"
+    cd ${APP_ROOT}/django/_searches
+    git fetch
+    git pull
   fi
+  cd ${APP_ROOT}
+  searches_import_dir="${APP_ROOT}/django/_searches"
+
+  search_types="adminaircraft briefingt contracts data dataprops experiment grants nap5 qpnotes travela travelq"
 
   # drop database and re-create
   printf "${SPACER}${Cyan}${INDENT}Remaking fresh database...${NC}${EOL}"
@@ -221,12 +72,12 @@ EOSQL
   python manage.py migrate
   cd ${APP_ROOT}
 
-  # create a super user
-  DJANGO_SUPERUSER_USERNAME='admin_local'
-  DJANGO_SUPERUSER_PASSWORD='12345678'
-  DJANGO_SUPERUSER_EMAIL='[email protected]'
+  # run import_search
   cd ${APP_ROOT}/django/src/oc-search
-  python manage.py createsuperuser --noinput
+  for type in $search_types; do
+    python manage.py import_search --import_dir ${searches_import_dir} --search ${type} --include_db
+  done
+  cd ${APP_ROOT}
 
   # decativate python environment
   deactivate

docker/install/django/install-static.sh

@@ -15,11 +15,57 @@ if [[ $installFiles_Django == "true" ]]; then
     printf "${SPACER}${Cyan}${INDENT}Creating static directory...${NC}${SPACER}"
     mkdir -p ${APP_ROOT}/django/static
 
-    # download wet-boew/themes-cdn
-    printf "${SPACER}${Cyan}${INDENT}Download the wet-boew theme${NC}${SPACER}"
-    mkdir -p ${APP_ROOT}/django/static/themes-dist-GCWeb
-    cd ${APP_ROOT}/django/static/themes-dist-GCWeb
-    curl -L https://github.com/wet-boew/themes-cdn/archive/${GCWEB_VERSION}-gcweb.tar.gz | tar -zvx --strip-components 1 --directory=${APP_ROOT}/django/static/themes-dist-GCWeb
+    # activate python environment
+    . ${APP_ROOT}/django/bin/activate
+    if [[ $? -eq 0 ]]; then
+        printf "${Green}${INDENT}${INDENT}Activate Python environment: OK${NC}${EOL}"
+    else
+        printf "${Red}${INDENT}${INDENT}Activate Python environment: FAIL${NC}${EOL}"
+    fi
+
+    # make temp cdts dir
+    mkdir -p ${APP_ROOT}/django/static/cdts
+
+    # gather static files
+    cd ${APP_ROOT}/django/src/oc-search
+    python manage.py collectstatic --no-input
+    cd ${APP_ROOT}
+
+    # decativate python environment
+    deactivate
+    if [[ $? -eq 0 ]]; then
+        printf "${Green}${INDENT}${INDENT}Deactivate Python environment: OK${NC}${EOL}"
+    else
+        printf "${Red}${INDENT}${INDENT}Deactivate Python environment: FAIL${NC}${EOL}"
+    fi
+
+    ramp_static_dir="${APP_ROOT}/django/static/ramp"
+
+    # install fgp to ramp static
+    printf "${SPACER}${Cyan}${INDENT}Download the FGPV distro${NC}${SPACER}"
+    cd ${ramp_static_dir}
+    wget https://github.com/fgpv-vpgf/fgpv-vpgf/releases/download/v${FGP_VERSION}/fgpv-${FGP_VERSION}.zip
+    unzip fgpv-${FGP_VERSION}.zip
+    rm -f fgpv-${FGP_VERSION}.zip
+    cd ${APP_ROOT}
+
+    # download the wet GCWeb distro into ramp viewer
+    printf "${SPACER}${Cyan}${INDENT}Download the GCWeb distro${NC}${SPACER}"
+    mkdir -p ${ramp_static_dir}/GCWeb
+    cd ${ramp_static_dir}/GCWeb
+    wget https://github.com/wet-boew/GCWeb/archive/v${GCWEB_DISTRO_VERSION}.zip
+    unzip v${GCWEB_DISTRO_VERSION}.zip
+    rm -f v${GCWEB_DISTRO_VERSION}.zip
+    cd ${APP_ROOT}
+
+    # download wet-boew distro into ramp viewer
+    printf "${SPACER}${Cyan}${INDENT}Download the wet-boew distro${NC}${SPACER}"
+    mkdir -p ${ramp_static_dir}/wet-boew
+    cd ${ramp_static_dir}/wet-boew
+    wget https://github.com/wet-boew/wet-boew/releases/download/v${WET_DISTRO_VERSION}/wet-boew-dist-${WET_DISTRO_VERSION}.zip
+    unzip wet-boew-dist-${WET_DISTRO_VERSION}.zip
+    rm -f wet-boew-dist-${WET_DISTRO_VERSION}.zip
+    cd ${APP_ROOT}
 
     # pull the cenw-wscoe/sgdc-cdts repo
     printf "${SPACER}${Cyan}${INDENT}Pull the CDTS repo${NC}${SPACER}"
@@ -29,13 +75,14 @@ if [[ $installFiles_Django == "true" ]]; then
     rm -rf ./.??*
     git clone https://github.com/cenw-wscoe/sgdc-cdts.git .
 
-    # unpack the GCWeb Static release
+    # unpack the cdts static release
     printf "${SPACER}${Cyan}${INDENT}Unpack the GCWeb Static release from the CDTS repo${NC}${SPACER}"
     unzip ${APP_ROOT}/django/static/cdts/source/GCWeb/static/Gcwebstatic-${CDTS_GCWEB_VERSION}.zip -d ${APP_ROOT}/django/static/cdts
     cd ${APP_ROOT}/django/static/cdts/static
     mv * ../
     cd ${APP_ROOT}/django/static/cdts
     rm -rf ${APP_ROOT}/django/static/cdts/static
+    cd ${APP_ROOT}
 
     # set ownership
     cd ${APP_ROOT}/django

docker/install/django/install-user.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+#
+# Create a super user account
+# cannot fully automate this due to Django bug
+# with --no-input does not make a password.
+#
+if [[ $installUser_Django == "true" ]]; then
+
+  printf "${Cyan}${INDENT}Creating a super user. Prompting for user input...${NC}${EOL}"
+
+  # activate python environment
+  . ${APP_ROOT}/django/bin/activate
+  if [[ $? -eq 0 ]]; then
+      printf "${Green}${INDENT}Activate Python environment: OK${NC}${EOL}"
+  else
+      printf "${Red}${INDENT}Activate Python environment: FAIL${NC}${EOL}"
+  fi
+
+  # create a super user
+  cd ${APP_ROOT}/django/src/oc-search
+  python manage.py createsuperuser
+  cd ${APP_ROOT}
+
+  # decativate python environment
+  deactivate
+  if [[ $? -eq 0 ]]; then
+      printf "${Green}${INDENT}Deactivate Python environment: OK${NC}${EOL}"
+  else
+      printf "${Red}${INDENT}Deactivate Python environment: FAIL${NC}${EOL}"
+  fi
+
+fi
+# END
+# Create a super user account
+# END