diff --git a/docker-compose.yml b/docker-compose.yml index ba9109f..cc86da9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -356,16 +356,17 @@ services: - ./solr/core_travelq:/var/solr/data/core_travelq - ./solr/core_wrongdoing:/var/solr/data/core_wrongdoing # Persisting Django Solr Cores - - ./solr/core_od_search:/var/solr/data/core_od_search - - ./solr/core_bn_search:/var/solr/data/core_bn_search - - ./solr/core_ati_search:/var/solr/data/core_ati_search - - ./solr/core_ei_search:/var/solr/data/core_ei_search - - ./solr/core_ct_search:/var/solr/data/core_ct_search - - ./solr/core_gc_search:/var/solr/data/core_gc_search - - ./solr/core_ap_search:/var/solr/data/core_ap_search - - ./solr/core_sv_search:/var/solr/data/core_sv_search - - ./solr/core_qp_search:/var/solr/data/core_qp_search - - ./solr/core_sd_search:/var/solr/data/core_sd_search + - ./solr/search_adminaircraft:/var/solr/data/search_adminaircraft + - ./solr/search_briefingt:/var/solr/data/search_briefingt + - ./solr/search_contracts:/var/solr/data/search_contracts + - ./solr/search_ei:/var/solr/data/search_ei + - ./solr/search_grants:/var/solr/data/search_grants + - ./solr/search_nap5:/var/solr/data/search_nap5 + - ./solr/search_opencanada:/var/solr/data/search_opencanada + - ./solr/search_qpnotes:/var/solr/data/search_qpnotes + - ./solr/search_suggesteddatasets:/var/solr/data/search_suggesteddatasets + - ./solr/search_travela:/var/solr/data/search_travela + - ./solr/search_travelq:/var/solr/data/search_travelq # Persisting Drupal Solr Cores - ./solr/drupal_content:/var/solr/data/drupal_content # Persisting CKAN Test Solr Cores @@ -381,16 +382,17 @@ services: - ./solr/test__core_travelq:/var/solr/data/test__core_travelq - ./solr/test__core_wrongdoing:/var/solr/data/test__core_wrongdoing # Persisting Django Test Solr Cores - - ./solr/test__core_od_search:/var/solr/data/test__core_od_search - - ./solr/test__core_bn_search:/var/solr/data/test__core_bn_search - - ./solr/test__core_ati_search:/var/solr/data/test__core_ati_search - - ./solr/test__core_ei_search:/var/solr/data/test__core_ei_search - - ./solr/test__core_ct_search:/var/solr/data/test__core_ct_search - - ./solr/test__core_gc_search:/var/solr/data/test__core_gc_search - - ./solr/test__core_ap_search:/var/solr/data/test__core_ap_search - - ./solr/test__core_sv_search:/var/solr/data/test__core_sv_search - - ./solr/test__core_qp_search:/var/solr/data/test__core_qp_search - - ./solr/test__core_sd_search:/var/solr/data/test__core_sd_search + - ./solr/test__search_adminaircraft:/var/solr/data/test__search_adminaircraft + - ./solr/test__search_briefingt:/var/solr/data/test__search_briefingt + - ./solr/test__search_contracts:/var/solr/data/test__search_contracts + - ./solr/test__search_ei:/var/solr/data/test__search_ei + - ./solr/test__search_grants:/var/solr/data/test__search_grants + - ./solr/test__search_nap5:/var/solr/data/test__search_nap5 + - ./solr/test__search_opencanada:/var/solr/data/test__search_opencanada + - ./solr/test__search_qpnotes:/var/solr/data/test__search_qpnotes + - ./solr/test__search_suggesteddatasets:/var/solr/data/test__search_suggesteddatasets + - ./solr/test__search_travela:/var/solr/data/test__search_travela + - ./solr/test__search_travelq:/var/solr/data/test__search_travelq # Persisting Drupal Test Solr Cores - ./solr/test__drupal_content:/var/solr/data/test__drupal_content environment: @@ -496,10 +498,10 @@ services: PATH: '$PATH:/usr/local/lib' APP_ROOT: /var/ocs PY_VERSION: 3.7 - WET_VERSION: v4.0.63 - CDTS_VERSION: v4_0_28 + FGP_VERSION: 3.3.7 + GCWEB_DISTRO_VERSION: 13.4.0 + WET_DISTRO_VERSION: 4.0.64.1 CDTS_GCWEB_VERSION: v4_0_31 - GCWEB_VERSION: v13.3.0 PYTHONHTTPSVERIFY: 0 PYTHONDONTWRITEBYTECODE: 1 SSL_VERIFY: 0 diff --git a/docker/config/nginx/proxied_sites/search.open.local b/docker/config/nginx/proxied_sites/search.open.local index 890d86c..d884032 100644 --- a/docker/config/nginx/proxied_sites/search.open.local +++ b/docker/config/nginx/proxied_sites/search.open.local @@ -30,7 +30,7 @@ server { location /static/ { - alias /var/ogproxy/django/src/oc-search/oc_search/static/; + alias /var/ogproxy/django/static/; autoindex off; } diff --git a/docker/install/django/install-searches.sh b/docker/install/django/install-searches.sh index 1446004..8ef6aef 100644 --- a/docker/install/django/install-searches.sh +++ b/docker/install/django/install-searches.sh @@ -30,171 +30,22 @@ if [[ $installSearches_Django == "true" ]]; then wget https://raw.githubusercontent.com/open-data/ckanext-canada/master/ckanext/canada/tables/choices/minister.json wget https://raw.githubusercontent.com/open-data/ckanext-canada/master/ckanext/canada/schemas/prop.yaml - # download oc_searches repo from https://github.com/thriuin/oc_searches - printf "${SPACER}${Cyan}${INDENT}Downloading search files from https://github.com/thriuin/oc_searches to temporary directory${NC}${EOL}" - mkdir -p ${APP_ROOT}/django/tmp - cd ${APP_ROOT}/django/tmp - wget https://github.com/thriuin/oc_searches/archive/refs/heads/main.zip - unzip main.zip - tmp_dir="${APP_ROOT}/django/tmp/oc_searches-main" - - search_types="adminaircraft briefingt contracts data dataprops experiment grants nap5 qpnotes travela travelq" - - # copy plugin files - printf "${SPACER}${Cyan}${INDENT}Copying plugin files...${NC}${EOL}" - function copy_plugin_files() { - - search_type="$1" - printf "${Cyan}${INDENT}${INDENT}Copying plugin files for ${search_type}${NC}${EOL}" - - if [[ ! -f "${tmp_dir}/${search_type}/plugins/${search_type}.py" ]]; then - - printf "${Yellow}${INDENT}${INDENT}${search_type} has no plugin file: SKIPPING${NC}${EOL}" - - else - - cp ${tmp_dir}/${search_type}/plugins/${search_type}.py ${APP_ROOT}/django/src/oc-search/search/plugins/${search_type}.py - - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}${INDENT}Copy plugins/${search_type}.py: OK${NC}${EOL}" - else - printf "${Red}${INDENT}${INDENT}Copy plugins/${search_type}.py: FAIL${NC}${EOL}" - fi - - fi - - } - - for type in $search_types; do - copy_plugin_files $type - done - # END copy plugin files END - - # copy custom snippet files - printf "${SPACER}${Cyan}${INDENT}Copying snippet files...${NC}${EOL}" - function copy_snippet_files() { - - search_type="$1" - printf "${Cyan}${INDENT}${INDENT}Copying snippet files for ${search_type}${NC}${EOL}" - - if [[ ! -d "${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type}" ]]; then - - mkdir -p ${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type} - - fi - - cp ${tmp_dir}/${search_type}/snippets/* ${APP_ROOT}/django/src/oc-search/search/templates/snippets/custom/${search_type}/ - - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}${INDENT}Copy ${search_type}/snippets: OK${NC}${EOL}" - else - printf "${Red}${INDENT}${INDENT}Copy ${search_type}/snippets: FAIL${NC}${EOL}" - fi - - } - - for type in $search_types; do - copy_snippet_files $type - done - # END copy custom snippet files END - - # copy command files - printf "${SPACER}${Cyan}${INDENT}Copying command files...${NC}${EOL}" - function copy_command_files() { - - search_type="$1" - printf "${Cyan}${INDENT}${INDENT}Copying command files for ${search_type}${NC}${EOL}" - - if [[ ! -d "${tmp_dir}/${search_type}/commands" ]]; then - - printf "${Yellow}${INDENT}${INDENT}${search_type} has no command files: SKIPPING${NC}${EOL}" - - else - - cp ${tmp_dir}/${search_type}/commands/* ${APP_ROOT}/django/src/oc-search/search/management/commands/ - - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}${INDENT}Copy ${search_type}/commands: OK${NC}${EOL}" - else - printf "${Red}${INDENT}${INDENT}Copy ${search_type}/commands: FAIL${NC}${EOL}" - fi - - fi - - } - - for type in $search_types; do - copy_command_files $type - done - # END copy command files END - - # copy locale files - printf "${SPACER}${Cyan}${INDENT}Copying locale files...${NC}${EOL}" - function copy_locale_files() { - - search_type="$1" - printf "${Cyan}${INDENT}${INDENT}Copying command files for ${search_type}${NC}${EOL}" - - if [[ ! -f "${tmp_dir}/${search_type}/locale/${search_type}.po" ]]; then - - printf "${Yellow}${INDENT}${INDENT}${search_type} has no locale files: SKIPPING${NC}${EOL}" - - else - - cp ${tmp_dir}/${search_type}/locale/${search_type}.po ${APP_ROOT}/django/src/oc-search/locale/fr/LC_MESSAGES/${search_type}.po - - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}${INDENT}Copy locale/${search_type}.po: OK${NC}${EOL}" - else - printf "${Red}${INDENT}${INDENT}Copy locale/${search_type}.po: FAIL${NC}${EOL}" - fi - - fi - - } - - for type in $search_types; do - copy_locale_files $type - done - # END copy locale files END - - # copy database files - printf "${SPACER}${Cyan}${INDENT}Copying database files...${NC}${EOL}" - function copy_database_files() { - - search_type="$1" - printf "${Cyan}${INDENT}${INDENT}Copying database files for ${search_type}${NC}${EOL}" - - if [[ ! -d "${tmp_dir}/${search_type}/db" ]]; then - - printf "${Yellow}${INDENT}${INDENT}${search_type} has no database files: SKIPPING${NC}${EOL}" - - else - - cp ${tmp_dir}/${search_type}/db/* ${APP_ROOT}/django/src/oc-search/data/ - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}${INDENT}Copy ${search_type}/db: OK${NC}${EOL}" - else - printf "${Red}${INDENT}${INDENT}Copy ${search_type}/db: FAIL${NC}${EOL}" - fi - - fi - - } - - for type in $search_types; do - copy_database_files $type - done - # END copy database files END - - # cleanup tmp directory - printf "${SPACER}${Cyan}${INDENT}Cleaning up temporary directory${NC}${EOL}" - rm -rf ${APP_ROOT}/django/tmp - if [[ $? -eq 0 ]]; then - printf "${Green}${INDENT}Cleanup django/tmp: OK${NC}${EOL}" + # clone oc_searches repo from https://github.com/thriuin/oc_searches + printf "${SPACER}${Cyan}${INDENT}Pulling search files from https://github.com/thriuin/oc_searches${NC}${EOL}" + if [[ ! -d "${APP_ROOT}/django/_searches" ]]; then + mkdir -p ${APP_ROOT}/django/_searches + fi + if [[ ! -d "${APP_ROOT}/django/_searches/.git" ]]; then + git clone https://github.com/thriuin/oc_searches.git ${APP_ROOT}/django/_searches else - printf "${Red}${INDENT}Cleanup django/tmp: FAIL${NC}${EOL}" + cd ${APP_ROOT}/django/_searches + git fetch + git pull fi + cd ${APP_ROOT} + searches_import_dir="${APP_ROOT}/django/_searches" + + search_types="adminaircraft briefingt contracts data dataprops experiment grants nap5 qpnotes travela travelq" # drop database and re-create printf "${SPACER}${Cyan}${INDENT}Remaking fresh database...${NC}${EOL}" @@ -221,12 +72,12 @@ EOSQL python manage.py migrate cd ${APP_ROOT} - # create a super user - DJANGO_SUPERUSER_USERNAME='admin_local' - DJANGO_SUPERUSER_PASSWORD='12345678' - DJANGO_SUPERUSER_EMAIL='temp@tbs-sct.gc.ca' + # run import_search cd ${APP_ROOT}/django/src/oc-search - python manage.py createsuperuser --noinput + for type in $search_types; do + python manage.py import_search --import_dir ${searches_import_dir} --search ${type} --include_db + done + cd ${APP_ROOT} # decativate python environment deactivate diff --git a/docker/install/django/install-static.sh b/docker/install/django/install-static.sh index dd6ef71..faa6a52 100755 --- a/docker/install/django/install-static.sh +++ b/docker/install/django/install-static.sh @@ -15,11 +15,57 @@ if [[ $installFiles_Django == "true" ]]; then printf "${SPACER}${Cyan}${INDENT}Creating static directory...${NC}${SPACER}" mkdir -p ${APP_ROOT}/django/static - # download wet-boew/themes-cdn - printf "${SPACER}${Cyan}${INDENT}Download the wet-boew theme${NC}${SPACER}" - mkdir -p ${APP_ROOT}/django/static/themes-dist-GCWeb - cd ${APP_ROOT}/django/static/themes-dist-GCWeb - curl -L https://github.com/wet-boew/themes-cdn/archive/${GCWEB_VERSION}-gcweb.tar.gz | tar -zvx --strip-components 1 --directory=${APP_ROOT}/django/static/themes-dist-GCWeb + # activate python environment + . ${APP_ROOT}/django/bin/activate + if [[ $? -eq 0 ]]; then + printf "${Green}${INDENT}${INDENT}Activate Python environment: OK${NC}${EOL}" + else + printf "${Red}${INDENT}${INDENT}Activate Python environment: FAIL${NC}${EOL}" + fi + + # make temp cdts dir + mkdir -p ${APP_ROOT}/django/static/cdts + + # gather static files + cd ${APP_ROOT}/django/src/oc-search + python manage.py collectstatic --no-input + cd ${APP_ROOT} + + # decativate python environment + deactivate + if [[ $? -eq 0 ]]; then + printf "${Green}${INDENT}${INDENT}Deactivate Python environment: OK${NC}${EOL}" + else + printf "${Red}${INDENT}${INDENT}Deactivate Python environment: FAIL${NC}${EOL}" + fi + + ramp_static_dir="${APP_ROOT}/django/static/ramp" + + # install fgp to ramp static + printf "${SPACER}${Cyan}${INDENT}Download the FGPV distro${NC}${SPACER}" + cd ${ramp_static_dir} + wget https://github.com/fgpv-vpgf/fgpv-vpgf/releases/download/v${FGP_VERSION}/fgpv-${FGP_VERSION}.zip + unzip fgpv-${FGP_VERSION}.zip + rm -f fgpv-${FGP_VERSION}.zip + cd ${APP_ROOT} + + # download the wet GCWeb distro into ramp viewer + printf "${SPACER}${Cyan}${INDENT}Download the GCWeb distro${NC}${SPACER}" + mkdir -p ${ramp_static_dir}/GCWeb + cd ${ramp_static_dir}/GCWeb + wget https://github.com/wet-boew/GCWeb/archive/v${GCWEB_DISTRO_VERSION}.zip + unzip v${GCWEB_DISTRO_VERSION}.zip + rm -f v${GCWEB_DISTRO_VERSION}.zip + cd ${APP_ROOT} + + # download wet-boew distro into ramp viewer + printf "${SPACER}${Cyan}${INDENT}Download the wet-boew distro${NC}${SPACER}" + mkdir -p ${ramp_static_dir}/wet-boew + cd ${ramp_static_dir}/wet-boew + wget https://github.com/wet-boew/wet-boew/releases/download/v${WET_DISTRO_VERSION}/wet-boew-dist-${WET_DISTRO_VERSION}.zip + unzip wet-boew-dist-${WET_DISTRO_VERSION}.zip + rm -f wet-boew-dist-${WET_DISTRO_VERSION}.zip + cd ${APP_ROOT} # pull the cenw-wscoe/sgdc-cdts repo printf "${SPACER}${Cyan}${INDENT}Pull the CDTS repo${NC}${SPACER}" @@ -29,13 +75,14 @@ if [[ $installFiles_Django == "true" ]]; then rm -rf ./.??* git clone https://github.com/cenw-wscoe/sgdc-cdts.git . - # unpack the GCWeb Static release + # unpack the cdts static release printf "${SPACER}${Cyan}${INDENT}Unpack the GCWeb Static release from the CDTS repo${NC}${SPACER}" unzip ${APP_ROOT}/django/static/cdts/source/GCWeb/static/Gcwebstatic-${CDTS_GCWEB_VERSION}.zip -d ${APP_ROOT}/django/static/cdts cd ${APP_ROOT}/django/static/cdts/static mv * ../ cd ${APP_ROOT}/django/static/cdts rm -rf ${APP_ROOT}/django/static/cdts/static + cd ${APP_ROOT} # set ownership cd ${APP_ROOT}/django diff --git a/docker/install/django/install-user.sh b/docker/install/django/install-user.sh new file mode 100644 index 0000000..e26fb48 --- /dev/null +++ b/docker/install/django/install-user.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# +# Create a super user account +# cannot fully automate this due to Django bug +# with --no-input does not make a password. +# +if [[ $installUser_Django == "true" ]]; then + + printf "${Cyan}${INDENT}Creating a super user. Prompting for user input...${NC}${EOL}" + + # activate python environment + . ${APP_ROOT}/django/bin/activate + if [[ $? -eq 0 ]]; then + printf "${Green}${INDENT}Activate Python environment: OK${NC}${EOL}" + else + printf "${Red}${INDENT}Activate Python environment: FAIL${NC}${EOL}" + fi + + # create a super user + cd ${APP_ROOT}/django/src/oc-search + python manage.py createsuperuser + cd ${APP_ROOT} + + # decativate python environment + deactivate + if [[ $? -eq 0 ]]; then + printf "${Green}${INDENT}Deactivate Python environment: OK${NC}${EOL}" + else + printf "${Red}${INDENT}Deactivate Python environment: FAIL${NC}${EOL}" + fi + +fi +# END +# Create a super user account +# END diff --git a/docker/install/install-django.sh b/docker/install/install-django.sh index 53ef680..d0eb2c7 100755 --- a/docker/install/install-django.sh +++ b/docker/install/install-django.sh @@ -7,6 +7,7 @@ options=( "OC Django Search App (version 2)" "Static Files" "Searches" + "Create Admin User" "Set File Permissions" "All" "Exit" @@ -36,23 +37,30 @@ case $opt in installSearches_Django='true' ;; - # "Set File Permissions" + # "Create Admin User" (3) + exitScript='false' + installUser_Django='true' + ;; + + # "Set File Permissions" + (4) exitScript='false' installFilePermissions_Django='true' ;; # "All" - (4) + (5) exitScript='false' installApp_Django='true' installFiles_Django='true' installSearches_Django='true' + installUser_Django='true' installFilePermissions_Django='true' ;; # "Exit" - (5) + (6) exitScript='true' ;; @@ -72,6 +80,8 @@ if [[ $exitScript != "true" ]]; then cd ${APP_ROOT} . ${PWD}/docker/install/django/install-searches.sh cd ${APP_ROOT} + . ${PWD}/docker/install/django/install-user.sh + cd ${APP_ROOT} . ${PWD}/docker/install/django/install-perms.sh cd ${APP_ROOT} diff --git a/docker/start.sh b/docker/start.sh index cd8e811..65f8e54 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -167,7 +167,7 @@ elif [[ "$role" = "search" ]]; then cp ${APP_ROOT}/_config/django/settings.py ${APP_ROOT}/django/src/oc-search/oc_search/settings.py printf "${Green}Copying the setup file to the virtual environment${NC}${EOL}" - cp ${APP_ROOT}/docker/config/django/etup.py ${APP_ROOT}/django/src/oc-search/setup.py + cp ${APP_ROOT}/docker/config/django/setup.py ${APP_ROOT}/django/src/oc-search/setup.py chown django:django ${APP_ROOT}/django/src/oc-search/setup.py fi;