Merge pull request #696 from QutEcoacoustics/pbs-analysis

Redesigned the analysis jobs system
QutEcoacoustics · Dec 9, 2024 · d3bb0f2 · d3bb0f2
2 parents fc5d35a + 9087251
commit d3bb0f2
Show file tree

Hide file tree

Showing 629 changed files with 34,348 additions and 20,918 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -13,35 +13,40 @@
   // The optional 'workspaceFolder' property is the path VS Code should open by default when
   // connected. This is typically a file mount in .devcontainer/docker-compose.yml
   "workspaceFolder": "/home/baw_web/baw-server",
-  // Use 'settings' to set *default* container specific settings.json values on container create.
-  // You can edit these settings after create using File > Preferences > Settings > Remote.
-  "settings": {
-    // This will ignore your local shell user setting for Linux since shells like zsh are typically
-    // not in base container images. You can also update this to an specific shell to ensure VS Code
-    // uses the right one for terminals and tasks. For example, /bin/bash (or /bin/ash for Alpine).
-    "terminal.integrated.defaultProfile.linux": "bash"
-  },
   // Uncomment the next line if you want start specific services in your Docker Compose config.
   // "runServices": [],
   // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
   // "shutdownAction": "none",
   // Uncomment the next line to run commands after the container is created - for example installing git.
   "postCreateCommand": "/home/baw_web/baw-server/provision/entrypoint.sh",
-  // Add the IDs of extensions you want installed when the container is created in the array below.
-  "extensions": [
-    "rebornix.ruby",
-    "misogi.ruby-rubocop",
-    "castwide.solargraph",
-    "bung87.vscode-gemfile",
-    "streetsidesoftware.code-spell-checker",
-    "EditorConfig.EditorConfig",
-    "ms-vscode-remote.vscode-remote-extensionpack",
-    "connorshea.vscode-ruby-test-adapter",
-    "knisterpeter.vscode-github",
-    "castwide.ruby-debug",
-    "pavlitsky.yard",
-    "stevejpurves.cucumber",
-    "ms-azuretools.vscode-docker",
-    "GitHub.vscode-pull-request-github"
-  ]
+  "customizations": {
+    // Use 'settings' to set *default* container specific settings.json values on container create.
+    // You can edit these settings after create using File > Preferences > Settings > Remote.
+    "settings": {
+      // This will ignore your local shell user setting for Linux since shells like zsh are typically
+      // not in base container images. You can also update this to an specific shell to ensure VS Code
+      // uses the right one for terminals and tasks. For example, /bin/bash (or /bin/ash for Alpine).
+      "terminal.integrated.defaultProfile.linux": "bash",
+      // https://github.com/microsoft/vscode-remote-release/issues/9194
+      "dev.containers.forwardWSLServices": false
+    },
+    "vscode": {
+      // Add the IDs of extensions you want installed when the container is created in the array below.
+      "extensions": [
+        "Shopify.ruby-lsp",
+        "misogi.ruby-rubocop",
+        "castwide.solargraph",
+        "bung87.vscode-gemfile",
+        "streetsidesoftware.code-spell-checker",
+        "EditorConfig.EditorConfig",
+        "ms-vscode-remote.vscode-remote-extensionpack",
+        "ms-vscode.live-server",
+        "connorshea.vscode-ruby-test-adapter",
+        "pavlitsky.yard",
+        "ms-azuretools.vscode-docker",
+        "GitHub.vscode-pull-request-github",
+        "eamodio.gitlens"
+      ]
+    }
+  }
 }
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -1,4 +1,5 @@
---- #-------------------------------------------------------------------------------------------------------------
+---
+#-------------------------------------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
@@ -21,15 +22,16 @@ services:
     # ports:
     #   - 3000:3000
     #group_add:
-      # - docker
-      # the docker group id on my host is 1001
-      # - 1001
+    # - docker
+    # the docker group id on my host is 1001
+    # - 1001
     volumes:
       # Update this to wherever you want VS Code to mount the folder of your project
       #- .:/home/baw_web/baw-server
       # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-in-docker-compose for details.
       # - /var/run/docker.sock:/var/run/docker.sock
       - baw-server-vscode-extensions:/root/.vscode-server/extensions
+      - baw-server-vscode-extensions:/home/baw-web/.vscode-server/extensions
       - baw-server-vscode-bash-history:/commandhistory
 
     # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.

diff --git a/.rubocop.yml b/.rubocop.yml
@@ -1,33 +1,50 @@
 ---
 require:
   - rubocop-rspec
+  - rubocop-rails
+  - rubocop-factory_bot
 AllCops:
   DisplayCopNames: true
   DisplayStyleGuide: true
   Include:
     - "**/Rakefile"
     - "**/config.ru"
     - "**/*.rb"
+  Exclude:
+    - "bin/**/*"
   TargetRubyVersion: 3.1
   NewCops: enable
 # Bug in rubocop-rspec config merge: https://github.com/rubocop/rubocop-rspec/pull/1163
+# https://github.com/rubocop/rubocop-rspec/blob/master/config/default.yml
 RSpec:
   Language:
     ExampleGroups:
       Regular:
         # rswag example group aliases
         - path
-        - get
-        - push
-        - post
-        - delete
+        # our multistep test module
         - stepwise
+        # our permissions helpers
+        - ensures
+        - the_users
+        - the_user
     Examples:
       Regular:
         # custom dsl for permissions tests
         - the_users
         - the_user
+        - ensures
+        # our multistep test module
         - step
+        # rswag 'examples'
+        - get
+        - push
+        - post
+        - delete
+    Expectations:
+      # rswag test helper that runs expect
+      - run_test!
+      - run_test
 
 # Customized options
 Lint/DeprecatedOpenSSLConstant:
@@ -40,6 +57,8 @@ Lint/StructNewOverride:
   Enabled: true
 Layout/LineLength:
   Max: 120
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
 Layout/EmptyLinesAroundAttributeAccessor:
   Enabled: true
 Layout/EndOfLine:
@@ -82,6 +101,8 @@ Metrics/BlockLength:
     - Async
     - aasm
     - stepwise
+    - included
+    - class_eval
 Metrics/MethodLength:
   Max: 30
   AllowedMethods:
@@ -90,6 +111,10 @@ Metrics/ModuleLength:
   Max: 150
 Metrics/AbcSize:
   Max: 60
+Rails/Output:
+  Exclude:
+    # the rails logger isn't available when patches are running
+    - '**/patches/**'
 # https://github.com/rubocop/rubocop-rspec/issues/795
 RSpec/DescribedClass:
   EnforcedStyle: explicit
@@ -107,7 +132,7 @@ RSpec/NestedGroups:
 RSpec/MultipleMemoizedHelpers:
   Max: 10
 RSpec/VariableName:
-  IgnoredPatterns:
+  AllowedPatterns:
     # Variable named after http header required by rswag
     - ^Authorization$
 RSpec/NamedSubject:

diff --git a/.solargraph.yml b/.solargraph.yml
@@ -6,7 +6,9 @@ exclude:
 # - test/**/*
 - vendor/**/*
 - ".bundle/**/*"
-require: []
+require:
+  - semantic_logger
+  - redis
 domains: []
 reporters:
   - rubocop

diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
-# syntax=docker/dockerfile:1.3
+# syntax=docker/dockerfile:1.4
 # Debian releases:
 #
-FROM ruby:3.1.1-slim-bullseye
+FROM ruby:3.2.2-slim-bullseye
 ARG app_name=baw-server
 ARG app_user=baw_web
 ARG version=
@@ -39,8 +39,10 @@ RUN --mount=type=bind,source=./provision,target=/provision \
   && mkdir -p "$GEM_HOME/bin" \
   && chmod 777 "$GEM_HOME/bin" \
   # https://github.com/moby/moby/issues/20437
-  && mkdir -p /home/${app_user}/${app_name}/tmp \
+  && mkdir -p /home/${app_user}/${app_name} \
+  && chmod g+srx /home/${app_user}/${app_name} \
   && chown -R 1000:1000 /home/${app_user} \
+  && mkdir -p /home/${app_user}/${app_name}/tmp \
   && mkdir /data \
   && chown -R 1000:1000 /data \
   && (if [ "x${trimmed}" != "xtrue" ]; then /provision/dev_setup.sh ; fi)
@@ -59,11 +61,10 @@ ENV RAILS_ENV=production \
   # generate assets for rails app
   # must be done in context (i.e. in production for production, not in dev for production)
   # should not be done for workers and in dev/test environments
-  GENERATE_ASSETS=false
+  GENERATE_ASSETS=false \
+  BINDING=0.0.0.0
 
 
-USER ${app_user}
-
 # "Install" our metadata utility
 COPY --from=qutecoacoustics/emu:7.0.3 --chown==${app_user} /emu /emu
 
@@ -75,15 +76,31 @@ COPY --chown=${app_user} Gemfile Gemfile.lock  /home/${app_user}/${app_name}/
 
 # install deps
 # skip installing gem documentation
-RUN (([ "x${trimmed}" != "xtrue" ] && echo 'gem: --no-rdoc --no-ri' >> "$HOME/.gemrc") || true) \
-  && (([ "x${trimmed}" = "xtrue" ] && bundle config set without development test) || true)
-# ensure required bundler version is installed
-# https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
-RUN gem install bundler -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1)" \
-  # install baw-server
-  && bundle install \
-  # install docs for dev work
-  && (([ "x${trimmed}" != "xtrue" ] && solargraph download-core && solargraph bundle) || true)
+  # run this as the app user
+RUN <<EOF
+  su ${app_user} << "EOF2"
+    echo "Running as: $(whoami) || $(id)"
+    set -eux
+
+    # last known version that worked
+    # gem version 3.5.21
+    # bundler version 2.5.21
+    gem update --system 3.5.21
+
+    # skip installing gem documentation
+    if [ "x${trimmed}" = "xtrue" ]; then
+      echo 'gem: --no-document' >> "$HOME/.gemrc"
+      bundle config set without development test
+    fi
+
+    # ensure required bundler version is installed
+    # https://bundler.io/blog/2019/05/14/solutions-for-cant-find-gem-bundler-with-executable-bundle.html
+    gem install bundler -v "$(grep -A 1 'BUNDLED WITH' Gemfile.lock | tail -n 1)"
+
+    # install baw-server
+    bundle install
+EOF2
+EOF
 
 # Add the Rails app
 COPY --chown=${app_user} ./ /home/${app_user}/${app_name}
@@ -95,12 +112,26 @@ COPY --chown=${app_user} ./ /home/${app_user}/${app_name}
 # Thus there is no conflict here
 COPY --chown=${app_user}:${app_user} ./provision/Passengerfile.production.json /home/${app_user}/${app_name}/Passengerfile.json
 
-# asign permissions to special things
-RUN  chmod a+x ./provision/*.sh \
-  && chmod a+x ./bin/* \
+# assign permissions to special things
+RUN <<EOF bash
+  set -ex
+
+  # allow group to read
+  chmod -R g+r .
+  find . -type d -exec chmod g+srx {} \;
+
+  # and write for assets compilation
+  chmod -R g+ws ./public
+
+  # ensure execute permissions for scripts
+  chmod a+x ./provision/*.sh
+  chmod a+x ./bin/*
+
   # https://github.com/moby/moby/issues/20437
-  && chmod 1777 ./tmp
+  chmod 1777 ./tmp
+EOF
 
+USER ${app_user}
 
 # precompile passenger standalone
 RUN bundle exec passenger start --runtime-check-only