Merge pull request #924 from bcgov/test-rook

9.5.10.1 changes to dev

.github/workflows/pagecountcalculator-cd.yml

@@ -0,0 +1,113 @@
+name: PageCountCalculator CD
+
+
+on:
+  push:
+    branches:
+      - dev
+      - main
+      - dev-marshal
+      - test-marshal
+      - dev-rook
+      - test-rook
+    paths:
+      - "computingservices/PageCountCalculator/**"
+      - ".github/workflows/pagecountcalculator-cd.yml"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./computingservices/PageCountCalculator
+
+env:
+  APP_NAME: "reviewer-pagecountcalculator"
+  TOOLS_NAME: "${{secrets.OPENSHIFT4_REPOSITORY}}"
+
+jobs:
+  pagecountcalculator-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'push' && github.repository == 'bcgov/foi-docreviewer'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables for dev branch
+        if: ${{ github.ref_name == 'dev' }}
+        shell: bash
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+
+      - name: Set ENV variables for main branch
+        if: ${{ github.ref_name == 'main' }}
+        shell: bash
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test" >> $GITHUB_ENV
+          echo "BRANCH_NAME=main" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+
+      - name: Set ENV variables for dev-marshal branch
+        if: ${{ github.ref_name == 'dev-marshal' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev-marshal" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev-marshal" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+
+      - name: Set ENV variables for test-marshal branch
+        if: ${{ github.ref_name == 'test-marshal' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test-marshal" >> $GITHUB_ENV
+          echo "BRANCH_NAME=test-marshal" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+      
+      - name: Set ENV variables for dev-rook branch
+        if: ${{ github.ref_name == 'dev-rook' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev-rook" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev-rook" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+          echo "ENV_TAG_NAME=rook" >> $GITHUB_ENV
+
+      - name: Set ENV variables for test-rook branch
+        if: ${{ github.ref_name == 'test-rook' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test-rook" >> $GITHUB_ENV
+          echo "BRANCH_NAME=test-rook" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+          echo "ENV_TAG_NAME=rook" >> $GITHUB_ENV
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ env.TOOLS_NAME }}-tools
+
+      - name: Build from ${{ env.BRANCH_NAME }} branch
+        shell: bash
+        run: |
+          oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"contextDir":"/computingservices/PageCountCalculator","git":{"ref":"${{ env.BRANCH_NAME }}"}}}}'
+
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-build --wait
+
+      - name: Tag+Deploy for ${{ env.TAG_NAME }}
+        shell: bash
+        run: |
+          oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }}
+
+      # - name: Watch new rollout (trigger by image change in Openshift)
+      #   shell: bash
+      #   run: |
+      #     oc rollout status dc/${{ env.APP_NAME }}-{{ env.ENV_TAG_NAME }} -n ${{ env.TOOLS_NAME }}-${{ env.ENV_NAME }} -w

.github/workflows/pagecountcalculator-ci.yml

@@ -0,0 +1,54 @@
+name: PageCountCalculator CI
+
+
+on:
+  pull_request:
+    branches:
+      - main
+      - dev
+      - dev-marshal
+      - test-marshal
+      - dev-rook
+      - test-rook
+    paths:
+      - "computingservices/PageCountCalculator/**"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./computingservices/PageCountCalculator
+
+jobs:
+  docker-build-check:
+    runs-on: ubuntu-20.04
+    name: Build dockerfile to ensure it works
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: docker build to check strictness
+        id: docker-build
+        run: |
+          docker build -f Dockerfile.local .
+
+  python-build-check:
+    runs-on: ubuntu-20.04
+    name: Build python to ensure it works
+
+    strategy:
+      matrix:
+        # python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+     

.gitignore

@@ -26,6 +26,7 @@ computingservices/poc/env/*
 computingservices/DedupeServices/env/*
 computingservices/PDFStitchServices/env/*
 computingservices/ZippingServices/env/*
+computingservices/PageCountCalculator/env/*
 
 
 ################################################################################

api/migrations/versions/18a45d1b33cc_DocumentDeletedPages.py

@@ -0,0 +1,40 @@
+"""empty message
+
+Revision ID: 18a45d1b33cc
+Revises: 7db1cd6f0b36
+Create Date: 2024-03-07 10:00:01.341943
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '18a45d1b33cc'
+down_revision = 'c787e6d82903'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('DocumentDeletedPages',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('ministryrequestid', sa.Integer(), nullable=False),
+    sa.Column('redactionlayerid', sa.Integer(), nullable=False),
+    sa.Column('documentid', sa.Integer(), nullable=False),
+    sa.Column('pagemetadata',  postgresql.JSON(astext_type=sa.Text()), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('createdby', postgresql.JSON(astext_type=sa.Text()), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    )
+    op.add_column('Documents', sa.Column('originalpagecount', sa.Integer, unique=False, nullable=True, default=False))
+    op.execute('UPDATE "Documents"  SET originalpagecount  = pagecount, updatedby= \'{"user":"System"}\', updated_at=now();commit;')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('DocumentDeletedPages')
+    op.drop_column('Documents', 'originalpagecount')
+    # ### end Alembic commands ###

api/migrations/versions/c787e6d82903_PageCalculatorJob.py

@@ -0,0 +1,36 @@
+"""empty message
+
+Revision ID: c787e6d82903
+Revises: e0e3a10b850d
+Create Date: 2024-02-06 15:12:03.310271
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+# revision identifiers, used by Alembic.
+revision = 'c787e6d82903'
+down_revision = 'e0e3a10b850d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('PageCalculatorJob',
+        sa.Column('pagecalculatorjobid', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+        sa.Column('version', sa.Integer(), nullable=False),
+        sa.Column('ministryrequestid', sa.Integer(), nullable=False),
+        sa.Column('inputmessage', postgresql.JSON(astext_type=sa.Text()), nullable=False), 
+        sa.Column('pagecount', postgresql.JSON(astext_type=sa.Text()), nullable=True), 
+        sa.Column('status', sa.String(length=120), nullable=False), 
+        sa.Column('message', sa.Text, nullable=True),
+        sa.Column('createdat', sa.TIMESTAMP, nullable=False, server_default=sa.func.now()),
+        sa.Column('createdby', sa.String(length=120), nullable=True), 
+        sa.PrimaryKeyConstraint('pagecalculatorjobid', 'version')
+    )
+
+
+def downgrade():
+    op.drop_table('PageCalculatorJob')

api/reviewer_api/models/DocumentDeletedPages.py

@@ -0,0 +1,49 @@
+from .db import db, ma
+from datetime import datetime as datetime2
+from sqlalchemy.dialects.postgresql import JSON, insert
+from sqlalchemy import or_, and_, text
+from .default_method_result import DefaultMethodResult
+from reviewer_api.models.Documents import Document
+import logging
+
+class DocumentDeletedPage(db.Model):
+    __tablename__ = 'DocumentDeletedPages'
+    # Defining the columns
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    redactionlayerid = db.Column(db.Integer, primary_key=True, nullable=False)
+    ministryrequestid = db.Column(db.Integer, nullable=False)  
+    documentid = db.Column(db.Integer, nullable=False)      
+    pagemetadata = db.Column(JSON, nullable=False)
+    created_at = db.Column(db.DateTime, default=datetime2.now)
+    createdby = db.Column(JSON, unique=False, nullable=False)
+
+
+    @classmethod
+    def create(cls, ministryrequestid, docpages, pagemappings) -> DefaultMethodResult:
+        try:
+            insertstmt = insert(DocumentDeletedPage).values(docpages)
+            db.session.execute(insertstmt)       
+            db.session.bulk_update_mappings(Document, pagemappings)     
+            db.session.commit()
+            return DefaultMethodResult(True, "Deleted page details saved", ministryrequestid)
+        except Exception as ex:
+            logging.error(ex)
+            return DefaultMethodResult(False, "Deleted page details persist operation failed", ministryrequestid)
+        finally:
+            db.session.close()
+
+    @classmethod
+    def getdeletedpages(cls, ministryrequestid, docids):
+        try:
+            deletepage_schema = DocumentDeletedSchema(many=True)
+            query = db.session.query(DocumentDeletedPage).filter(DocumentDeletedPage.ministryrequestid == ministryrequestid, DocumentDeletedPage.documentid.in_(docids)).all()
+            return deletepage_schema.dump(query)
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
+
+class DocumentDeletedSchema(ma.Schema):
+    class Meta:
+        fields = ('id', 'version', 'redactionlayerid', 'ministryrequestid', 'documentid','pagemetadata', 'created_at', 'createdby')

api/reviewer_api/models/DocumentMaster.py

@@ -208,15 +208,15 @@ def getdocumentproperty(cls, ministryrequestid, deleted):
         documentmasters = []
         try:
             sql = """select dm.documentmasterid,  dm.processingparentid, d.documentid, d.version,
-                        dhc.rank1hash, d.filename, d.pagecount, dm.parentid from "DocumentMaster" dm, 
+                        dhc.rank1hash, d.filename, d.originalpagecount, d.pagecount, dm.parentid from "DocumentMaster" dm, 
                         "Documents" d, "DocumentHashCodes" dhc  
                         where dm.ministryrequestid = :ministryrequestid and dm.ministryrequestid  = d.foiministryrequestid   
                         and dm.documentmasterid = d.documentmasterid 
                         and d.documentid = dhc.documentid order by dm.documentmasterid;"""
             rs = db.session.execute(text(sql), {'ministryrequestid': ministryrequestid})
             for row in rs:
                 if (row["processingparentid"] is not None and row["processingparentid"] not in deleted) or (row["processingparentid"] is None and row["documentmasterid"] not in deleted):
-                    documentmasters.append({"documentmasterid": row["documentmasterid"], "processingparentid": row["processingparentid"], "documentid": row["documentid"], "rank1hash": row["rank1hash"], "filename": row["filename"], "pagecount": row["pagecount"], "parentid": row["parentid"], "version": row["version"]})
+                    documentmasters.append({"documentmasterid": row["documentmasterid"], "processingparentid": row["processingparentid"], "documentid": row["documentid"], "rank1hash": row["rank1hash"], "filename": row["filename"], "originalpagecount": row["originalpagecount"],"pagecount": row["pagecount"], "parentid": row["parentid"], "version": row["version"]})
         except Exception as ex:
             logging.error(ex)
             db.session.close()

api/reviewer_api/models/DocumentPageflags.py

@@ -212,7 +212,44 @@ def getpageflag(
             db.session.close()
 
     @classmethod
-    def getpageflag_by_request(cls, _foiministryrequestid, redactionlayerid, documentids):
+    def getpageflag_by_request(cls, _foiministryrequestid, redactionlayerid):
+        pageflags = []
+        try:
+            sql = """select distinct on (dp.documentid) dp.documentid, dp.documentversion, dp.pageflag
+                     from "DocumentPageflags" dp
+                     join "Documents" d on dp.documentid = d.documentid and d.foiministryrequestid = :foiministryrequestid
+                     --join "DocumentMaster" dm on dm.documentmasterid = d.documentmasterid and dm.ministryrequestid = :foiministryrequestid
+                     --left join "DocumentDeleted" dd on dm.filepath ilike dd.filepath || '%' and dd.ministryrequestid = :foiministryrequestid
+                     where dp.foiministryrequestid = :foiministryrequestid --and (dd.deleted is false or dd.deleted is null)
+                     and redactionlayerid in :redactionlayerid
+                     order by dp.documentid, dp.documentversion desc, dp.id desc;
+                    """
+            rs = db.session.execute(
+                text(sql),
+                {
+                    "foiministryrequestid": _foiministryrequestid,
+                    "redactionlayerid": tuple(redactionlayerid),
+                },
+            )
+
+            for row in rs:
+                pageflags.append(
+                    {
+                        "documentid": row["documentid"],
+                        "documentversion": row["documentversion"],
+                        "pageflag": row["pageflag"],
+                    }
+                )
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return pageflags
+
+    @classmethod
+    def getpageflag_by_request_documentids(cls, _foiministryrequestid, redactionlayerid, documentids):
         pageflags = []
         try:
             sql = """select distinct on (dp.documentid) dp.documentid, dp.documentversion, dp.pageflag