Bug fix #4704 and #4705

api/reviewer_api/models/Annotations.py

@@ -106,6 +106,9 @@ def get_request_annotations_pagination(
         cls, ministryrequestid, mappedlayerids, page, size
     ):
         _deleted = DocumentMaster.getdeleted(ministryrequestid)
+        _originalnodonversionfiles = DocumentMaster.filteroriginalnoconversionfiles(ministryrequestid)
+        _replacednoconversionfiles = DocumentMaster.filterreplacednoconversionfiles(ministryrequestid)
+        _replacedotherfiles = DocumentMaster.filterreplacedfiles(ministryrequestid)
         _session = db.session
         _subquery_annotation = (
             _session.query(
@@ -116,6 +119,8 @@ def get_request_annotations_pagination(
                 and_(
                     Annotation.documentid == Document.documentid,
                     Document.documentmasterid.notin_(_deleted),
+                    or_(Document.documentmasterid.in_(_replacedotherfiles), Document.documentmasterid.in_(_originalnodonversionfiles)),
+                    Document.documentmasterid.notin_(_replacednoconversionfiles),
                     Document.foiministryrequestid == ministryrequestid,
                 ),
             )
@@ -547,6 +552,33 @@ def bulkdeleteannotations(
         finally:
             db.session.close()
 
+    @classmethod
+    def deletedocumentannotations(
+        cls, documentids, userinfo
+    ) -> DefaultMethodResult:
+        try:
+            sql = """with annotationames as (update "Annotations" a set isactive = false, updatedby = :userinfo, updated_at=now()
+                    where a.documentid in :documentids
+                    and a.isactive = True returning annotationname)
+                    update public."AnnotationSections" as1 set isactive = false, updatedby = :userinfo, updated_at=now()
+                    where annotationname in (select * from annotationames)
+                    and as1.isactive = True"""
+            db.session.execute(
+                text(sql),
+                {
+                    "userinfo": json.dumps(userinfo),
+                    "documentids": tuple(documentids),
+                },
+            )
+            db.session.commit()
+            return DefaultMethodResult(
+                True, "Annotations for documentids" + ",".join(str(documentids)) + "are deleted", ",".join(str(documentids))
+            )
+        except Exception as ex:
+            logging.error(ex)
+        finally:
+            db.session.close()
+
     @classmethod
     def getredactionsbydocumentpages(cls, _documentid, _pages, redactionlayerid):
         try:

api/reviewer_api/models/DocumentMaster.py

@@ -85,8 +85,102 @@ def getdeleted(cls, ministryrequestid):
         finally:
             db.session.close()
         return documentmasters
+
+
+    @classmethod
+    def getprocessingchilddocumentids(cls, documentmasterids):
+        documentmasters = []
+        try:
+            sql = """select d.documentid
+                    from public."DocumentMaster" dm
+                    left join public."Documents" d on d.documentmasterid = dm.documentmasterid
+                    where processingparentid = :documentmasterids or dm.documentmasterid = :documentmasterids"""
+            rs = db.session.execute(text(sql), {'documentmasterids': documentmasterids})
+            for row in rs:
+                documentmasters.append(row["documentid"])
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return documentmasters
 
 
+    @classmethod 
+    def filterreplacednoconversionfiles(cls, ministryrequestid):
+        documentmasters = []
+        try:
+            # filter out replaced jpg, png & pdf files - files do not need conversion
+            sql = """select processingparentid
+						from "DocumentMaster"
+						where processingparentid is not Null and ministryrequestid =:ministryrequestid"""
+            rs = db.session.execute(text(sql), {'ministryrequestid': ministryrequestid})
+            for row in rs:
+                documentmasters.append(row["processingparentid"])
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return documentmasters
+
+    @classmethod 
+    def filterreplacedfiles(cls, ministryrequestid):
+        documentmasters = []
+        try:
+            # all original/replaced other type of files + all original/replaced (jpg, png & pdf) files
+            sql = """select MAX(documentmasterid) as documentmasterid
+						from public."DocumentMaster"
+						where processingparentid is not null and ministryrequestid =:ministryrequestid
+						group by processingparentid"""
+            rs = db.session.execute(text(sql), {'ministryrequestid': ministryrequestid})
+            for row in rs:
+                documentmasters.append(row["documentmasterid"])
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return documentmasters
+
+    @classmethod 
+    def filteroriginalnoconversionfiles(cls, ministryrequestid):
+        documentmasters = []
+        try:
+            # all original/replaced other type of files + all original/replaced (jpg, png & pdf) files
+            sql = """select documentmasterid
+						from public."DocumentMaster"
+						where processingparentid is null and ministryrequestid =:ministryrequestid"""
+            rs = db.session.execute(text(sql), {'ministryrequestid': ministryrequestid})
+            for row in rs:
+                documentmasters.append(row["documentmasterid"])
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return documentmasters
+
+
+    @classmethod
+    def getfilepathbydocumentid(cls, documentid):
+        try:
+            sql = """select dm.filepath
+                    from public."DocumentMaster" dm
+                    join public."Documents" d on d.documentmasterid = dm.documentmasterid
+                    where d.documentid = :documentid"""
+            rs = db.session.execute(text(sql), {'documentid': documentid}).first()
+        except Exception as ex:
+            logging.error(ex)
+            db.session.close()
+            raise ex
+        finally:
+            db.session.close()
+        return rs[0]
 
     @classmethod 
     def getredactionready(cls, ministryrequestid):

api/reviewer_api/models/DocumentPageflagHistory.py

@@ -30,12 +30,10 @@ class DocumentPageflagHistory(db.Model):
 
     @classmethod
     def createpageflag(cls, documentpageflaghistory)->DefaultMethodResult:
+        # no db close or commit because this function is called in a loop
         try:
             db.session.add(documentpageflaghistory)
-            db.session.commit()
             return DefaultMethodResult(True, 'Page Flag history is saved', documentpageflaghistory.id)  
         except Exception as ex:
             logging.error(ex)
-            return DefaultMethodResult(True, 'Page Flag history is not saved', documentpageflaghistory.id) 
-        finally:
-            db.session.close()
+            return DefaultMethodResult(True, 'Page Flag history is not saved', documentpageflaghistory.id)

api/reviewer_api/models/DocumentPageflags.py

@@ -133,6 +133,61 @@ def savepageflag(
         finally:
             db.session.close()
 
+    @classmethod
+    def bulkarchivepageflag(
+        cls,
+        _foiministryrequestid,
+        _documentids,
+        userinfo,
+    ) -> DefaultMethodResult:
+        try:
+            dbquery = db.session.query(DocumentPageflag)
+            pageflags = dbquery.filter(
+                and_(
+                    DocumentPageflag.foiministryrequestid == _foiministryrequestid,
+                    DocumentPageflag.documentid.in_(_documentids),
+                )
+            )
+            for pageflagobj in pageflags:
+                DocumentPageflagHistory.createpageflag(
+                    DocumentPageflagHistory(
+                        documentpageflagid=pageflagobj.id,
+                        foiministryrequestid=pageflagobj.foiministryrequestid,
+                        documentid=pageflagobj.documentid,
+                        documentversion=pageflagobj.documentversion,
+                        pageflag=json.dumps(pageflagobj.pageflag),
+                        attributes=json.dumps(pageflagobj.attributes),
+                        createdby=json.dumps(pageflagobj.createdby),
+                        created_at=pageflagobj.created_at,
+                        updatedby=json.dumps(pageflagobj.updatedby),
+                        updated_at=pageflagobj.updated_at,
+                        redactionlayerid=pageflagobj.redactionlayerid,
+                    )
+                )
+                DocumentPageflagHistory.createpageflag(
+                    DocumentPageflagHistory(
+                        documentpageflagid=pageflagobj.id,
+                        foiministryrequestid=pageflagobj.foiministryrequestid,
+                        documentid=pageflagobj.documentid,
+                        documentversion=pageflagobj.documentversion,
+                        pageflag=json.dumps(pageflagobj.pageflag),
+                        attributes=json.dumps(pageflagobj.attributes),
+                        createdby=json.dumps(pageflagobj.createdby),
+                        created_at=pageflagobj.created_at,
+                        updatedby=json.dumps(userinfo),
+                        updated_at=pageflagobj.updated_at,
+                        redactionlayerid=pageflagobj.redactionlayerid,
+                    )
+                )
+            pageflags.delete(synchronize_session='fetch')
+            db.session.commit()
+            return DefaultMethodResult(True, "Page Flag is saved", _documentids)
+        except Exception as ex:
+            logging.error(ex)
+            return DefaultMethodResult(True, "Page Flag is not saved", _documentids)
+        finally:
+            db.session.close()
+
     @classmethod
     def getpageflag(
         cls, _foiministryrequestid, _documentid, _documentversion, _redactionlayerid
@@ -162,23 +217,25 @@ def getpageflag(
             db.session.close()
 
     @classmethod
-    def getpageflag_by_request(cls, _foiministryrequestid, redactionlayerid):
+    def getpageflag_by_request(cls, _foiministryrequestid, redactionlayerid, documentids):
         pageflags = []
         try:
             sql = """select distinct on (dp.documentid) dp.documentid, dp.documentversion, dp.pageflag
                      from "DocumentPageflags" dp
                      join "Documents" d on dp.documentid = d.documentid and d.foiministryrequestid = :foiministryrequestid
-                     join "DocumentMaster" dm on dm.documentmasterid = d.documentmasterid and dm.ministryrequestid = :foiministryrequestid
-                     left join "DocumentDeleted" dd on dm.filepath ilike dd.filepath || '%' and dd.ministryrequestid = :foiministryrequestid
-                     where dp.foiministryrequestid = :foiministryrequestid and (dd.deleted is false or dd.deleted is null)
+                     --join "DocumentMaster" dm on dm.documentmasterid = d.documentmasterid and dm.ministryrequestid = :foiministryrequestid
+                     --left join "DocumentDeleted" dd on dm.filepath ilike dd.filepath || '%' and dd.ministryrequestid = :foiministryrequestid
+                     where dp.foiministryrequestid = :foiministryrequestid --and (dd.deleted is false or dd.deleted is null)
                      and redactionlayerid in :redactionlayerid
+                     and dp.documentid in :documentids
                      order by dp.documentid, dp.documentversion desc, dp.id desc;
                     """
             rs = db.session.execute(
                 text(sql),
                 {
                     "foiministryrequestid": _foiministryrequestid,
                     "redactionlayerid": tuple(redactionlayerid),
+                    "documentids": tuple(documentids),
                 },
             )
 

api/reviewer_api/resources/documentpageflag.py

@@ -75,7 +75,7 @@ def get(requestid, documentid, documentversion, redactionlayerid):
 
 
 @cors_preflight('GET,OPTIONS')
-@API.route('/ministryrequest/<requestid>/pageflag/<redactionlayerid>')
+@API.route('/ministryrequest/<requestid>/pageflag/<redactionlayerid>/<documentids>')
 class GetDocumentPageflag(Resource):
     """Get document page flag list.
     """
@@ -84,9 +84,10 @@ class GetDocumentPageflag(Resource):
     @cross_origin(origins=allowedorigins())
     @auth.require
     @auth.ismemberofgroups(getrequiredmemberships())
-    def get(requestid, redactionlayerid):
+    def get(requestid, redactionlayerid, documentids):
         try:
-            result = documentpageflagservice().getpageflags(requestid, redactionlayerid)
+            documentids = documentids.split(",")
+            result = documentpageflagservice().getpageflags(requestid, redactionlayerid, documentids)
             return json.dumps(result), 200
         except KeyError as error:
             return {'status': False, 'message': CUSTOM_KEYERROR_MESSAGE + str(error)}, 400

api/reviewer_api/resources/foiflowmasterdata.py

@@ -313,7 +313,9 @@ def post(ministryrequestid):
                             # for save/put - stitch by division
                         div["s3path_save"] = s3path_save
                     for doc in div["documentlist"]:
-                        filepathlist = doc["filepath"].split("/")[4:]
+                        realfilepath = documentservice().getfilepathbydocumentid(doc["documentid"])
+                        # filepathlist = doc["filepath"].split("/")[4:]
+                        filepathlist = realfilepath.split("/")[4:]
                         # for load/get
                         filepath_get = "/".join(filepathlist)
                         filename_get, file_extension_get = os.path.splitext(

api/reviewer_api/services/annotationservice.py

@@ -2,6 +2,7 @@
 from re import VERBOSE
 from reviewer_api.models.Annotations import Annotation
 from reviewer_api.models.AnnotationSections import AnnotationSection
+from reviewer_api.models.DocumentMaster import DocumentMaster
 from reviewer_api.schemas.annotationrequest import SectionAnnotationSchema
 
 from reviewer_api.models.default_method_result import DefaultMethodResult
@@ -174,6 +175,9 @@ def saveannotation(self, annotationschema, userinfo):
 
     def deactivateannotation(self, annotationnames, redactionlayerid, userinfo):
         return self.__deleteannotations(annotationnames, redactionlayerid, userinfo)
+
+    async def deactivatedocumentannotations(self, documentids, userinfo):
+        return Annotation.deletedocumentannotations(documentids, userinfo)
 
     def __deleteannotations(self, annotationnames, redactionlayerid, userinfo):
         if annotationnames not in (None, []) and len(annotationnames) > 0:

api/reviewer_api/services/documentpageflagservice.py

@@ -9,11 +9,11 @@
 
 
 class documentpageflagservice:
-    def getpageflags(self, requestid, redactionlayerid):
+    def getpageflags(self, requestid, redactionlayerid, documentids):
         layerids = redactionlayerservice().getmappedredactionlayers(
             {"redactionlayerid": redactionlayerid}
         )
-        return DocumentPageflag.getpageflag_by_request(requestid, layerids)
+        return DocumentPageflag.getpageflag_by_request(requestid, layerids, documentids)
 
     def getpublicbody(self, requestid):
         return DocumentPageflag.getpublicbody_by_request(requestid)
@@ -77,6 +77,15 @@ def bulksavedocumentpageflag(
             __docpgattributes,
         )
 
+    async def bulkarchivedocumentpageflag(
+        self, requestid, documentid, userinfo
+    ):
+        return DocumentPageflag.bulkarchivepageflag(
+            requestid,
+            documentid,
+            userinfo
+        )
+
     def bulksavepageflag(self, requestid, data, userinfo):
         results = []
         for entry in data["documentpageflags"]:

api/reviewer_api/services/documentservice.py

@@ -473,3 +473,6 @@ def savedocument(self, documentid, documentversion, newfilepath, userid):
 
     def deleterequestdocument(self, documentid, documentversion):
         return
+
+    def getfilepathbydocumentid(self, documentid):
+        return DocumentMaster.getfilepathbydocumentid(documentid)

api/reviewer_api/services/jobrecordservice.py

@@ -3,9 +3,13 @@
 from reviewer_api.models.PDFStitchJob import PDFStitchJob
 from reviewer_api.models.DocumentMaster import DocumentMaster
 from reviewer_api.models.DocumentAttributes import DocumentAttributes
+from reviewer_api.services.annotationservice import annotationservice
+from reviewer_api.services.documentpageflagservice import documentpageflagservice
+from reviewer_api.auth import auth, AuthHelper
 from datetime import datetime as datetime2
 from reviewer_api.utils.constants import FILE_CONVERSION_FILE_TYPES, DEDUPE_FILE_TYPES
 import json, os
+import asyncio
 from reviewer_api.utils.util import pstformat
 
 class jobrecordservice:
@@ -70,6 +74,12 @@ def recordjobstatus(self, batchinfo, userid):
                 jobids[record['s3uripath']] = {'masterid': masterid, 'jobid': job.identifier}
             elif extension in DEDUPE_FILE_TYPES:
                 if batchinfo['trigger'] in ['recordupload', 'recordreplace']:
+                    if batchinfo['trigger'] == 'recordreplace':
+                        userinfo = AuthHelper.getuserinfo()
+                        userinfo['trigger'] = batchinfo['trigger']
+                        documentids = DocumentMaster.getprocessingchilddocumentids(record.get('documentmasterid'))
+                        asyncio.ensure_future(annotationservice().deactivatedocumentannotations(documentids, userinfo))
+                        # asyncio.ensure_future(documentpageflagservice().bulkarchivedocumentpageflag(batchinfo['ministryrequestid'], documentids, userinfo))
                     master = DocumentMaster.create(
                         DocumentMaster(
                             filepath=record['s3uripath'],