Merge pull request #760 from bcgov/dev-marshal-AS-4975

Fixes and latest merge from Main branch for consult updates

MCS.FOI.S3FileConversion/MCS.FOI.S3FileConversion/Program.cs

@@ -118,6 +118,14 @@ static async System.Threading.Tasks.Task Main(string[] args)
                                         {
                                             for (int i = 0; i < attachments.Count; i++)
                                             {
+                                                Dictionary<string, object> attributesDictionary = JsonSerializer.Deserialize<Dictionary<string, object>>(attachments[i]["attributes"]);
+                                                string _incompatible;
+                                                if (attributesDictionary.ContainsKey("incompatible"))
+                                                {
+                                                    _incompatible = attributesDictionary["incompatible"].ToString().ToLower();
+                                                } else {
+                                                    _incompatible = "false";
+                                                }
                                                 if (Array.IndexOf(ConversionSettings.ConversionFormats, attachments[i]["extension"].ToLower()) == -1)
                                                 {
                                                     db.StreamAdd(dedupeStreamKey, new NameValueEntry[]
@@ -131,6 +139,7 @@ static async System.Threading.Tasks.Task Main(string[] args)
                                                         new("batch", message["batch"]),
                                                         new("jobid", jobIDs[attachments[i]["filepath"]]["jobID"]),
                                                         new("documentmasterid", jobIDs[attachments[i]["filepath"]]["masterID"]),
+                                                        new("incompatible", _incompatible),
                                                         new("trigger", "attachment"),
                                                         new("createdby", message["createdby"]),
                                                         new("usertoken", message["usertoken"])

api/reviewer_api/services/documentpageflagservice.py

@@ -224,7 +224,7 @@ def __createnewpageflag(self, pageflag, data):
             if entry["page"] == data["page"]:
                 isnew = False
                 if data["flagid"] == 4:
-                    pageflag, updated = self.__handleconsultflag(data, entry, index, pageflag, formatted_data)
+                    pageflag, updated = self.__handleconsultflag(data, entry, index, pageflag, formatted_data, updated)
                 # handle all other flags except consult
                 elif data["flagid"] != 4 and entry["flagid"] != 4:
                     del pageflag[index]
@@ -237,8 +237,7 @@ def __createnewpageflag(self, pageflag, data):
 
         return pageflag
 
-    def __handleconsultflag(self, data, entry, index, pageflag, formatted_data):
-        updated = False
+    def __handleconsultflag(self, data, entry, index, pageflag, formatted_data, updated):
         # remove consult flag
         if data["flagid"] == 4 and not data["other"] and not data["programareaid"]:
             if entry["flagid"] == data["flagid"]:

api/reviewer_api/services/documentservice.py

@@ -462,6 +462,10 @@ def getdocuments(self, requestid,bcgovcode):
                 map(lambda d: {"divisionid": d}, documentdivisions)
             )
             document["divisions"] = list(map(lambda d: divisions[d], documentdivisions))
+            # For replaced attachments, change filepath to .pdf instead of original extension
+            if "trigger" in document["attributes"] and document["attributes"]["trigger"] == "recordreplace":
+                base_path, current_extension = path.splitext(document["filepath"])
+                document["filepath"] = base_path + ".pdf"
 
         return [documents[documentid] for documentid in documents]
 

computingservices/DedupeServices/services/s3documentservice.py

@@ -72,7 +72,10 @@ def gets3documenthashcode(producermessage):
         filepath = path.splitext(filepath)[0] + extension
     response = requests.get("{0}".format(filepath), auth=auth, stream=True)
     reader = None
-    if extension.lower() in [".pdf"]:
+
+    if extension.lower() in [".pdf"] or (
+        producermessage.attributes.get("isattachment", False) and producermessage.trigger == "recordreplace"
+        ):
         reader = PdfReader(BytesIO(response.content))
 
         # "No of pages in {0} is {1} ".format(_filename, len(reader.pages)))

computingservices/DocumentServices/services/cdogsapiservice.py

@@ -11,15 +11,8 @@
 class cdogsapiservice:
     """cdogs api Service class."""
 
-    # def __init__(self):
-    #     self.access_token = self._get_access_token()
-
-    def demomethod(self):
-        print("demomethod!!!")
 
     def generate_pdf(self, template_hash_code, data, access_token):
-        #access_token= self._get_access_token()
-        print('\n\ntemplate_hash_code:',template_hash_code)
         request_body = {
             "options": {
                 "cachereport": False,
@@ -30,7 +23,6 @@ def generate_pdf(self, template_hash_code, data, access_token):
             "data": data
         }
         json_request_body = json.dumps(request_body)
-        print("\njson_request_body:",json_request_body)
         headers = {
             'Content-Type': 'application/json',
             'Authorization': f'Bearer {access_token}'
@@ -41,44 +33,26 @@ def generate_pdf(self, template_hash_code, data, access_token):
     def _post_generate_pdf(self, json_request_body, headers, url):
         return requests.post(url, data= json_request_body, headers= headers)
 
-    def upload_template(self, receipt_template_path, access_token):
-        print('\nreceipt_template_path:',receipt_template_path)
-        #access_token= self._get_access_token()
-        file_dir = os.path.dirname(os.path.realpath('__file__'))
-        #print("file_dir:",file_dir)
-        template_file_path = os.path.join(file_dir, receipt_template_path)
-        #print("template_file_path:",template_file_path)
+    def upload_template(self, template_path, access_token):
         headers = {
         "Authorization": f'Bearer {access_token}'
         }
-
         url = f"{cdogs_base_url}/api/v2/template"
-
-        if os.path.exists(receipt_template_path):
-            print("@@@Exists!!")
-        template = {'template':('template', open(receipt_template_path, 'rb'), "multipart/form-data")}
-        print("\n\n1)url:",url)
-        print('\n\n2)Uploading template %s', template)
-        print("\n\n3)headers:",headers)
+        if os.path.exists(template_path):
+            print("Exists!!")
+        template = {'template':('template', open(template_path, 'rb'), "multipart/form-data")}
         response = self._post_upload_template(headers, url, template)
-        print("\n\nRESPONSE from UPLOAD ->",response)
         if response.status_code == 200:
-            # if response.headers.get("X-Template-Hash") is None:
-            #     raise BusinessException(Error.DATA_NOT_FOUND)
-
             print('Returning new hash %s', response.headers['X-Template-Hash'])
             return response.headers['X-Template-Hash'];
 
         response_json = json.loads(response.content)
-
         if response.status_code == 405 and response_json['detail'] is not None:
             match = re.findall(r"Hash '(.*?)'", response_json['detail']);
             if match:
-                print('Template already hashed with code %s', match[0])
                 print('Template already hashed with code %s', match[0])
                 return match[0]
 
-        #raise BusinessException(Error.DATA_NOT_FOUND)
 
     def _post_upload_template(self, headers, url, template):
         response = requests.post(url, headers= headers, files= template)
@@ -95,14 +69,12 @@ def check_template_cached(self, template_hash_code, access_token):
 
 
     def _get_access_token(self):
-        print("Inside _get_access_token!!")
         token_url = cdogs_token_url
         service_client = cdogs_service_client
         service_client_secret = cdogs_service_client_secret
         basic_auth_encoded = base64.b64encode(
             bytes(service_client + ':' + service_client_secret, 'utf-8')).decode('utf-8')
         data = 'grant_type=client_credentials'
-        print("basic_auth_encoded:",basic_auth_encoded)
         response = requests.post(
             token_url,
             data=data,

computingservices/DocumentServices/services/dal/documentpageflag.py

@@ -12,7 +12,7 @@ def get_all_pageflags(cls):
             cursor = conn.cursor()
             cursor.execute(
                 """select pageflagid, name, description from "Pageflags" 
-                where isactive = true order by sortorder
+                where isactive = true and name not in ('Consult') order by sortorder
                 """
             )
 
@@ -32,6 +32,33 @@ def get_all_pageflags(cls):
             if conn is not None:
                 conn.close()
 
+    @classmethod
+    def get_all_programareas(cls):
+        conn = getdbconnection()
+        programareas = {}
+        try:
+            cursor = conn.cursor()
+            cursor.execute(
+                """select programareaid, bcgovcode, iaocode from "ProgramAreas" 
+                where isactive = true order by programareaid
+                """
+            )
+
+            result = cursor.fetchall()
+            cursor.close()
+            if result is not None:
+                for entry in result:
+                    programareas[entry[0]] = {"bcgovcode": entry[1], "iaocode": entry[2]}
+                return programareas
+            return None
+
+        except Exception as error:
+            logging.error("Error in getting program areas")
+            logging.error(error)
+            raise
+        finally:
+            if conn is not None:
+                conn.close()
 
     @classmethod
     def get_documentpageflag(cls, ministryrequestid, redactionlayerid, documentids):
@@ -127,23 +154,20 @@ def getpagecount_by_documentid(cls, ministryrequestid, documentids):
                 conn.close()
 
     @classmethod
-    def getsections_by_documentid_pageno(cls, ministryrequestid, redactionlayerid, documentid, pagenos):
+    def getsections_by_documentid_pageno(cls, redactionlayerid, documentid, pagenos):
         conn = getdbconnection()
         sections = []
         try:
             cursor = conn.cursor()
             cursor.execute(
-                """select a.pagenumber, as2."section"  from "AnnotationSections" as2,
-                    "Annotations" a
-                    where as2.foiministryrequestid = %s::integer
-                    and as2.isactive = true
-                    and as2.annotationname  = a.annotationname 
-                    and as2.redactionlayerid = a.redactionlayerid 
-                    and a.redactionlayerid = %s::integer 
-                    and a.documentid =  %s::integer
-                    and a.pagenumber in %s
-                    order by a.pagenumber;""",
-                (ministryrequestid, redactionlayerid, documentid, tuple(pagenos)),
+                """select pagenumber , unnest(xpath('//contents/text()', annotation::xml))::text as sections 
+                   from "Annotations" a 
+                   where annotation like '%%freetext%%' and isactive = true 
+                   and redactionlayerid = %s::integer
+                   and documentid = %s::integer
+                   and pagenumber in %s
+                   order by pagenumber;""",
+                (redactionlayerid, documentid, tuple(pagenos)),
             )
 
             result = cursor.fetchall()

computingservices/DocumentServices/services/dal/documenttemplateservice.py

@@ -11,12 +11,11 @@ def gettemplatebytype(cls, documenttypeid, extension= "docx"):
         try:
             cursor = conn.cursor()
             query = '''
-                SELECT *
+                SELECT cdogs_hash_code
                 FROM public."DocumentTemplates" 
-                WHERE document_type_id = %s::integer 
-                ORDER BY version DESC LIMIT 1;
+                WHERE document_type_id = %s and extension = %s;
             '''
-            parameters = (documenttypeid)
+            parameters = (documenttypeid,extension,)
             cursor.execute(query, parameters)
             documenttemplate = cursor.fetchone()[0]
             return documenttemplate
@@ -25,6 +24,7 @@ def gettemplatebytype(cls, documenttypeid, extension= "docx"):
             logging.error(error)
             raise
         finally:
+            cursor.close()
             if conn is not None:
                 conn.close()
 
@@ -34,16 +34,16 @@ def updatecdogshashcode(cls, documenttypeid, cdogshashcode):
         try:        
             cursor = conn.cursor()
             query = '''
-                    UPDATE public."DocumentTemplates" SET cdogs_hash_code = %s::str
-                    WHERE document_type_id = %s::integer;
+                    UPDATE public."DocumentTemplates" SET cdogs_hash_code = %s
+                    WHERE document_type_id = %s;
                 '''
             parameters = (cdogshashcode, documenttypeid,)
             cursor.execute(query, parameters)
             conn.commit()
-            cursor.close()
         except(Exception) as error:
             print("Exception while executing func updatecdogshashcode, Error : {0} ".format(error))
             raise
         finally:
+            cursor.close()
             if conn is not None:
                 conn.close()

computingservices/DocumentServices/services/dal/documenttypeservice.py

@@ -6,17 +6,16 @@
 class documenttypeservice:
 
     @classmethod 
-    def getdocumenttypebyname(cls, document_type_name, extension= "docx"):
+    def getdocumenttypebyname(cls, document_type_name):
         conn = getfoidbconnection()
         try:
             cursor = conn.cursor()
             query = '''
                 SELECT *
                 FROM public."DocumentTypes" 
-                WHERE document_type_name = %s::integer 
-                ORDER BY version DESC LIMIT 1;
+                WHERE document_type_name = %s;
             '''
-            parameters = (document_type_name)
+            parameters = (document_type_name,)
             cursor.execute(query, parameters)
             documenttemplate = cursor.fetchone()[0]
             return documenttemplate
@@ -25,6 +24,7 @@ def getdocumenttypebyname(cls, document_type_name, extension= "docx"):
             logging.error(error)
             raise
         finally:
+            cursor.close()
             if conn is not None:
                 conn.close()
 

computingservices/DocumentServices/services/documentgenerationservice.py

@@ -33,43 +33,34 @@ class documentgenerationservice:
     #         raise BusinessException(Error.DATA_NOT_FOUND)  
 
 
-    def generate_pdf(self, documenttypename, data, receipt_template_path='templates/redline_redaction_summary.docx'):
+    def generate_pdf(self, documenttypename, data, template_path='templates/redline_redaction_summary.docx'):
         access_token= cdogsapiservice()._get_access_token()
         template_cached = False
         templatefromdb= self.__gettemplate(documenttypename)
         print("\n***templatefromdb:",templatefromdb)
-        if templatefromdb is not None and templatefromdb["cdogs_hash_code"]:
+        if templatefromdb is not None and templatefromdb["cdogs_hash_code"] is not None:
             print('Checking if template %s is cached', templatefromdb["cdogs_hash_code"])
             template_cached = cdogsapiservice().check_template_cached(templatefromdb["cdogs_hash_code"], access_token)
             templatecdogshashcode = templatefromdb["cdogs_hash_code"]
 
         if templatefromdb is None or templatefromdb["cdogs_hash_code"] is None or not template_cached:
-            templatecdogshashcode = cdogsapiservice().upload_template(receipt_template_path, access_token)
+            templatecdogshashcode = cdogsapiservice().upload_template(template_path, access_token)
+            templatefromdb["cdogs_hash_code"] = templatecdogshashcode
             print('Uploading new template--->',templatecdogshashcode)
-            if templatefromdb is not None:
+            if templatefromdb is not None and templatefromdb["document_type_id"] is not None:
                 documenttemplateservice().updatecdogshashcode(templatefromdb["document_type_id"], templatefromdb["cdogs_hash_code"])
-            # receipt_template.flush()
-            # receipt_template.commit()
         print('Generating redaction summary')
         return cdogsapiservice().generate_pdf(templatecdogshashcode, data,access_token)
 
     def __gettemplate(self,documenttypename='redline_redaction_summary'):
         try:
-            receipt_document_type =documenttypeservice().getdocumenttypebyname(documenttypename)
-            receipt_template=None
-            if receipt_document_type is not None:                          
-                receipt_template=documenttemplateservice().gettemplatebytype(receipt_document_type.document_type_id)
-            return receipt_template
+            templatefromdb=None
+            summary_cdogs_hash_code=None
+            summary_document_type_id =documenttypeservice().getdocumenttypebyname(documenttypename)
+            if summary_document_type_id is not None:             
+                summary_cdogs_hash_code=documenttemplateservice().gettemplatebytype(summary_document_type_id)
+                templatefromdb = {"document_type_id": summary_document_type_id, "cdogs_hash_code":summary_cdogs_hash_code}
+            return templatefromdb
         except (Exception) as error:
             print('error occured in document generation service - gettemplate method: ', error)
 
-    # def upload_receipt(self, filename, filebytes, ministryrequestid, ministrycode, filenumber):
-    #     try:
-    #         logging.info("Upload summary for ministry request id"+ str(ministryrequestid))
-    #         _response =  self.__uploadbytes(filename, filebytes, ministrycode, filenumber)
-    #         logging.info("Upload status for payload"+ json.dumps(_response))
-    #         if _response["success"] == True:
-    #             _documentschema = {"documents": [{"filename": _response["filename"], "documentpath": _response["documentpath"]}]}
-    #         return _response
-    #     except Exception as ex:
-    #         logging.exception(ex)