Merge branch 'main' of github.com:Zipstack/unstract into UN-2301-ETL-Error-is-not-captured-and-added-in-logs-if-a-connector-stopped-working-suddenly

Author: kirtimanmishrazipstack

backend/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     "python-socketio==5.9.0", # For log_events
     "social-auth-app-django==5.3.0", # For OAuth
     "social-auth-core==4.4.2", # For OAuth
-    "unstract-sdk[aws,gcs,azure]~=0.78.0",
+    "unstract-sdk[aws,gcs,azure]~=0.79.0",
     "azure-identity==1.16.0",
     "azure-mgmt-apimanagement==3.0.0",
     "croniter>=3.0.3",

backend/uv.lock

@@ -59,6 +59,7 @@ directories=(
     "unstract/core"
     "unstract/flags"
     "unstract/connectors"
+    "workers"
 )
 
 # If directories are passed as arguments, override the default

docker/scripts/uv-lock-gen/uv-lock.sh

@@ -374,11 +374,25 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
               <div>
                 <Tag color="rgb(45, 183, 245)">
                   Confidence Score:{" "}
-                  {selectedHighlight?.confidence?.[0]?.[0]?.confidence ||
-                    (Array.isArray(selectedHighlight?.confidence?.[0]) &&
-                      selectedHighlight?.confidence?.[0]?.length === 0 &&
-                      "1") ||
-                    "NA"}
+                  {(() => {
+                    const { confidence } = selectedHighlight || {};
+                    // Handle new format: confidence is a number (average)
+                    if (typeof confidence === "number") {
+                      return confidence.toFixed(2);
+                    }
+                    // Handle old format: nested array structure
+                    if (confidence?.[0]?.[0]?.confidence) {
+                      return confidence[0][0].confidence;
+                    }
+                    // Handle old format: empty array means confidence = 1
+                    if (
+                      Array.isArray(confidence?.[0]) &&
+                      confidence[0].length === 0
+                    ) {
+                      return "1";
+                    }
+                    return "NA";
+                  })()}
                 </Tag>
               </div>
             )}

frontend/src/components/custom-tools/document-manager/DocumentManager.jsx

@@ -23,11 +23,18 @@ function PdfViewer({ fileUrl, highlightData, currentHighlightIndex }) {
 
   function removeZerosAndDeleteIfAllZero(highlightData) {
     if (Array.isArray(highlightData))
-      return highlightData?.filter((innerArray) => {
-        return (
-          Array.isArray(innerArray) && innerArray?.some((value) => value !== 0)
-        );
-      });
+      return highlightData
+        ?.filter((innerArray) => {
+          if (!Array.isArray(innerArray)) return false;
+          // Strip 5th element (confidence) if present, keep only first 4 elements
+          const coordsOnly =
+            innerArray.length >= 5 ? innerArray.slice(0, 4) : innerArray;
+          return coordsOnly.some((value) => value !== 0);
+        })
+        .map((innerArray) => {
+          // Return only the first 4 elements (strip confidence)
+          return innerArray.length >= 5 ? innerArray.slice(0, 4) : innerArray;
+        });
   }
 
   const processHighlightData = highlightData

frontend/src/components/custom-tools/pdf-viewer/PdfViewer.jsx

@@ -171,7 +171,9 @@ const PromptCard = memo(
 
     const addCoordsToFlattened = (coords, flattened) => {
       if (Array.isArray(coords)) {
-        flattened.push(coords);
+        // Strip 5th element (confidence) if present, keep only first 4 elements
+        const coordsOnly = coords.length >= 5 ? coords.slice(0, 4) : coords;
+        flattened.push(coordsOnly);
       }
     };
 
@@ -211,6 +213,47 @@ const PromptCard = memo(
       return flattened;
     };
 
+    const extractConfidenceFromHighlightData = (data) => {
+      if (!data) return null;
+
+      const confidenceValues = [];
+
+      const extractFromArray = (arr) => {
+        if (Array.isArray(arr)) {
+          for (const item of arr) {
+            if (Array.isArray(item)) {
+              // Check if this is a coordinate array with 5 elements
+              if (item.length >= 5 && typeof item[4] === "number") {
+                confidenceValues.push(item[4]);
+              } else {
+                // Recursively check nested arrays
+                extractFromArray(item);
+              }
+            } else if (typeof item === "object" && item !== null) {
+              // Recursively check objects
+              for (const val of Object.values(item)) {
+                extractFromArray(val);
+              }
+            }
+          }
+        } else if (typeof arr === "object" && arr !== null) {
+          for (const val of Object.values(arr)) {
+            extractFromArray(val);
+          }
+        }
+      };
+
+      extractFromArray(data);
+
+      // Calculate average confidence if we found any values
+      if (confidenceValues.length > 0) {
+        const sum = confidenceValues.reduce((acc, val) => acc + val, 0);
+        return sum / confidenceValues.length;
+      }
+
+      return null;
+    };
+
     const handleSelectHighlight = (
       highlightData,
       highlightedPrompt,
@@ -224,12 +267,18 @@ const PromptCard = memo(
           !Array.isArray(highlightData)
             ? flattenHighlightData(highlightData)
             : highlightData;
+
+        // Try to extract confidence from 5th element first, fall back to API confidenceData
+        const extractedConfidence =
+          extractConfidenceFromHighlightData(highlightData);
+        const finalConfidence = extractedConfidence ?? confidenceData;
+
         updateCustomTool({
           selectedHighlight: {
             highlight: processedHighlight,
             highlightedPrompt: highlightedPrompt,
             highlightedProfile: highlightedProfile,
-            confidence: confidenceData,
+            confidence: finalConfidence,
           },
         });
       }

frontend/src/components/custom-tools/prompt-card/PromptCard.jsx

@@ -15,7 +15,7 @@ dependencies = [
     "redis~=5.2.1",
     "cryptography>=41.0.7",
     "requests>=2.31.0",
-    "unstract-sdk[gcs, azure, aws]~=0.78.0", # Add version
+    "unstract-sdk[gcs, azure, aws]~=0.79.0", # Add version
     "unstract-flags",
     "unstract-core[flask]",
     "unstract-sdk1[gcs, azure, aws]"

platform-service/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "python-dotenv==1.0.1",
     "json-repair~=0.42.0",
     "requests>=2.28,<3.0",
-    "unstract-sdk[aws,gcs,azure]~=0.78.0",
+    "unstract-sdk[aws,gcs,azure]~=0.79.0",
     "redis>=5.0.3,<5.3",
     "unstract-core",
     "unstract-flags",