hotfix (crawl): log content byte size

indexnetwork · Feb 14, 2024 · 4b5894b · 4b5894b
1 parent 82db2e6
commit 4b5894b
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 3 deletions.
diff --git a/api/src/libs/kafka-indexer.js b/api/src/libs/kafka-indexer.js
@@ -204,8 +204,6 @@ export const createEmbeddingEvent = async (id) => {
         payload.indexOwnerBio = embedding.index.ownerDID.bio
     }
 
-    // console.log("Payload", payload)
-
     try {
         const indexResponse = await axios.post(`${process.env.LLM_INDEXER_HOST}/indexer/index?indexId=${embedding.item.id}`, payload)
         console.log(`IndexItem ${payload.webPageId} with ${payload.webPageUrl} Indexed with it's content and embeddings`)

diff --git a/indexer/src/indexer/service/indexer.service.ts b/indexer/src/indexer/service/indexer.service.ts
@@ -90,7 +90,7 @@ export class IndexerService {
             content = content + '\n' + doc?.pageContent;
         }
 
-        Logger.log(`Extracted ${docs[0].pageContent.length} bytes from ${url}`, 'indexerService:crawl');
+        Logger.log(`Extracted ${content.length} bytes from ${url}`, 'indexerService:crawl');
 
         return {
             url: url,