From f574802340627e7f2d6c47879a7b40f183b3b996 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Tue, 29 Oct 2024 21:00:54 +0200
Subject: [PATCH 01/47] modified margins

---
 frontend/nextjs/app/page.tsx          | 2 +-
 frontend/nextjs/components/Header.tsx | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 5af811970..899f0b6f5 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -395,7 +395,7 @@ export default function Home() {
   return (
     <>
       <Header />
-      <main className="h-full px-4 pb-4 pt-10">
+      <main className="h-full px-4 pb-4">
         {!showResult && (
           <Hero
             promptValue={promptValue}
diff --git a/frontend/nextjs/components/Header.tsx b/frontend/nextjs/components/Header.tsx
index 98919cd09..7ce3d8744 100644
--- a/frontend/nextjs/components/Header.tsx
+++ b/frontend/nextjs/components/Header.tsx
@@ -2,7 +2,7 @@ import Image from "next/image";
 
 const Header = () => {
   return (
-    <div className="container h-[60px] px-4 lg:h-[80px] lg:px-0 pt-10 sm:mb-12 sm:mb-8">
+    <div className="container h-[60px] px-4 lg:h-[80px] lg:px-0 pt-10">
       <div className="grid h-full grid-cols-12">
         <div className="col-span-5"></div>
         <div className="col-span-2 flex items-center justify-center">

From 173e832e96868ebdec0fd1d2af38ef604ff8f877 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Thu, 31 Oct 2024 12:06:29 +0200
Subject: [PATCH 02/47] added starting boilderplate to ux

---
 frontend/nextjs/app/page.tsx          | 2 +-
 frontend/nextjs/components/Footer.tsx | 6 ++++--
 frontend/nextjs/components/Hero.tsx   | 4 ++--
 frontend/styles.css                   | 1 -
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 899f0b6f5..6ac6869b3 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -395,7 +395,7 @@ export default function Home() {
   return (
     <>
       <Header />
-      <main className="h-full px-4 pb-4">
+      <main className="h-full px-4 pb-[72px]">
         {!showResult && (
           <Hero
             promptValue={promptValue}
diff --git a/frontend/nextjs/components/Footer.tsx b/frontend/nextjs/components/Footer.tsx
index 56492cf38..878d924ff 100644
--- a/frontend/nextjs/components/Footer.tsx
+++ b/frontend/nextjs/components/Footer.tsx
@@ -16,7 +16,9 @@ interface ChatBoxProps {
 const Footer = ({ setChatBoxSettings, chatBoxSettings}: ChatBoxProps) => {
   
   return (
-    <>
+    <footer className="fixed bottom-0 left-0 right-0" style={{
+      background: 'linear-gradient(45deg, #151A2D, #111827, #151A2D)'
+    }}>
       <div className="container flex min-h-[72px] items-center justify-between border-t border-[#D2D2D2] px-4 pb-3 pt-5 lg:min-h-[72px] lg:px-0 lg:py-5">
         <Modal setChatBoxSettings={setChatBoxSettings} chatBoxSettings={chatBoxSettings} />
         <div className="text-sm text-gray-500">
@@ -49,7 +51,7 @@ const Footer = ({ setChatBoxSettings, chatBoxSettings}: ChatBoxProps) => {
           </Link>
         </div>
       </div>
-    </>
+    </footer>
   );
 };
 
diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index eb21ddb83..2eee7acd5 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -21,7 +21,7 @@ const Hero: FC<THeroProps> = ({
     <div>
       <div className="flex flex-col items-center justify-center sm:mt-12 mt-8">
         <div className="landing flex flex-col items-center">
-          <h1 className="text-4xl font-extrabold text-center lg:text-7xl mb-6 lg:mb-10">
+          <h1 className="text-4xl font-extrabold text-center lg:text-7xl">
             Say Goodbye to <br />
             <span
               style={{
@@ -39,7 +39,7 @@ const Hero: FC<THeroProps> = ({
         </div>
 
         {/* Input section */}
-        <div className="w-full max-w-[708px] pb-6 mt-20">
+        <div className="w-full max-w-[708px] pb-6">
           <InputArea
             promptValue={promptValue}
             setPromptValue={setPromptValue}
diff --git a/frontend/styles.css b/frontend/styles.css
index f22acac5d..cad1cadf6 100644
--- a/frontend/styles.css
+++ b/frontend/styles.css
@@ -100,7 +100,6 @@ footer {
     left: 0;
     bottom: 0;
     width: 100%;
-    background-color: #1e272e;
     color: white;
     text-align: center;
     padding: 10px 0;

From 9f9ebe7802385e4c2ecee4b03ada1ce4e399e2e5 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:04:53 +0200
Subject: [PATCH 03/47] updated margins

---
 frontend/nextjs/components/Hero.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index b25e2819a..9a75357eb 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -19,7 +19,7 @@ const Hero: FC<THeroProps> = ({
 
   return (
     <div>
-      <div className="flex flex-col items-center justify-center sm:mt-12 mt-8">
+      <div className="flex flex-col items-center justify-center">
         <div className="landing flex flex-col items-center">
           <h1 className="text-4xl font-extrabold text-center lg:text-7xl">
             Say Goodbye to <br />

From 8b525224fa097e8e943fb9c54ebb9d19b37e8a5a Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:25:05 +0200
Subject: [PATCH 04/47] fixed some ux issues

---
 frontend/nextjs/app/page.tsx          | 2 +-
 frontend/nextjs/components/Footer.tsx | 8 +++-----
 frontend/nextjs/components/Header.tsx | 2 +-
 frontend/nextjs/components/Hero.tsx   | 2 +-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 86a310fb7..e60ffb3ec 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -414,7 +414,7 @@ export default function Home() {
   return (
     <>
       <Header />
-      <main className="h-full px-4 pb-[72px]">
+      <main className="min-h-[calc(100vh-64px)]">
         {!showResult && (
           <Hero
             promptValue={promptValue}
diff --git a/frontend/nextjs/components/Footer.tsx b/frontend/nextjs/components/Footer.tsx
index 878d924ff..9bd451c9a 100644
--- a/frontend/nextjs/components/Footer.tsx
+++ b/frontend/nextjs/components/Footer.tsx
@@ -16,10 +16,8 @@ interface ChatBoxProps {
 const Footer = ({ setChatBoxSettings, chatBoxSettings}: ChatBoxProps) => {
   
   return (
-    <footer className="fixed bottom-0 left-0 right-0" style={{
-      background: 'linear-gradient(45deg, #151A2D, #111827, #151A2D)'
-    }}>
-      <div className="container flex min-h-[72px] items-center justify-between border-t border-[#D2D2D2] px-4 pb-3 pt-5 lg:min-h-[72px] lg:px-0 lg:py-5">
+    <>
+      <div className="container flex min-h-[72px] mt-2 items-center justify-between border-t border-[#D2D2D2] px-4 pb-3 pt-5 lg:min-h-[72px] lg:px-0 lg:py-5">
         <Modal setChatBoxSettings={setChatBoxSettings} chatBoxSettings={chatBoxSettings} />
         <div className="text-sm text-gray-500">
             © {new Date().getFullYear()} GPT Researcher. All rights reserved.
@@ -51,7 +49,7 @@ const Footer = ({ setChatBoxSettings, chatBoxSettings}: ChatBoxProps) => {
           </Link>
         </div>
       </div>
-    </footer>
+    </>
   );
 };
 
diff --git a/frontend/nextjs/components/Header.tsx b/frontend/nextjs/components/Header.tsx
index 7ce3d8744..2ba6caff9 100644
--- a/frontend/nextjs/components/Header.tsx
+++ b/frontend/nextjs/components/Header.tsx
@@ -2,7 +2,7 @@ import Image from "next/image";
 
 const Header = () => {
   return (
-    <div className="container h-[60px] px-4 lg:h-[80px] lg:px-0 pt-10">
+    <div className="container h-[60px] px-4 lg:h-[80px] lg:px-0 pt-10 mb-10">
       <div className="grid h-full grid-cols-12">
         <div className="col-span-5"></div>
         <div className="col-span-2 flex items-center justify-center">
diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index 9a75357eb..997a45aff 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -19,7 +19,7 @@ const Hero: FC<THeroProps> = ({
 
   return (
     <div>
-      <div className="flex flex-col items-center justify-center">
+      <div className="flex flex-col items-center justify-center mt-20">
         <div className="landing flex flex-col items-center">
           <h1 className="text-4xl font-extrabold text-center lg:text-7xl">
             Say Goodbye to <br />

From b168e7ff8111cdf425394847fc8b3661758e8cbf Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:49:53 +0200
Subject: [PATCH 05/47] updated ux

---
 frontend/nextjs/components/InputArea.tsx      |  33 +---
 frontend/nextjs/components/Task/ImageModal.js |  90 -----------
 .../nextjs/components/Task/ImageModal.jsx     |  35 +++++
 .../nextjs/components/Task/ImagesCarousel.js  | 142 ------------------
 .../nextjs/components/Task/ImagesCarousel.jsx |  73 +++++++++
 .../nextjs/components/Task/ImagesGrid.jsx     |  83 ++++++++++
 .../nextjs/components/Task/LogMessage.tsx     |   2 +-
 frontend/nextjs/tsconfig.json                 |   2 +-
 8 files changed, 194 insertions(+), 266 deletions(-)
 delete mode 100644 frontend/nextjs/components/Task/ImageModal.js
 create mode 100644 frontend/nextjs/components/Task/ImageModal.jsx
 delete mode 100644 frontend/nextjs/components/Task/ImagesCarousel.js
 create mode 100644 frontend/nextjs/components/Task/ImagesCarousel.jsx
 create mode 100644 frontend/nextjs/components/Task/ImagesGrid.jsx

diff --git a/frontend/nextjs/components/InputArea.tsx b/frontend/nextjs/components/InputArea.tsx
index 783f6ba0a..1459dd94f 100644
--- a/frontend/nextjs/components/InputArea.tsx
+++ b/frontend/nextjs/components/InputArea.tsx
@@ -33,7 +33,7 @@ const InputArea: FC<TInputAreaProps> = ({
   reset,
 }) => {
   const placeholder = handleSecondary
-    ? "Follow up questions..."
+    ? "Have any follow up questions?"
     : "What would you like to research next?";
 
   const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -83,37 +83,6 @@ const InputArea: FC<TInputAreaProps> = ({
         resetHeight();
       }}
     >
-      {handleSecondary && (
-        <div
-          role="button"
-          aria-disabled={disabled}
-          className="relative flex h-[50px] w-[50px] shrink-0 items-center justify-center rounded-[3px] bg-[linear-gradient(154deg,#1B1B16_23.37%,#565646_91.91%)] disabled:pointer-events-none disabled:opacity-75"
-          onClick={(e) => {
-            if (!disabled) {
-              e.preventDefault();
-              if (reset) reset();
-              handleSecondary(promptValue);
-              setPromptValue(''); // Clear prompt value
-              resetHeight();
-            }
-          }}
-        >
-          {disabled && (
-            <div className="absolute inset-0 flex items-center justify-center">
-              <TypeAnimation />
-            </div>
-          )}
-
-          <Image
-            unoptimized
-            src={"/img/search.svg"}
-            alt="search"
-            width={24}
-            height={24}
-            className={disabled ? "invisible" : ""}
-          />
-        </div>
-      )}
 
       <textarea
         placeholder={placeholder}
diff --git a/frontend/nextjs/components/Task/ImageModal.js b/frontend/nextjs/components/Task/ImageModal.js
deleted file mode 100644
index 4bf2c93b0..000000000
--- a/frontend/nextjs/components/Task/ImageModal.js
+++ /dev/null
@@ -1,90 +0,0 @@
-import React, { useEffect } from 'react';
-
-export default function ImageModal({ imageSrc, isOpen, onClose, onNext, onPrev }) {
-    if (!isOpen) return null;
-
-    // Set up keyboard event listeners
-    useEffect(() => {
-        const handleKeyDown = (e) => {
-            if (e.key === 'ArrowLeft') {
-                onPrev();
-            } else if (e.key === 'ArrowRight') {
-                onNext();
-            } else if (e.key === 'Escape') {
-                onClose();
-            }
-        };
-
-        document.addEventListener('keydown', handleKeyDown);
-        return () => document.removeEventListener('keydown', handleKeyDown);
-    }, [onClose, onNext, onPrev]);
-
-    // Swipe detection for mobile
-    let touchStartX = 0;
-    let touchEndX = 0;
-
-    const handleTouchStart = (e) => {
-        touchStartX = e.changedTouches[0].screenX;
-    };
-
-    const handleTouchEnd = (e) => {
-        touchEndX = e.changedTouches[0].screenX;
-        handleSwipeGesture();
-    };
-
-    const handleSwipeGesture = () => {
-        if (touchEndX < touchStartX - 50) {
-            onNext();
-        } else if (touchEndX > touchStartX + 50) {
-            onPrev();
-        }
-    };
-
-    const handleClose = (e) => {
-        if (e.target === e.currentTarget) {
-            onClose();
-        }
-    };
-
-    return (
-        <div
-            className="fixed inset-0 z-50 flex items-center justify-center bg-black bg-opacity-60 overflow-auto"
-            onClick={handleClose}
-            onTouchStart={handleTouchStart}
-            onTouchEnd={handleTouchEnd}
-        >
-            <div className="relative m-4 max-w-full">
-                <button
-                    className="absolute top-0 right-0 p-2 text-white"
-                    onClick={onClose}
-                >
-                    <svg
-                        xmlns="http://www.w3.org/2000/svg"
-                        fill="none"
-                        viewBox="0 0 24 24"
-                        strokeWidth={2}
-                        stroke="currentColor"
-                        className="w-6 h-6"
-                    >
-                        <path
-                            strokeLinecap="round"
-                            strokeLinejoin="round"
-                            d="M6 18L18 6M6 6l12 12"
-                        />
-                    </svg>
-                </button>
-                <img
-                    src={imageSrc}
-                    alt="Full view"
-                    className="max-h-[80vh] max-w-full object-contain"
-                />
-                <button onClick={onPrev} className="absolute left-0 p-2 text-white">
-                    Prev
-                </button>
-                <button onClick={onNext} className="absolute right-0 p-2 text-white">
-                    Next
-                </button>
-            </div>
-        </div>
-    );
-}
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/ImageModal.jsx b/frontend/nextjs/components/Task/ImageModal.jsx
new file mode 100644
index 000000000..c833069c1
--- /dev/null
+++ b/frontend/nextjs/components/Task/ImageModal.jsx
@@ -0,0 +1,35 @@
+import React, { useEffect } from 'react';
+
+export default function ImageModal({ imageSrc, isOpen, onClose, onNext, onPrev }) {
+    if (!isOpen) return null;
+
+    return (
+        <div className="fixed inset-0 bg-black bg-opacity-75 z-50 flex items-center justify-center p-4">
+            <div className="relative max-w-[90vw] max-h-[90vh] flex items-center justify-center">
+                <button
+                    onClick={onPrev}
+                    className="absolute left-4 z-10 bg-black bg-opacity-50 text-white p-2 rounded-full hover:bg-opacity-75"
+                >
+                    ←
+                </button>
+                <img
+                    src={imageSrc}
+                    alt="Modal view"
+                    className="max-h-[90vh] max-w-[90vw] object-contain"
+                />
+                <button
+                    onClick={onNext}
+                    className="absolute right-4 z-10 bg-black bg-opacity-50 text-white p-2 rounded-full hover:bg-opacity-75"
+                >
+                    →
+                </button>
+                <button
+                    onClick={onClose}
+                    className="absolute top-4 right-4 z-10 bg-black bg-opacity-50 text-white p-2 rounded-full hover:bg-opacity-75"
+                >
+                    ×
+                </button>
+            </div>
+        </div>
+    );
+}
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/ImagesCarousel.js b/frontend/nextjs/components/Task/ImagesCarousel.js
deleted file mode 100644
index cbc3931b5..000000000
--- a/frontend/nextjs/components/Task/ImagesCarousel.js
+++ /dev/null
@@ -1,142 +0,0 @@
-import React, { useState, useEffect } from 'react';
-import ImageModal from './ImageModal'; // Import the ImageModal component
-
-export default function ImagesCarousel({ images }) {
-    const [activeIndex, setActiveIndex] = useState(0);
-    const [isModalOpen, setIsModalOpen] = useState(false);
-    const [selectedImage, setSelectedImage] = useState(null);
-    const [validImages, setValidImages] = useState(images); // Keep track of valid images
-
-    const nextSlide = () => {
-        setActiveIndex((prevIndex) => (prevIndex + 1) % validImages.length);
-    };
-
-    const prevSlide = () => {
-        setActiveIndex((prevIndex) => (prevIndex - 1 + validImages.length) % validImages.length);
-    };
-
-    const openModal = (image) => {
-        setSelectedImage(image);
-        setIsModalOpen(true);
-    };
-
-    const closeModal = () => {
-        setIsModalOpen(false);
-        setSelectedImage(null);
-    };
-
-    // Handle broken images by filtering them out
-    const handleImageError = (brokenImage) => {
-        setValidImages((prevImages) => prevImages.filter((img) => img !== brokenImage));
-    };
-
-    useEffect(() => {
-        const imagesToHide = ['https://gptr.dev/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75',"https://tavily.com/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75"]
-        const filteredImages = images.filter((img) => !imagesToHide.includes(img));
-        setValidImages(filteredImages);
-    }, [images]);
-
-    // Hide the entire component if there are no valid images
-    if (validImages.length === 0) return null;
-
-    return (
-        <div id="default-carousel" className="relative w-full" data-carousel="slide">
-            <div className="relative h-56 overflow-hidden rounded-lg md:h-96">
-                {validImages.map((image, index) => (
-                    <div
-                        key={index}
-                        className={`duration-700 ease-in-out ${index === activeIndex ? '' : 'hidden'}`}
-                        data-carousel-item
-                    >
-                        <img
-                            src={image}
-                            className="absolute block w-full -translate-x-1/2 -translate-y-1/2 top-1/2 left-1/2 cursor-pointer"
-                            alt={`Slide ${index + 1}`}
-                            onClick={() => openModal(image)} // Open modal on image click
-                            onError={() => handleImageError(image)} // Handle broken images
-                        />
-                    </div>
-                ))}
-            </div>
-
-            {/* Only show indicators and buttons if there is more than 1 valid image */}
-            {validImages.length > 1 && (
-                <>
-                    <div className="absolute z-30 flex -translate-x-1/2 bottom-5 left-1/2 space-x-3 rtl:space-x-reverse">
-                        {validImages.map((_, index) => (
-                            <button
-                                key={index}
-                                type="button"
-                                className={`w-3 h-3 rounded-full ${index === activeIndex ? 'bg-gray-800' : 'bg-gray-400'}`}
-                                aria-current={index === activeIndex ? 'true' : 'false'}
-                                aria-label={`Slide ${index + 1}`}
-                                onClick={() => setActiveIndex(index)}
-                                data-carousel-slide-to={index}
-                            ></button>
-                        ))}
-                    </div>
-
-                    <button
-                        type="button"
-                        className="absolute top-0 start-0 z-30 flex items-center justify-center h-full px-4 cursor-pointer group focus:outline-none"
-                        onClick={prevSlide}
-                        data-carousel-prev
-                    >
-                        <span className="inline-flex items-center justify-center w-10 h-10 rounded-full bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none">
-                            <svg
-                                className="w-4 h-4 text-white dark:text-gray-800 rtl:rotate-180"
-                                xmlns="http://www.w3.org/2000/svg"
-                                fill="none"
-                                viewBox="0 0 6 10"
-                            >
-                                <path
-                                    stroke="currentColor"
-                                    strokeLinecap="round"
-                                    strokeLinejoin="round"
-                                    strokeWidth="2"
-                                    d="M5 1 1 5l4 4"
-                                />
-                            </svg>
-                            <span className="sr-only">Previous</span>
-                        </span>
-                    </button>
-                    <button
-                        type="button"
-                        className="absolute top-0 end-0 z-30 flex items-center justify-center h-full px-4 cursor-pointer group focus:outline-none"
-                        onClick={nextSlide}
-                        data-carousel-next
-                    >
-                        <span className="inline-flex items-center justify-center w-10 h-10 rounded-full bg-white/30 dark:bg-gray-800/30 group-hover:bg-white/50 dark:group-hover:bg-gray-800/60 group-focus:ring-4 group-focus:ring-white dark:group-focus:ring-gray-800/70 group-focus:outline-none">
-                            <svg
-                                className="w-4 h-4 text-white dark:text-gray-800 rtl:rotate-180"
-                                xmlns="http://www.w3.org/2000/svg"
-                                fill="none"
-                                viewBox="0 0 6 10"
-                            >
-                                <path
-                                    stroke="currentColor"
-                                    strokeLinecap="round"
-                                    strokeLinejoin="round"
-                                    strokeWidth="2"
-                                    d="m1 9 4-4-4-4"
-                                />
-                            </svg>
-                            <span className="sr-only">Next</span>
-                        </span>
-                    </button>
-                </>
-            )}
-
-            {/* Render the modal when it's open */}
-            {selectedImage && (
-                <ImageModal
-                    imageSrc={validImages[activeIndex]}
-                    isOpen={isModalOpen}
-                    onClose={closeModal}
-                    onNext={nextSlide}
-                    onPrev={prevSlide}
-                />
-            )}
-        </div>
-    );
-}
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/ImagesCarousel.jsx b/frontend/nextjs/components/Task/ImagesCarousel.jsx
new file mode 100644
index 000000000..a9801cea3
--- /dev/null
+++ b/frontend/nextjs/components/Task/ImagesCarousel.jsx
@@ -0,0 +1,73 @@
+import React, { useState, useEffect } from 'react';
+import ImageModal from './ImageModal'; // Import the ImageModal component
+
+export default function ImagesGrid({ images }) {
+    const [isModalOpen, setIsModalOpen] = useState(false);
+    const [selectedImage, setSelectedImage] = useState(null);
+    const [selectedIndex, setSelectedIndex] = useState(0);
+    const [validImages, setValidImages] = useState(images);
+
+    const openModal = (image, index) => {
+        setSelectedImage(image);
+        setSelectedIndex(index);
+        setIsModalOpen(true);
+    };
+
+    const closeModal = () => {
+        setIsModalOpen(false);
+        setSelectedImage(null);
+    };
+
+    // Handle navigation in modal
+    const nextImage = () => {
+        setSelectedIndex((prevIndex) => (prevIndex + 1) % validImages.length);
+        setSelectedImage(validImages[(selectedIndex + 1) % validImages.length]);
+    };
+
+    const prevImage = () => {
+        setSelectedIndex((prevIndex) => (prevIndex - 1 + validImages.length) % validImages.length);
+        setSelectedImage(validImages[(selectedIndex - 1 + validImages.length) % validImages.length]);
+    };
+
+    // Handle broken images by filtering them out
+    const handleImageError = (brokenImage) => {
+        setValidImages((prevImages) => prevImages.filter((img) => img !== brokenImage));
+    };
+
+    useEffect(() => {
+        const imagesToHide = ['https://gptr.dev/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75',"https://tavily.com/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75"]
+        const filteredImages = images.filter((img) => !imagesToHide.includes(img));
+        setValidImages(filteredImages);
+    }, [images]);
+
+    // Hide the entire component if there are no valid images
+    if (validImages.length === 0) return null;
+
+    return (
+        <div className="w-full mt-5 mb-5">
+            <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+                {validImages.map((image, index) => (
+                    <div key={index} className="relative aspect-square">
+                        <img
+                            src={image}
+                            alt={`Image ${index + 1}`}
+                            className="absolute inset-0 w-full h-full object-cover rounded-lg cursor-pointer hover:opacity-90 transition-opacity"
+                            onClick={() => openModal(image, index)}
+                            onError={() => handleImageError(image)}
+                        />
+                    </div>
+                ))}
+            </div>
+
+            {selectedImage && (
+                <ImageModal
+                    imageSrc={selectedImage}
+                    isOpen={isModalOpen}
+                    onClose={closeModal}
+                    onNext={nextImage}
+                    onPrev={prevImage}
+                />
+            )}
+        </div>
+    );
+}
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/ImagesGrid.jsx b/frontend/nextjs/components/Task/ImagesGrid.jsx
new file mode 100644
index 000000000..f3dd24cf3
--- /dev/null
+++ b/frontend/nextjs/components/Task/ImagesGrid.jsx
@@ -0,0 +1,83 @@
+import React, { useState, useEffect } from 'react';
+import ImageModal from './ImageModal';
+
+const ImagesGrid = () => {
+    const [validImages, setValidImages] = useState([]);
+    const [selectedImage, setSelectedImage] = useState(null);
+    const [isModalOpen, setIsModalOpen] = useState(false);
+
+    useEffect(() => {
+        // Fetch valid images from the backend
+        fetchValidImages();
+    }, []);
+
+    const fetchValidImages = async () => {
+        try {
+            const response = await fetch('/api/images');
+            const data = await response.json();
+            setValidImages(data);
+        } catch (error) {
+            console.error('Error fetching valid images:', error);
+        }
+    };
+
+    const openModal = (image, index) => {
+        setSelectedImage(image);
+        setIsModalOpen(true);
+    };
+
+    const closeModal = () => {
+        setSelectedImage(null);
+        setIsModalOpen(false);
+    };
+
+    const nextImage = () => {
+        const currentIndex = validImages.indexOf(selectedImage);
+        const nextIndex = (currentIndex + 1) % validImages.length;
+        setSelectedImage(validImages[nextIndex]);
+    };
+
+    const prevImage = () => {
+        const currentIndex = validImages.indexOf(selectedImage);
+        const prevIndex = (currentIndex - 1 + validImages.length) % validImages.length;
+        setSelectedImage(validImages[prevIndex]);
+    };
+
+    const handleImageError = (image) => {
+        console.error(`Error loading image: ${image}`);
+    };
+
+    return (
+        <div className="w-full mt-5 mb-5">
+            <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+                {validImages.map((image, index) => (
+                    <div 
+                        key={index} 
+                        className="relative pt-[100%]" // Creates a square aspect ratio container
+                    >
+                        <img
+                            src={image}
+                            alt={`Image ${index + 1}`}
+                            className="absolute top-0 left-0 w-full h-full object-contain bg-gray-100 rounded-lg cursor-pointer hover:opacity-90 transition-opacity"
+                            onClick={() => openModal(image, index)}
+                            onError={() => handleImageError(image)}
+                        />
+                    </div>
+                ))}
+            </div>
+
+            {selectedImage && (
+                <ImageModal
+                    imageSrc={selectedImage}
+                    isOpen={isModalOpen}
+                    onClose={closeModal}
+                    onNext={nextImage}
+                    onPrev={prevImage}
+                    className="max-h-[80vh] w-auto" // Constrain modal image height
+                />
+            )}
+        </div>
+    );
+};
+
+export default ImagesGrid; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/Task/LogMessage.tsx
index 22354404b..8a77caa72 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/Task/LogMessage.tsx
@@ -3,7 +3,7 @@ import Accordion from './Accordion';
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
-import ImagesCarousel from './ImagesCarousel';
+import ImagesCarousel from './ImagesGrid';
 
 type ProcessedData = {
   field: string;
diff --git a/frontend/nextjs/tsconfig.json b/frontend/nextjs/tsconfig.json
index 3865f7381..0d86f12a1 100644
--- a/frontend/nextjs/tsconfig.json
+++ b/frontend/nextjs/tsconfig.json
@@ -21,6 +21,6 @@
       "@/*": ["./*"]
     }
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "components/Task/ImagesCarousel.js"],
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "components/Task/ImagesCarousel.jsx"],
   "exclude": ["node_modules"]
 }

From 27eab5655a2e7564fcbdf44df91ed9ecb4589ea2 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:53:25 +0200
Subject: [PATCH 06/47] removed input area when research is loading

---
 frontend/nextjs/app/page.tsx             | 4 ++--
 frontend/nextjs/components/InputArea.tsx | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index e60ffb3ec..a312d21b1 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -441,14 +441,14 @@ export default function Home() {
               <div className="pt-1 sm:pt-2" ref={chatContainerRef}></div>
             </div>
             <div id="input-area" className="container px-4 lg:px-0">
-              <InputArea
+              {!loading && <InputArea
                 promptValue={promptValue}
                 setPromptValue={setPromptValue}
                 handleSubmit={handleChat}
                 handleSecondary={handleDisplayResult}
                 disabled={loading}
                 reset={reset}
-              />
+              />}
             </div>
           </div>
         )}
diff --git a/frontend/nextjs/components/InputArea.tsx b/frontend/nextjs/components/InputArea.tsx
index 1459dd94f..caa1c7a56 100644
--- a/frontend/nextjs/components/InputArea.tsx
+++ b/frontend/nextjs/components/InputArea.tsx
@@ -12,9 +12,9 @@ type TInputAreaProps = {
 };
 
 // Debounce function to limit the rate at which a function can fire
-function debounce(func, wait) {
-  let timeout;
-  return function executedFunction(...args) {
+function debounce(func: Function, wait: number) {
+  let timeout: NodeJS.Timeout | undefined;
+  return function executedFunction(...args: any[]) {
     const later = () => {
       clearTimeout(timeout);
       func(...args);

From 9468d983f2e3ff02505736b39abb89543293e463 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:57:18 +0200
Subject: [PATCH 07/47] added loading indicator for more subtle experience

---
 frontend/nextjs/app/page.tsx               | 21 +++++++++++++--------
 frontend/nextjs/components/LoadingDots.tsx | 13 +++++++++++++
 2 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 frontend/nextjs/components/LoadingDots.tsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index a312d21b1..c17f8df66 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -17,6 +17,7 @@ import LogMessage from '../components/Task/LogMessage';
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
 import HumanFeedback from "@/components/HumanFeedback";
+import LoadingDots from "@/components/LoadingDots";
 
 interface BaseData {
   type: string;
@@ -441,14 +442,18 @@ export default function Home() {
               <div className="pt-1 sm:pt-2" ref={chatContainerRef}></div>
             </div>
             <div id="input-area" className="container px-4 lg:px-0">
-              {!loading && <InputArea
-                promptValue={promptValue}
-                setPromptValue={setPromptValue}
-                handleSubmit={handleChat}
-                handleSecondary={handleDisplayResult}
-                disabled={loading}
-                reset={reset}
-              />}
+              {loading ? (
+                <LoadingDots />
+              ) : (
+                <InputArea
+                  promptValue={promptValue}
+                  setPromptValue={setPromptValue}
+                  handleSubmit={handleChat}
+                  handleSecondary={handleDisplayResult}
+                  disabled={loading}
+                  reset={reset}
+                />
+              )}
             </div>
           </div>
         )}
diff --git a/frontend/nextjs/components/LoadingDots.tsx b/frontend/nextjs/components/LoadingDots.tsx
new file mode 100644
index 000000000..d385956c2
--- /dev/null
+++ b/frontend/nextjs/components/LoadingDots.tsx
@@ -0,0 +1,13 @@
+const LoadingDots = () => {
+  return (
+    <div className="flex justify-center py-4">
+      <div className="animate-pulse flex space-x-2">
+        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
+        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
+        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
+      </div>
+    </div>
+  );
+};
+
+export default LoadingDots; 
\ No newline at end of file

From 2db52ec50f3596b86f4584486e5fd11670526673 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 10:59:30 +0200
Subject: [PATCH 08/47] updated min height for better ux

---
 frontend/nextjs/app/page.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index c17f8df66..8c56afd12 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -425,7 +425,7 @@ export default function Home() {
         )}
 
         {showResult && (
-          <div className="flex h-full min-h-[68vh] w-full grow flex-col justify-between">
+          <div className="flex h-full w-full grow flex-col justify-between">
             <div className="container w-full space-y-2">
               <div className="container space-y-2 task-components">
                 {renderComponentsInOrder()}

From 4dae65ae34a3fcaec4ec697b2acb0df473ee5e8f Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 14:19:49 +0200
Subject: [PATCH 09/47] updated settings to preferences

---
 frontend/nextjs/components/Hero.tsx           | 2 +-
 frontend/nextjs/components/Settings/Modal.tsx | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index 997a45aff..2f4fcfdd2 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -19,7 +19,7 @@ const Hero: FC<THeroProps> = ({
 
   return (
     <div>
-      <div className="flex flex-col items-center justify-center mt-20">
+      <div className="flex flex-col items-center justify-center mt-10">
         <div className="landing flex flex-col items-center">
           <h1 className="text-4xl font-extrabold text-center lg:text-7xl">
             Say Goodbye to <br />
diff --git a/frontend/nextjs/components/Settings/Modal.tsx b/frontend/nextjs/components/Settings/Modal.tsx
index f4e37c535..bde682061 100644
--- a/frontend/nextjs/components/Settings/Modal.tsx
+++ b/frontend/nextjs/components/Settings/Modal.tsx
@@ -132,7 +132,7 @@ export default function Modal({ setChatBoxSettings, chatBoxSettings }: ChatBoxPr
         type="button"
         onClick={() => setShowModal(true)}
       >
-        Settings
+        Preferences
       </button>
       {showModal ? (
         <>

From 361d282fb9e02cbb2d3cc747e04e9eb33c5bd505 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 14:29:12 +0200
Subject: [PATCH 10/47] updated background color

---
 frontend/nextjs/app/globals.css       | 4 ++--
 frontend/nextjs/components/Footer.tsx | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/frontend/nextjs/app/globals.css b/frontend/nextjs/app/globals.css
index 7a0b0b158..8f699f64c 100644
--- a/frontend/nextjs/app/globals.css
+++ b/frontend/nextjs/app/globals.css
@@ -29,8 +29,8 @@ body {
   font-family: 'Montserrat', sans-serif;
   line-height: 1.6;
   background-size: 200% 200%;
-  background-image: linear-gradient(45deg, #151A2D, #111827, #151A2D);
-  animation: gradientBG 10s ease infinite;
+  background-image: linear-gradient(170deg, #151A2D, #036f73, #151A2D);
+  /*animation: gradientBG 10s ease infinite;*/
 }
 
 .landing {
diff --git a/frontend/nextjs/components/Footer.tsx b/frontend/nextjs/components/Footer.tsx
index 9bd451c9a..c30e8c8d5 100644
--- a/frontend/nextjs/components/Footer.tsx
+++ b/frontend/nextjs/components/Footer.tsx
@@ -19,7 +19,7 @@ const Footer = ({ setChatBoxSettings, chatBoxSettings}: ChatBoxProps) => {
     <>
       <div className="container flex min-h-[72px] mt-2 items-center justify-between border-t border-[#D2D2D2] px-4 pb-3 pt-5 lg:min-h-[72px] lg:px-0 lg:py-5">
         <Modal setChatBoxSettings={setChatBoxSettings} chatBoxSettings={chatBoxSettings} />
-        <div className="text-sm text-gray-500">
+        <div className="text-sm text-gray-300">
             © {new Date().getFullYear()} GPT Researcher. All rights reserved.
         </div>
         <div className="flex items-center gap-3">

From 719389c7e2ffefab9c401183d876fb18a1dba80c Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 16:38:09 +0200
Subject: [PATCH 11/47] updated card heights

---
 frontend/nextjs/components/InputArea.tsx       | 2 +-
 frontend/nextjs/components/LoadingDots.tsx     | 6 +++---
 frontend/nextjs/components/Question.tsx        | 2 +-
 frontend/nextjs/components/SubQuestions.tsx    | 4 ++--
 frontend/nextjs/components/Task/LogMessage.tsx | 4 ++--
 gpt_researcher/scraper/utils.py                | 8 +++++---
 6 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/frontend/nextjs/components/InputArea.tsx b/frontend/nextjs/components/InputArea.tsx
index caa1c7a56..8f75af2a5 100644
--- a/frontend/nextjs/components/InputArea.tsx
+++ b/frontend/nextjs/components/InputArea.tsx
@@ -33,7 +33,7 @@ const InputArea: FC<TInputAreaProps> = ({
   reset,
 }) => {
   const placeholder = handleSecondary
-    ? "Have any follow up questions?"
+    ? "Any questions about this report?"
     : "What would you like to research next?";
 
   const textareaRef = useRef<HTMLTextAreaElement>(null);
diff --git a/frontend/nextjs/components/LoadingDots.tsx b/frontend/nextjs/components/LoadingDots.tsx
index d385956c2..ce7efa2a8 100644
--- a/frontend/nextjs/components/LoadingDots.tsx
+++ b/frontend/nextjs/components/LoadingDots.tsx
@@ -2,9 +2,9 @@ const LoadingDots = () => {
   return (
     <div className="flex justify-center py-4">
       <div className="animate-pulse flex space-x-2">
-        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
-        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
-        <div className="w-2 h-2 bg-gray-500 rounded-full"></div>
+        <div className="w-2 h-2 bg-gray-300 rounded-full"></div>
+        <div className="w-2 h-2 bg-gray-300 rounded-full"></div>
+        <div className="w-2 h-2 bg-gray-300 rounded-full"></div>
       </div>
     </div>
   );
diff --git a/frontend/nextjs/components/Question.tsx b/frontend/nextjs/components/Question.tsx
index c54065ade..b31e13398 100644
--- a/frontend/nextjs/components/Question.tsx
+++ b/frontend/nextjs/components/Question.tsx
@@ -6,7 +6,7 @@ interface QuestionProps {
 
 const Question: React.FC<QuestionProps> = ({ question }) => {
   return (
-    <div className="container w-full flex flex-col sm:flex-row items-start gap-3 pt-2">
+    <div className="container w-full flex flex-col sm:flex-row items-start gap-3 pt-2 mb-2">
       <div className="flex items-center gap-2 sm:gap-4">
         <Image
           src={"/img/message-question-circle.svg"}
diff --git a/frontend/nextjs/components/SubQuestions.tsx b/frontend/nextjs/components/SubQuestions.tsx
index 23201823e..3c4bf66b0 100644
--- a/frontend/nextjs/components/SubQuestions.tsx
+++ b/frontend/nextjs/components/SubQuestions.tsx
@@ -7,7 +7,7 @@ interface SubQuestionsProps {
 
 const SubQuestions: React.FC<SubQuestionsProps> = ({ metadata, handleClickSuggestion }) => {
   return (
-    <div className="container flex w-full items-start gap-3 pt-2">
+    <div className="container flex w-full items-start gap-3 pt-5 pb-2">
       <div className="flex w-fit items-center gap-4">
         <Image
           src={"/img/thinking.svg"}
@@ -18,7 +18,7 @@ const SubQuestions: React.FC<SubQuestionsProps> = ({ metadata, handleClickSugges
         />
       </div>
       <div className="grow text-white">
-        <p className="pr-5 font-bold leading-[152%] text-white pb-[30px]">
+        <p className="pr-5 font-bold leading-[152%] text-white pb-[20px]">
           Pondering your question from several angles
         </p>
         <div className="flex flex-row flex-wrap items-center gap-2.5 pb-[20px]">
diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/Task/LogMessage.tsx
index 8a77caa72..e05fce710 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/Task/LogMessage.tsx
@@ -52,7 +52,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
   }, [logs]);
 
   return (
-          <div className="w-full px-4 log-message">
+          <div className="w-full log-message">
             {processedLogs.map((log, index) => {
               if (log.header === 'subquery_context_window' || log.header === 'differences') {
                 return <Accordion key={index} logs={[log]} />;
@@ -66,7 +66,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
                 return (
                   <div
                     key={index}
-                    className="mb-4 w-full max-w-4xl mx-auto rounded-lg p-4 bg-gray-800 shadow-md log-message"
+                    className="w-full max-w-4xl mx-auto rounded-lg pt-2 mt-3 pb-2 px-4 bg-gray-800 shadow-md log-message"
                   >
                     <p className="py-3 text-base leading-relaxed text-white dark:text-white log-message">
                       {log.text}
diff --git a/gpt_researcher/scraper/utils.py b/gpt_researcher/scraper/utils.py
index 41b7b7144..962404bc0 100644
--- a/gpt_researcher/scraper/utils.py
+++ b/gpt_researcher/scraper/utils.py
@@ -24,10 +24,12 @@ def get_relevant_images(soup: BeautifulSoup, url: str) -> list:
                     height = parse_dimension(img['height'])
                     if width and height:
                         if width >= 2000 and height >= 1000:
-                            score = 2  # Medium score (very large images)
+                            score = 3  # Medium score (very large images)
                         elif width >= 1600 or height >= 800:
-                            score = 1  # Lower score
-                        elif width >= 800 or height >= 400:
+                            score = 2  # Lower score
+                        elif width >= 800 or height >= 500:
+                            score = 1  # Lowest score
+                        elif width >= 600 or height >= 300:
                             score = 0  # Lowest score
                         else:
                             continue  # Skip small images

From 0e76ebf617571bf607ca570d036e42ba200808f7 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 17:04:02 +0200
Subject: [PATCH 12/47] finalized ratios between logs

---
 frontend/nextjs/components/Answer.tsx          | 10 +++++-----
 frontend/nextjs/components/Task/Accordion.tsx  |  6 +++---
 frontend/nextjs/components/Task/AgentLogs.tsx  |  2 +-
 frontend/nextjs/components/Task/LogMessage.tsx |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/frontend/nextjs/components/Answer.tsx b/frontend/nextjs/components/Answer.tsx
index a7bb1cb89..0aaf749ea 100644
--- a/frontend/nextjs/components/Answer.tsx
+++ b/frontend/nextjs/components/Answer.tsx
@@ -26,7 +26,7 @@ export default function Answer({ answer }: { answer: string }) {
   }, [answer]);
   
   return (
-    <div className="container flex h-auto w-full shrink-0 gap-4 bg-gray-800 shadow-md rounded-lg border border-solid border-[#C2C2C2] p-5">
+    <div className="container flex h-auto w-full shrink-0 gap-4 bg-gray-900 shadow-md rounded-lg border border-solid border-[#C2C2C2] p-5">
       <div className="w-full">
         <div className="flex items-center justify-between pb-3">
           {answer && (
@@ -82,7 +82,7 @@ export default function Answer({ answer }: { answer: string }) {
                     padding: 0;
                     /* Override existing styles for headings */
                     h1, h2, h3, h4, h5, h6 {
-                      font-size: 18px; font-weight: 400; padding: 0;
+                      font-size: 22px; font-weight: 400; padding: 0;
                       line-height: 1.2;
                     }
                     /* Optionally add more specific styling */
@@ -108,17 +108,17 @@ export default function Answer({ answer }: { answer: string }) {
                     ul {
                       list-style-type: none;
                       padding-left: 0;
-                      margin-bottom: 1em;
+                      margin-bottom: 0.2em;
                     }
                     ul > li {
-                      margin-bottom: 0.5em;
+                      margin-bottom: 0.2em;
                     }
                     ul > li > ul {
                       margin-left: 1em;
                       list-style-type: disc;
                     }
                     ul > li > ul > li {
-                      margin-bottom: 0.3em;
+                      margin-bottom: 0.2em;
                     }
                     ul > li > ul > li > ul {
                       margin-left: 1em;
diff --git a/frontend/nextjs/components/Task/Accordion.tsx b/frontend/nextjs/components/Task/Accordion.tsx
index a36782d95..b81c87bba 100644
--- a/frontend/nextjs/components/Task/Accordion.tsx
+++ b/frontend/nextjs/components/Task/Accordion.tsx
@@ -114,13 +114,13 @@ const Accordion: React.FC<AccordionProps> = ({ logs }) => {
   };
 
   return (
-    <div id="accordion-collapse" data-accordion="collapse" className="mb-4 bg-gray-800 rounded-lg">
+    <div id="accordion-collapse" data-accordion="collapse" className="mt-4 bg-gray-900 rounded-lg">
       {logs.map((log, index) => (
         <div key={index}>
           <h2 id={`accordion-collapse-heading-${index}`}>
             <button
               type="button"
-              className="flex items-center w-full p-5 font-medium rtl:text-right text-white rounded-t-xl focus:ring-4 focus:ring-gray-200 dark:focus:ring-gray-800 hover:bg-gray-700 hover:text-white gap-3"
+              className="flex items-center w-full p-5 font-medium rtl:text-right text-white rounded-t-xl gap-3"
               onClick={() => handleToggle(index)}
               aria-expanded={openIndex === index}
               aria-controls={`accordion-collapse-body-${index}`}
@@ -149,7 +149,7 @@ const Accordion: React.FC<AccordionProps> = ({ logs }) => {
             className={`${openIndex === index ? '' : 'hidden'}`}
             aria-labelledby={`accordion-collapse-heading-${index}`}
           >
-            <div className="p-5 border border-b-0 border-gray-200 dark:border-gray-700 dark:bg-gray-800 text-white">
+            <div className="p-5 border border-b-0 border-gray-900 dark:border-gray-900 dark:bg-gray-900 text-white">
               {renderLogContent(log)}
             </div>
           </div>
diff --git a/frontend/nextjs/components/Task/AgentLogs.tsx b/frontend/nextjs/components/Task/AgentLogs.tsx
index 021dec598..4075dc7f5 100644
--- a/frontend/nextjs/components/Task/AgentLogs.tsx
+++ b/frontend/nextjs/components/Task/AgentLogs.tsx
@@ -1,7 +1,7 @@
 export default function AgentLogs({agentLogs}:any){  
   const renderAgentLogs = (agentLogs:any)=>{
     return agentLogs && agentLogs.map((agentLog:any, index:number)=>{
-      return (<div key={index} className="agent_response">{agentLog.output}</div>)
+      return (<div key={index}>{agentLog.output}</div>)
     })
   }
 
diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/Task/LogMessage.tsx
index e05fce710..5356f9fa7 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/Task/LogMessage.tsx
@@ -66,7 +66,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
                 return (
                   <div
                     key={index}
-                    className="w-full max-w-4xl mx-auto rounded-lg pt-2 mt-3 pb-2 px-4 bg-gray-800 shadow-md log-message"
+                    className="w-full max-w-4xl mx-auto rounded-lg pt-2 mt-3 pb-2 px-4 bg-gray-900 shadow-md log-message"
                   >
                     <p className="py-3 text-base leading-relaxed text-white dark:text-white log-message">
                       {log.text}

From a6b7a7e8b9bb1deae9c5185eda2340856008b97d Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 17:35:13 +0200
Subject: [PATCH 13/47] modified font size of reports

---
 frontend/nextjs/components/Answer.tsx  |  6 ++++-
 frontend/nextjs/services/websocket.ts  | 20 ++++++++++++++++
 frontend/nextjs/types/data.ts          | 32 ++++++++++++++++++++++++++
 frontend/nextjs/utils/dataProcessor.ts | 32 ++++++++++++++++++++++++++
 gpt_researcher/scraper/utils.py        |  2 +-
 5 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 frontend/nextjs/services/websocket.ts
 create mode 100644 frontend/nextjs/types/data.ts
 create mode 100644 frontend/nextjs/utils/dataProcessor.ts

diff --git a/frontend/nextjs/components/Answer.tsx b/frontend/nextjs/components/Answer.tsx
index 0aaf749ea..6710f8387 100644
--- a/frontend/nextjs/components/Answer.tsx
+++ b/frontend/nextjs/components/Answer.tsx
@@ -102,13 +102,17 @@ export default function Answer({ answer }: { answer: string }) {
                       font-size: 1.2em;
                       color: white;
                     }
-                    /* Add more styles as needed */
+                    p {
+                      font-size: 18px;
+                      line-height: 1.5;
+                    }
 
                     /* Table of Contents Styling */
                     ul {
                       list-style-type: none;
                       padding-left: 0;
                       margin-bottom: 0.2em;
+                      font-size: 18px;
                     }
                     ul > li {
                       margin-bottom: 0.2em;
diff --git a/frontend/nextjs/services/websocket.ts b/frontend/nextjs/services/websocket.ts
new file mode 100644
index 000000000..7fb820609
--- /dev/null
+++ b/frontend/nextjs/services/websocket.ts
@@ -0,0 +1,20 @@
+export const createWebSocket = (onMessage: (data: any) => void, onOpen?: () => void) => {
+  if (typeof window === 'undefined') return null;
+
+  const { protocol, pathname, host } = window.location;
+  const wsHost = host.includes('localhost') ? 'localhost:8000' : host;
+  const ws_uri = `${protocol === 'https:' ? 'wss:' : 'ws:'}//${wsHost}${pathname}ws`;
+
+  const socket = new WebSocket(ws_uri);
+  
+  socket.onmessage = (event) => {
+    const data = JSON.parse(event.data);
+    onMessage(data);
+  };
+
+  if (onOpen) {
+    socket.onopen = onOpen;
+  }
+
+  return socket;
+}; 
\ No newline at end of file
diff --git a/frontend/nextjs/types/data.ts b/frontend/nextjs/types/data.ts
new file mode 100644
index 000000000..ed95c3f1d
--- /dev/null
+++ b/frontend/nextjs/types/data.ts
@@ -0,0 +1,32 @@
+export interface BaseData {
+  type: string;
+}
+
+export interface BasicData extends BaseData {
+  type: 'basic';
+  content: string;
+}
+
+export interface LanggraphButtonData extends BaseData {
+  type: 'langgraphButton';
+  link: string;
+}
+
+export interface DifferencesData extends BaseData {
+  type: 'differences';
+  content: string;
+  output: string;
+}
+
+export interface QuestionData extends BaseData {
+  type: 'question';
+  content: string;
+}
+
+export type Data = BasicData | LanggraphButtonData | DifferencesData | QuestionData;
+
+export interface ChatBoxSettings {
+  report_source: string;
+  report_type: string;
+  tone: string;
+} 
\ No newline at end of file
diff --git a/frontend/nextjs/utils/dataProcessor.ts b/frontend/nextjs/utils/dataProcessor.ts
new file mode 100644
index 000000000..7f398d072
--- /dev/null
+++ b/frontend/nextjs/utils/dataProcessor.ts
@@ -0,0 +1,32 @@
+import { Data } from '../types/data';
+
+export const preprocessOrderedData = (data: Data[]) => {
+  const groupedData: any[] = [];
+  let currentAccordionGroup: any = null;
+  let currentSourceGroup: any = null;
+  let currentReportGroup: any = null;
+  let finalReportGroup: any = null;
+  let sourceBlockEncountered = false;
+  let lastSubqueriesIndex = -1;
+
+  data.forEach((item: any) => {
+    const { type, content, metadata, output, link } = item;
+
+    if (type === 'report') {
+      if (!currentReportGroup) {
+        currentReportGroup = { type: 'reportBlock', content: '' };
+        groupedData.push(currentReportGroup);
+      }
+      currentReportGroup.content += output;
+    } else if (type === 'logs' && content === 'research_report') {
+      if (!finalReportGroup) {
+        finalReportGroup = { type: 'reportBlock', content: '' };
+        groupedData.push(finalReportGroup);
+      }
+      finalReportGroup.content += output.report;
+    } 
+    // ... rest of the preprocessing logic
+  });
+
+  return groupedData;
+}; 
\ No newline at end of file
diff --git a/gpt_researcher/scraper/utils.py b/gpt_researcher/scraper/utils.py
index 962404bc0..c1283d50c 100644
--- a/gpt_researcher/scraper/utils.py
+++ b/gpt_researcher/scraper/utils.py
@@ -29,7 +29,7 @@ def get_relevant_images(soup: BeautifulSoup, url: str) -> list:
                             score = 2  # Lower score
                         elif width >= 800 or height >= 500:
                             score = 1  # Lowest score
-                        elif width >= 600 or height >= 300:
+                        elif width >= 400 or height >= 200:
                             score = 0  # Lowest score
                         else:
                             continue  # Skip small images

From b7f7caba2ae919fccee123c9c6a1f5fd9a020402 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 17:39:58 +0200
Subject: [PATCH 14/47] modified research task title

---
 frontend/nextjs/components/Question.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/nextjs/components/Question.tsx b/frontend/nextjs/components/Question.tsx
index b31e13398..d148645c7 100644
--- a/frontend/nextjs/components/Question.tsx
+++ b/frontend/nextjs/components/Question.tsx
@@ -16,7 +16,7 @@ const Question: React.FC<QuestionProps> = ({ question }) => {
           className="w-6 h-6"
         />
         <p className="font-bold uppercase leading-[152%] text-white">
-          Question:
+          Research Task:
         </p><br/>
       </div>
       <div className="grow text-white break-words max-w-full log-message">&quot;{question}&quot;</div>

From 0375822686a5d99b1b3dd7296ea3f59095e256f4 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 18:15:04 +0200
Subject: [PATCH 15/47] fixed images

---
 frontend/nextjs/app/page.tsx                  | 56 +++++--------
 .../nextjs/components/Task/ImagesCarousel.jsx |  2 +-
 .../nextjs/components/Task/ImagesGrid.jsx     | 83 -------------------
 .../nextjs/components/Task/LogMessage.tsx     |  2 +-
 frontend/nextjs/services/websocket.ts         |  6 +-
 frontend/nextjs/types/data.ts                 |  7 +-
 6 files changed, 34 insertions(+), 122 deletions(-)
 delete mode 100644 frontend/nextjs/components/Task/ImagesGrid.jsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 8c56afd12..4a7a25b38 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -18,36 +18,18 @@ import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
 import HumanFeedback from "@/components/HumanFeedback";
 import LoadingDots from "@/components/LoadingDots";
-
-interface BaseData {
-  type: string;
-}
-
-interface BasicData extends BaseData {
-  type: 'basic';
-  content: string;
-}
-
-interface LanggraphButtonData extends BaseData {
-  type: 'langgraphButton';
-  link: string;
-}
-
-interface DifferencesData extends BaseData {
-  type: 'differences';
-  content: string;
-  output: string;
-}
-
-type Data = BasicData | LanggraphButtonData | DifferencesData;
-
+import { Data, ChatBoxSettings, QuestionData } from '../types/data';
 
 export default function Home() {
   const [promptValue, setPromptValue] = useState("");
   const [showResult, setShowResult] = useState(false);
   const [answer, setAnswer] = useState("");
   const [loading, setLoading] = useState(false);
-  const [chatBoxSettings, setChatBoxSettings] = useState({ report_source: 'web', report_type: 'research_report', tone: 'Objective' });
+  const [chatBoxSettings, setChatBoxSettings] = useState<ChatBoxSettings>({ 
+    report_source: 'web', 
+    report_type: 'research_report', 
+    tone: 'Objective' 
+  });
   const chatContainerRef = useRef<HTMLDivElement>(null);
 
   const [question, setQuestion] = useState("");
@@ -57,7 +39,7 @@ export default function Home() {
   const [orderedData, setOrderedData] = useState<Data[]>([]);
   const heartbeatInterval = useRef<number>();
   const [showHumanFeedback, setShowHumanFeedback] = useState(false);
-  const [questionForHuman, setQuestionForHuman] = useState(false);
+  const [questionForHuman, setQuestionForHuman] = useState<true | false>(false);
   
 
   useEffect(() => {
@@ -150,25 +132,31 @@ export default function Home() {
     setShowHumanFeedback(false);
   };
 
-  const handleChat = async (message : string) =>{
-    if(socket){
+  const handleChat = async (message: string) => {
+    if (socket) {
       setShowResult(true);
       setQuestion(message);
       setLoading(true);
       setPromptValue("");
-      setAnswer(""); // Reset answer for new query
-      setOrderedData((prevOrder) => [...prevOrder, { type: 'question', content: message }]);
-      const data : string =  "chat" + JSON.stringify({"message": message});
-      socket.send(data)
-    } 
-  }
+      setAnswer("");
+
+      const questionData: QuestionData = { 
+        type: 'question', 
+        content: message 
+      };
+      setOrderedData(prevOrder => [...prevOrder, questionData]);
+      
+      const data = `chat${JSON.stringify({ message })}`;
+      socket.send(data);
+    }
+  };
 
   const handleDisplayResult = async (newQuestion: string) => {
     setShowResult(true);
     setLoading(true);
     setQuestion(newQuestion);
     setPromptValue("");
-    setAnswer(""); // Reset answer for new query
+    setAnswer("");
 
     // Add the new question to orderedData
     setOrderedData((prevOrder:any) => [...prevOrder, { type: 'question', content: newQuestion }]);
diff --git a/frontend/nextjs/components/Task/ImagesCarousel.jsx b/frontend/nextjs/components/Task/ImagesCarousel.jsx
index a9801cea3..93a21b0fa 100644
--- a/frontend/nextjs/components/Task/ImagesCarousel.jsx
+++ b/frontend/nextjs/components/Task/ImagesCarousel.jsx
@@ -35,7 +35,7 @@ export default function ImagesGrid({ images }) {
     };
 
     useEffect(() => {
-        const imagesToHide = ['https://gptr.dev/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75',"https://tavily.com/_ipx/w_3840,q_75/%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp?url=%2F_next%2Fstatic%2Fmedia%2Fbg-pattern.5aa07776.webp&w=3840&q=75"]
+        const imagesToHide = []
         const filteredImages = images.filter((img) => !imagesToHide.includes(img));
         setValidImages(filteredImages);
     }, [images]);
diff --git a/frontend/nextjs/components/Task/ImagesGrid.jsx b/frontend/nextjs/components/Task/ImagesGrid.jsx
deleted file mode 100644
index f3dd24cf3..000000000
--- a/frontend/nextjs/components/Task/ImagesGrid.jsx
+++ /dev/null
@@ -1,83 +0,0 @@
-import React, { useState, useEffect } from 'react';
-import ImageModal from './ImageModal';
-
-const ImagesGrid = () => {
-    const [validImages, setValidImages] = useState([]);
-    const [selectedImage, setSelectedImage] = useState(null);
-    const [isModalOpen, setIsModalOpen] = useState(false);
-
-    useEffect(() => {
-        // Fetch valid images from the backend
-        fetchValidImages();
-    }, []);
-
-    const fetchValidImages = async () => {
-        try {
-            const response = await fetch('/api/images');
-            const data = await response.json();
-            setValidImages(data);
-        } catch (error) {
-            console.error('Error fetching valid images:', error);
-        }
-    };
-
-    const openModal = (image, index) => {
-        setSelectedImage(image);
-        setIsModalOpen(true);
-    };
-
-    const closeModal = () => {
-        setSelectedImage(null);
-        setIsModalOpen(false);
-    };
-
-    const nextImage = () => {
-        const currentIndex = validImages.indexOf(selectedImage);
-        const nextIndex = (currentIndex + 1) % validImages.length;
-        setSelectedImage(validImages[nextIndex]);
-    };
-
-    const prevImage = () => {
-        const currentIndex = validImages.indexOf(selectedImage);
-        const prevIndex = (currentIndex - 1 + validImages.length) % validImages.length;
-        setSelectedImage(validImages[prevIndex]);
-    };
-
-    const handleImageError = (image) => {
-        console.error(`Error loading image: ${image}`);
-    };
-
-    return (
-        <div className="w-full mt-5 mb-5">
-            <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
-                {validImages.map((image, index) => (
-                    <div 
-                        key={index} 
-                        className="relative pt-[100%]" // Creates a square aspect ratio container
-                    >
-                        <img
-                            src={image}
-                            alt={`Image ${index + 1}`}
-                            className="absolute top-0 left-0 w-full h-full object-contain bg-gray-100 rounded-lg cursor-pointer hover:opacity-90 transition-opacity"
-                            onClick={() => openModal(image, index)}
-                            onError={() => handleImageError(image)}
-                        />
-                    </div>
-                ))}
-            </div>
-
-            {selectedImage && (
-                <ImageModal
-                    imageSrc={selectedImage}
-                    isOpen={isModalOpen}
-                    onClose={closeModal}
-                    onNext={nextImage}
-                    onPrev={prevImage}
-                    className="max-h-[80vh] w-auto" // Constrain modal image height
-                />
-            )}
-        </div>
-    );
-};
-
-export default ImagesGrid; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/Task/LogMessage.tsx
index 5356f9fa7..b3002e760 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/Task/LogMessage.tsx
@@ -3,7 +3,7 @@ import Accordion from './Accordion';
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
-import ImagesCarousel from './ImagesGrid';
+import ImagesCarousel from './ImagesCarousel';
 
 type ProcessedData = {
   field: string;
diff --git a/frontend/nextjs/services/websocket.ts b/frontend/nextjs/services/websocket.ts
index 7fb820609..9bc2183ba 100644
--- a/frontend/nextjs/services/websocket.ts
+++ b/frontend/nextjs/services/websocket.ts
@@ -1,4 +1,6 @@
-export const createWebSocket = (onMessage: (data: any) => void, onOpen?: () => void) => {
+import { Data } from '../types/data';
+
+export const createWebSocket = (onMessage: (data: Data) => void, onOpen?: () => void) => {
   if (typeof window === 'undefined') return null;
 
   const { protocol, pathname, host } = window.location;
@@ -8,7 +10,7 @@ export const createWebSocket = (onMessage: (data: any) => void, onOpen?: () => v
   const socket = new WebSocket(ws_uri);
   
   socket.onmessage = (event) => {
-    const data = JSON.parse(event.data);
+    const data = JSON.parse(event.data) as Data;
     onMessage(data);
   };
 
diff --git a/frontend/nextjs/types/data.ts b/frontend/nextjs/types/data.ts
index ed95c3f1d..ece521865 100644
--- a/frontend/nextjs/types/data.ts
+++ b/frontend/nextjs/types/data.ts
@@ -23,7 +23,12 @@ export interface QuestionData extends BaseData {
   content: string;
 }
 
-export type Data = BasicData | LanggraphButtonData | DifferencesData | QuestionData;
+export interface ChatData extends BaseData {
+  type: 'chat';
+  content: string;
+}
+
+export type Data = BasicData | LanggraphButtonData | DifferencesData | QuestionData | ChatData;
 
 export interface ChatBoxSettings {
   report_source: string;

From f2efd4ae856d6b8dc64f921c3453a2f629ea87a2 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 18:16:24 +0200
Subject: [PATCH 16/47] fixed type error in images

---
 frontend/nextjs/components/Task/LogMessage.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/Task/LogMessage.tsx
index b3002e760..e26beb1df 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/Task/LogMessage.tsx
@@ -15,6 +15,7 @@ type Log = {
   header: string;
   text: string;
   processedData?: ProcessedData[];
+  metadata?: any;
 };
 
 interface LogMessageProps {

From a55c9c5e0518db74b39018f050c31a769923e2bd Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 18:20:35 +0200
Subject: [PATCH 17/47] cleaned a bit

---
 frontend/nextjs/app/page.tsx          | 12 -----
 frontend/nextjs/hooks/useWebSocket.ts | 77 +++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 12 deletions(-)
 create mode 100644 frontend/nextjs/hooks/useWebSocket.ts

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 4a7a25b38..a1760a80f 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -91,8 +91,6 @@ export default function Home() {
               setAnswer((prev:any) => prev + data.output);
             } else if (data.type === 'path') {
               setLoading(false);
-              // newSocket.close(); We do not want to close the connection since we are chatting
-              // setSocket(null);
             } else if (data.type === 'chat'){
               setLoading(false);
             }
@@ -104,11 +102,6 @@ export default function Home() {
           const { task, report_type, report_source, tone } = chatBoxSettings;
           let data = "start " + JSON.stringify({ task: promptValue, report_type, report_source, tone, headers });
           newSocket.send(data);
-
-          // Start sending heartbeat messages every 30 seconds
-          // heartbeatInterval.current = setInterval(() => {
-          //   newSocket.send(JSON.stringify({ type: 'ping' }));
-          // }, 30000);
         };
 
         newSocket.onclose = () => {
@@ -193,11 +186,6 @@ export default function Home() {
       }
     } else {
       startResearch(chatBoxSettings);
-
-      // await Promise.all([
-      //   handleSourcesAndAnswer(newQuestion),
-      //   handleSimilarQuestions(newQuestion),
-      // ]);
     }
   };
 
diff --git a/frontend/nextjs/hooks/useWebSocket.ts b/frontend/nextjs/hooks/useWebSocket.ts
new file mode 100644
index 000000000..8c4774891
--- /dev/null
+++ b/frontend/nextjs/hooks/useWebSocket.ts
@@ -0,0 +1,77 @@
+import { useState, useEffect, useCallback } from 'react';
+import { createWebSocket, sendWebSocketMessage } from '../services/websocket';
+import { Data, ChatBoxSettings } from '../types/data';
+
+export const useWebSocket = (onDataReceived: (data: Data) => void) => {
+  const [socket, setSocket] = useState<WebSocket | null>(null);
+  const [isConnected, setIsConnected] = useState(false);
+
+  useEffect(() => {
+    return () => {
+      if (socket) {
+        socket.close();
+      }
+    };
+  }, [socket]);
+
+  const connect = useCallback((onHumanFeedback?: (output: any) => void) => {
+    const newSocket = createWebSocket({
+      onMessage: onDataReceived,
+      onHumanFeedback,
+      onOpen: () => setIsConnected(true),
+    });
+    setSocket(newSocket);
+    return newSocket;
+  }, [onDataReceived]);
+
+  const startResearch = useCallback((prompt: string, settings: ChatBoxSettings) => {
+    const storedConfig = localStorage.getItem('apiVariables');
+    const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
+    
+    const headers = {
+      'retriever': apiVariables.RETRIEVER,
+      'langchain_api_key': apiVariables.LANGCHAIN_API_KEY,
+      'openai_api_key': apiVariables.OPENAI_API_KEY,
+      'tavily_api_key': apiVariables.TAVILY_API_KEY,
+      'google_api_key': apiVariables.GOOGLE_API_KEY,
+      'google_cx_key': apiVariables.GOOGLE_CX_KEY,
+      'bing_api_key': apiVariables.BING_API_KEY,
+      'searchapi_api_key': apiVariables.SEARCHAPI_API_KEY,
+      'serpapi_api_key': apiVariables.SERPAPI_API_KEY,
+      'serper_api_key': apiVariables.SERPER_API_KEY,
+      'searx_url': apiVariables.SEARX_URL
+    };
+
+    const { report_type, report_source, tone } = settings;
+    const message = "start " + JSON.stringify({ 
+      task: prompt, 
+      report_type, 
+      report_source, 
+      tone, 
+      headers 
+    });
+
+    sendWebSocketMessage(socket, message);
+  }, [socket]);
+
+  const sendChat = useCallback((message: string) => {
+    const data = `chat${JSON.stringify({ message })}`;
+    sendWebSocketMessage(socket, data);
+  }, [socket]);
+
+  const sendFeedback = useCallback((feedback: string | null) => {
+    sendWebSocketMessage(socket, JSON.stringify({ 
+      type: 'human_feedback', 
+      content: feedback 
+    }));
+  }, [socket]);
+
+  return {
+    socket,
+    isConnected,
+    connect,
+    startResearch,
+    sendChat,
+    sendFeedback
+  };
+}; 
\ No newline at end of file

From 16094b809a2b722d5b3a151fd3f4b45cc3b41170 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 18:22:58 +0200
Subject: [PATCH 18/47] removed irrelevant files

---
 frontend/nextjs/hooks/useWebSocket.ts  | 77 --------------------------
 frontend/nextjs/services/websocket.ts  | 22 --------
 frontend/nextjs/utils/dataProcessor.ts | 32 -----------
 3 files changed, 131 deletions(-)
 delete mode 100644 frontend/nextjs/hooks/useWebSocket.ts
 delete mode 100644 frontend/nextjs/services/websocket.ts
 delete mode 100644 frontend/nextjs/utils/dataProcessor.ts

diff --git a/frontend/nextjs/hooks/useWebSocket.ts b/frontend/nextjs/hooks/useWebSocket.ts
deleted file mode 100644
index 8c4774891..000000000
--- a/frontend/nextjs/hooks/useWebSocket.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-import { useState, useEffect, useCallback } from 'react';
-import { createWebSocket, sendWebSocketMessage } from '../services/websocket';
-import { Data, ChatBoxSettings } from '../types/data';
-
-export const useWebSocket = (onDataReceived: (data: Data) => void) => {
-  const [socket, setSocket] = useState<WebSocket | null>(null);
-  const [isConnected, setIsConnected] = useState(false);
-
-  useEffect(() => {
-    return () => {
-      if (socket) {
-        socket.close();
-      }
-    };
-  }, [socket]);
-
-  const connect = useCallback((onHumanFeedback?: (output: any) => void) => {
-    const newSocket = createWebSocket({
-      onMessage: onDataReceived,
-      onHumanFeedback,
-      onOpen: () => setIsConnected(true),
-    });
-    setSocket(newSocket);
-    return newSocket;
-  }, [onDataReceived]);
-
-  const startResearch = useCallback((prompt: string, settings: ChatBoxSettings) => {
-    const storedConfig = localStorage.getItem('apiVariables');
-    const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
-    
-    const headers = {
-      'retriever': apiVariables.RETRIEVER,
-      'langchain_api_key': apiVariables.LANGCHAIN_API_KEY,
-      'openai_api_key': apiVariables.OPENAI_API_KEY,
-      'tavily_api_key': apiVariables.TAVILY_API_KEY,
-      'google_api_key': apiVariables.GOOGLE_API_KEY,
-      'google_cx_key': apiVariables.GOOGLE_CX_KEY,
-      'bing_api_key': apiVariables.BING_API_KEY,
-      'searchapi_api_key': apiVariables.SEARCHAPI_API_KEY,
-      'serpapi_api_key': apiVariables.SERPAPI_API_KEY,
-      'serper_api_key': apiVariables.SERPER_API_KEY,
-      'searx_url': apiVariables.SEARX_URL
-    };
-
-    const { report_type, report_source, tone } = settings;
-    const message = "start " + JSON.stringify({ 
-      task: prompt, 
-      report_type, 
-      report_source, 
-      tone, 
-      headers 
-    });
-
-    sendWebSocketMessage(socket, message);
-  }, [socket]);
-
-  const sendChat = useCallback((message: string) => {
-    const data = `chat${JSON.stringify({ message })}`;
-    sendWebSocketMessage(socket, data);
-  }, [socket]);
-
-  const sendFeedback = useCallback((feedback: string | null) => {
-    sendWebSocketMessage(socket, JSON.stringify({ 
-      type: 'human_feedback', 
-      content: feedback 
-    }));
-  }, [socket]);
-
-  return {
-    socket,
-    isConnected,
-    connect,
-    startResearch,
-    sendChat,
-    sendFeedback
-  };
-}; 
\ No newline at end of file
diff --git a/frontend/nextjs/services/websocket.ts b/frontend/nextjs/services/websocket.ts
deleted file mode 100644
index 9bc2183ba..000000000
--- a/frontend/nextjs/services/websocket.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { Data } from '../types/data';
-
-export const createWebSocket = (onMessage: (data: Data) => void, onOpen?: () => void) => {
-  if (typeof window === 'undefined') return null;
-
-  const { protocol, pathname, host } = window.location;
-  const wsHost = host.includes('localhost') ? 'localhost:8000' : host;
-  const ws_uri = `${protocol === 'https:' ? 'wss:' : 'ws:'}//${wsHost}${pathname}ws`;
-
-  const socket = new WebSocket(ws_uri);
-  
-  socket.onmessage = (event) => {
-    const data = JSON.parse(event.data) as Data;
-    onMessage(data);
-  };
-
-  if (onOpen) {
-    socket.onopen = onOpen;
-  }
-
-  return socket;
-}; 
\ No newline at end of file
diff --git a/frontend/nextjs/utils/dataProcessor.ts b/frontend/nextjs/utils/dataProcessor.ts
deleted file mode 100644
index 7f398d072..000000000
--- a/frontend/nextjs/utils/dataProcessor.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import { Data } from '../types/data';
-
-export const preprocessOrderedData = (data: Data[]) => {
-  const groupedData: any[] = [];
-  let currentAccordionGroup: any = null;
-  let currentSourceGroup: any = null;
-  let currentReportGroup: any = null;
-  let finalReportGroup: any = null;
-  let sourceBlockEncountered = false;
-  let lastSubqueriesIndex = -1;
-
-  data.forEach((item: any) => {
-    const { type, content, metadata, output, link } = item;
-
-    if (type === 'report') {
-      if (!currentReportGroup) {
-        currentReportGroup = { type: 'reportBlock', content: '' };
-        groupedData.push(currentReportGroup);
-      }
-      currentReportGroup.content += output;
-    } else if (type === 'logs' && content === 'research_report') {
-      if (!finalReportGroup) {
-        finalReportGroup = { type: 'reportBlock', content: '' };
-        groupedData.push(finalReportGroup);
-      }
-      finalReportGroup.content += output.report;
-    } 
-    // ... rest of the preprocessing logic
-  });
-
-  return groupedData;
-}; 
\ No newline at end of file

From 86aac6b3a4da50c27815a93c53d61168fda44872 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 19:20:51 +0200
Subject: [PATCH 19/47] modified image sorting

---
 frontend/nextjs/app/page.tsx             | 4 +---
 frontend/nextjs/components/InputArea.tsx | 2 +-
 gpt_researcher/scraper/utils.py          | 5 +++--
 gpt_researcher/skills/browser.py         | 2 +-
 gpt_researcher/skills/researcher.py      | 2 +-
 5 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index a1760a80f..9c4bc2118 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -325,9 +325,7 @@ export default function Home() {
         }));
 
         return (
-           <div className="container mx-auto">
-              <LogMessage key={uniqueKey} logs={logs} />
-            </div>
+          <LogMessage key={uniqueKey} logs={logs} />
         )
 
       } else if (data.type === 'sourceBlock') {
diff --git a/frontend/nextjs/components/InputArea.tsx b/frontend/nextjs/components/InputArea.tsx
index 8f75af2a5..52b7c92a6 100644
--- a/frontend/nextjs/components/InputArea.tsx
+++ b/frontend/nextjs/components/InputArea.tsx
@@ -61,7 +61,7 @@ const InputArea: FC<TInputAreaProps> = ({
   };
 
   // Debounced version of the height adjustment function
-  const adjustHeight = debounce((target) => {
+  const adjustHeight = debounce((target: HTMLTextAreaElement) => {
     target.style.height = 'auto'; // Reset height to auto to allow shrinking
     target.style.height = `${target.scrollHeight}px`; // Adjust height
   }, 100); // Adjust the delay as needed
diff --git a/gpt_researcher/scraper/utils.py b/gpt_researcher/scraper/utils.py
index c1283d50c..44f579066 100644
--- a/gpt_researcher/scraper/utils.py
+++ b/gpt_researcher/scraper/utils.py
@@ -17,7 +17,7 @@ def get_relevant_images(soup: BeautifulSoup, url: str) -> list:
                 score = 0
                 # Check for relevant classes
                 if any(cls in img.get('class', []) for cls in ['header', 'featured', 'hero', 'thumbnail', 'main', 'content']):
-                    score = 3  # Higher score
+                    score = 4  # Higher score
                 # Check for size attributes
                 elif img.get('width') and img.get('height'):
                     width = parse_dimension(img['width'])
@@ -56,7 +56,8 @@ def parse_dimension(value: str) -> int:
         value = value[:-2]  # Remove 'px' suffix
     try:
         return int(value)  # Convert to float first to handle decimal values
-    except ValueError:
+    except ValueError as e:
+        print(f"Error parsing dimension value {value}: {e}")
         return None
 
 def extract_title(soup: BeautifulSoup) -> str:
diff --git a/gpt_researcher/skills/browser.py b/gpt_researcher/skills/browser.py
index d2e47b573..5e7fda4ed 100644
--- a/gpt_researcher/skills/browser.py
+++ b/gpt_researcher/skills/browser.py
@@ -31,7 +31,7 @@ async def browse_urls(self, urls: List[str]) -> List[Dict]:
 
         scraped_content, images = scrape_urls(urls, self.researcher.cfg)
         self.researcher.add_research_sources(scraped_content)
-        new_images = self.select_top_images(images, k=2)  # Select top 2 images
+        new_images = self.select_top_images(images, k=4)  # Select top 2 images
         self.researcher.add_research_images(new_images)
 
         if self.researcher.verbose:
diff --git a/gpt_researcher/skills/researcher.py b/gpt_researcher/skills/researcher.py
index 47aee379d..948f57329 100644
--- a/gpt_researcher/skills/researcher.py
+++ b/gpt_researcher/skills/researcher.py
@@ -318,7 +318,7 @@ async def plan_research(self, query):
         await stream_output(
             "logs",
             "planning_research",
-            f"🤔 Planning the research strategy and subtasks...",
+            f"🤔 Planning the research strategy and subtasks (this may take a minute)...",
             self.researcher.websocket,
         )
 

From 65ab9b3236cebe26e53ac5ebd181123231ef374b Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 19:55:00 +0200
Subject: [PATCH 20/47] fixed chat on report

---
 backend/chat/chat.py                | 6 ++++++
 backend/server/server_utils.py      | 1 +
 backend/server/websocket_manager.py | 2 ++
 frontend/nextjs/app/page.tsx        | 4 +---
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/backend/chat/chat.py b/backend/chat/chat.py
index 49c0cf739..95e7ddc7f 100644
--- a/backend/chat/chat.py
+++ b/backend/chat/chat.py
@@ -84,6 +84,12 @@ def _process_document(self, report):
 
     async def chat(self, message, websocket):
         """Chat with React Agent"""
+        message = f"""
+         This is a chat message between the user and you: GPT Researcher - the leading research agent. 
+         The chat is about a research reports that you created. Answer based on the given context and report.
+         Report: {self.report}
+         User Message: {message}
+        """
         inputs = {"messages": [("user", message)]}
         response = await self.graph.ainvoke(inputs, config=self.chat_config)
         ai_message = response["messages"][-1].content
diff --git a/backend/server/server_utils.py b/backend/server/server_utils.py
index 9f77223f5..77bc8aba3 100644
--- a/backend/server/server_utils.py
+++ b/backend/server/server_utils.py
@@ -40,6 +40,7 @@ async def handle_human_feedback(data: str):
 
 async def handle_chat(websocket, data: str, manager):
     json_data = json.loads(data[4:])
+    print(f"Received chat message: {json_data.get('message')}")
     await manager.chat(json_data.get("message"), websocket)
 
 async def generate_report_files(report: str, filename: str) -> Dict[str, str]:
diff --git a/backend/server/websocket_manager.py b/backend/server/websocket_manager.py
index 4bcb5016e..413374172 100644
--- a/backend/server/websocket_manager.py
+++ b/backend/server/websocket_manager.py
@@ -70,6 +70,8 @@ async def start_streaming(self, task, report_type, report_source, source_urls, t
 
     async def chat(self, message, websocket):
         """Chat with the agent based message diff"""
+        print("Chat message: ", message)
+        print("Chat agent: ", self.chat_agent)
         if self.chat_agent:
             await self.chat_agent.chat(message, websocket)
         else:
diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 9c4bc2118..54e918c43 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -89,9 +89,7 @@ export default function Home() {
 
             if (data.type === 'report') {
               setAnswer((prev:any) => prev + data.output);
-            } else if (data.type === 'path') {
-              setLoading(false);
-            } else if (data.type === 'chat'){
+            } else if (data.type === 'path' || data.type === 'chat') {
               setLoading(false);
             }
           }

From f9bca78621ab42e75603f0250161fc1103c6ef69 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 20:12:43 +0200
Subject: [PATCH 21/47] refactored structure

---
 frontend/nextjs/app/page.tsx                  | 12 +--
 frontend/nextjs/components/Hero.tsx           |  2 +-
 .../{ => ResearchBlocks}/Answer.tsx           |  6 ++
 .../{ => ResearchBlocks}/InputArea.tsx        |  2 +-
 .../{Task => ResearchBlocks}/LogMessage.tsx   |  4 +-
 .../{ => ResearchBlocks}/Question.tsx         |  0
 .../{ => ResearchBlocks}/SourceCard.tsx       |  0
 .../{ => ResearchBlocks}/Sources.tsx          |  0
 .../{ => ResearchBlocks}/SubQuestions.tsx     |  0
 frontend/nextjs/components/Search.tsx         | 76 -------------------
 10 files changed, 16 insertions(+), 86 deletions(-)
 rename frontend/nextjs/components/{ => ResearchBlocks}/Answer.tsx (95%)
 rename frontend/nextjs/components/{ => ResearchBlocks}/InputArea.tsx (98%)
 rename frontend/nextjs/components/{Task => ResearchBlocks}/LogMessage.tsx (96%)
 rename frontend/nextjs/components/{ => ResearchBlocks}/Question.tsx (100%)
 rename frontend/nextjs/components/{ => ResearchBlocks}/SourceCard.tsx (100%)
 rename frontend/nextjs/components/{ => ResearchBlocks}/Sources.tsx (100%)
 rename frontend/nextjs/components/{ => ResearchBlocks}/SubQuestions.tsx (100%)
 delete mode 100644 frontend/nextjs/components/Search.tsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 54e918c43..588649ba5 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -1,18 +1,18 @@
 "use client";
 
-import Answer from "@/components/Answer";
+import Answer from "@/components/ResearchBlocks/Answer";
 import Footer from "@/components/Footer";
 import Header from "@/components/Header";
 import Hero from "@/components/Hero";
-import InputArea from "@/components/InputArea";
+import InputArea from "@/components/ResearchBlocks/InputArea";
 
-import Sources from "@/components/Sources";
-import Question from "@/components/Question";
-import SubQuestions from "@/components/SubQuestions";
+import Sources from "@/components/ResearchBlocks/Sources";
+import Question from "@/components/ResearchBlocks/Question";
+import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
 import { useRef, useState, useEffect } from "react";
 import AccessReport from '../components/Task/AccessReport';
 import Accordion from '../components/Task/Accordion';
-import LogMessage from '../components/Task/LogMessage';
+import LogMessage from '../components/ResearchBlocks/LogMessage';
 
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index 2f4fcfdd2..1706ab4e9 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -1,6 +1,6 @@
 import Image from "next/image";
 import { FC } from "react";
-import InputArea from "./InputArea";
+import InputArea from "./ResearchBlocks/InputArea";
 
 type THeroProps = {
   promptValue: string;
diff --git a/frontend/nextjs/components/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
similarity index 95%
rename from frontend/nextjs/components/Answer.tsx
rename to frontend/nextjs/components/ResearchBlocks/Answer.tsx
index 6710f8387..fce721c0a 100644
--- a/frontend/nextjs/components/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -106,6 +106,12 @@ export default function Answer({ answer }: { answer: string }) {
                       font-size: 18px;
                       line-height: 1.5;
                     }
+                    a {
+                      color: rgb(168 85 247 / var(--tw-bg-opacity));
+                      text-decoration: underline;
+                      font-weight: 500;
+                      target: _blank;
+                    }
 
                     /* Table of Contents Styling */
                     ul {
diff --git a/frontend/nextjs/components/InputArea.tsx b/frontend/nextjs/components/ResearchBlocks/InputArea.tsx
similarity index 98%
rename from frontend/nextjs/components/InputArea.tsx
rename to frontend/nextjs/components/ResearchBlocks/InputArea.tsx
index 52b7c92a6..2df7fe50a 100644
--- a/frontend/nextjs/components/InputArea.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/InputArea.tsx
@@ -1,6 +1,6 @@
 import Image from "next/image";
 import { FC, useRef } from "react";
-import TypeAnimation from "./TypeAnimation";
+import TypeAnimation from "../TypeAnimation";
 
 type TInputAreaProps = {
   promptValue: string;
diff --git a/frontend/nextjs/components/Task/LogMessage.tsx b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
similarity index 96%
rename from frontend/nextjs/components/Task/LogMessage.tsx
rename to frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
index e26beb1df..620cda98c 100644
--- a/frontend/nextjs/components/Task/LogMessage.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
@@ -1,9 +1,9 @@
 // LogMessage.tsx
-import Accordion from './Accordion';
+import Accordion from '../Task/Accordion';
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
-import ImagesCarousel from './ImagesCarousel';
+import ImagesCarousel from '../Task/ImagesCarousel';
 
 type ProcessedData = {
   field: string;
diff --git a/frontend/nextjs/components/Question.tsx b/frontend/nextjs/components/ResearchBlocks/Question.tsx
similarity index 100%
rename from frontend/nextjs/components/Question.tsx
rename to frontend/nextjs/components/ResearchBlocks/Question.tsx
diff --git a/frontend/nextjs/components/SourceCard.tsx b/frontend/nextjs/components/ResearchBlocks/SourceCard.tsx
similarity index 100%
rename from frontend/nextjs/components/SourceCard.tsx
rename to frontend/nextjs/components/ResearchBlocks/SourceCard.tsx
diff --git a/frontend/nextjs/components/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
similarity index 100%
rename from frontend/nextjs/components/Sources.tsx
rename to frontend/nextjs/components/ResearchBlocks/Sources.tsx
diff --git a/frontend/nextjs/components/SubQuestions.tsx b/frontend/nextjs/components/ResearchBlocks/SubQuestions.tsx
similarity index 100%
rename from frontend/nextjs/components/SubQuestions.tsx
rename to frontend/nextjs/components/ResearchBlocks/SubQuestions.tsx
diff --git a/frontend/nextjs/components/Search.tsx b/frontend/nextjs/components/Search.tsx
deleted file mode 100644
index ac0c5f63b..000000000
--- a/frontend/nextjs/components/Search.tsx
+++ /dev/null
@@ -1,76 +0,0 @@
-import React, { useState, useEffect } from 'react';
-import ResearchForm from './Task/ResearchForm';
-import Report from './Task/Report';
-import AgentLogs from './Task/AgentLogs';
-import AccessReport from './Task/AccessReport';
-
-const Search = () => {
-  // State for chatBoxSettings
-  const [chatBoxSettings, setChatBoxSettings] = useState({
-    report_type: '',
-    report_source: '',
-    tone: '',
-  });
-
-  const [task, setTask] = useState<string>('');
-  const [agentLogs, setAgentLogs] = useState<any[]>([]);
-  const [report, setReport] = useState<string>('');
-  const [accessData, setAccessData] = useState<string>('');
-  const [socket, setSocket] = useState<WebSocket | null>(null);
-
-  useEffect(() => {
-    if (typeof window !== 'undefined') {
-      const { protocol, pathname } = window.location;
-      let { host } = window.location;
-      host = host.includes('localhost') ? 'localhost:8000' : host;
-      const ws_uri = `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}${pathname}ws`;
-
-      const newSocket = new WebSocket(ws_uri);
-      setSocket(newSocket);
-
-      newSocket.onmessage = (event) => {
-        const data = JSON.parse(event.data);
-        if (data.type === 'agentLogs') {
-          setAgentLogs((prevLogs) => [...prevLogs, data.output]);
-        } else if (data.type === 'report') {
-          setReport(data.output);
-        } else if (data.type === 'accessData') {
-          setAccessData(data.output);
-        }
-      };
-
-      return () => newSocket.close();
-    }
-  }, []);
-
-  const handleFormSubmit = (task: string, reportType: string, reportSource: string) => {
-    setTask(task);
-
-    // Update reportType and reportSource in the settings
-    setChatBoxSettings((prev) => ({
-      ...prev,
-      report_type: reportType,
-      report_source: reportSource,
-    }));
-
-    // Sending data to WebSocket server
-    const data = JSON.stringify({ task, report_type: reportType, report_source: reportSource });
-    if (socket) socket.send(`start ${data}`);
-  };
-
-  return (
-    <div>
-      {/* Pass chatBoxSettings and setChatBoxSettings to ResearchForm */}
-      <ResearchForm
-        onFormSubmit={handleFormSubmit}
-        chatBoxSettings={chatBoxSettings}
-        setChatBoxSettings={setChatBoxSettings}
-      />
-      <AgentLogs agentLogs={agentLogs} />
-      <Report report={report} />
-      <AccessReport accessData={accessData} report={undefined} />
-    </div>
-  );
-};
-
-export default Search;
\ No newline at end of file

From 7048cac014a00cef9cc8355af82d8dc56e93a713 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 20:23:35 +0200
Subject: [PATCH 22/47] updated header background with blur and fixed

---
 frontend/nextjs/app/page.tsx          |  2 +-
 frontend/nextjs/components/Header.tsx | 10 +++++-----
 frontend/nextjs/components/Hero.tsx   |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 588649ba5..fe719a8e8 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -387,7 +387,7 @@ export default function Home() {
   return (
     <>
       <Header />
-      <main className="min-h-[calc(100vh-64px)]">
+      <main className="min-h-[100vh] pt-[120px]">
         {!showResult && (
           <Hero
             promptValue={promptValue}
diff --git a/frontend/nextjs/components/Header.tsx b/frontend/nextjs/components/Header.tsx
index 2ba6caff9..42646445b 100644
--- a/frontend/nextjs/components/Header.tsx
+++ b/frontend/nextjs/components/Header.tsx
@@ -2,17 +2,17 @@ import Image from "next/image";
 
 const Header = () => {
   return (
-    <div className="container h-[60px] px-4 lg:h-[80px] lg:px-0 pt-10 mb-10">
-      <div className="grid h-full grid-cols-12">
-        <div className="col-span-5"></div>
-        <div className="col-span-2 flex items-center justify-center">
+    <div className="fixed top-0 left-0 right-0 z-50">
+      <div className="absolute inset-0 backdrop-blur-sm bg-gradient-to-b to-transparent"></div>
+      <div className="container relative h-[60px] px-4 lg:h-[80px] lg:px-0 pt-4 pb-4">
+        <div className="flex justify-center items-center h-full">
           <a href="/">
             <Image
               src="/img/gptr-logo.png"
               alt="logo"
               width={60}
               height={60}
-              className="lg:h-20 lg:w-20"
+              className="lg:h-16 lg:w-16"
             />
           </a>
         </div>
diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index 1706ab4e9..9eea1721c 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -19,7 +19,7 @@ const Hero: FC<THeroProps> = ({
 
   return (
     <div>
-      <div className="flex flex-col items-center justify-center mt-10">
+      <div className="flex flex-col items-center justify-center">
         <div className="landing flex flex-col items-center">
           <h1 className="text-4xl font-extrabold text-center lg:text-7xl">
             Say Goodbye to <br />

From 504af6fe2afa37763c2738f4147eb4e7ca4ed7ab Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 20:27:05 +0200
Subject: [PATCH 23/47] updated backend prompt for chat

---
 backend/chat/chat.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/chat/chat.py b/backend/chat/chat.py
index 95e7ddc7f..73e9783e2 100644
--- a/backend/chat/chat.py
+++ b/backend/chat/chat.py
@@ -85,7 +85,10 @@ def _process_document(self, report):
     async def chat(self, message, websocket):
         """Chat with React Agent"""
         message = f"""
-         This is a chat message between the user and you: GPT Researcher - the leading research agent. 
+         You are GPT Researcher, a autonomous research agent created by an open source community at https://github.com/assafelovic/gpt-researcher, homepage: https://gptr.dev. 
+         To learn more about GPT Researcher you can suggest to check out: docs.gptr.dev.
+         
+         This is a chat message between the user and you: GPT Researcher. 
          The chat is about a research reports that you created. Answer based on the given context and report.
          Report: {self.report}
          User Message: {message}

From cb0de6176bffcff41408544b82ca8c9a82a6d855 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 20:32:56 +0200
Subject: [PATCH 24/47] fixed markdown style

---
 frontend/nextjs/components/ResearchBlocks/Answer.tsx | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/frontend/nextjs/components/ResearchBlocks/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
index fce721c0a..a8fe03372 100644
--- a/frontend/nextjs/components/ResearchBlocks/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -114,14 +114,21 @@ export default function Answer({ answer }: { answer: string }) {
                     }
 
                     /* Table of Contents Styling */
-                    ul {
+                    ul, ol {
                       list-style-type: none;
                       padding-left: 0;
                       margin-bottom: 0.2em;
                       font-size: 18px;
                     }
+                    
+                    ol > li {
+                      padding: 0;
+                      margin: 0;
+                      line-height: 0;
+                    }
                     ul > li {
                       margin-bottom: 0.2em;
+                      line-height: 0.5em;
                     }
                     ul > li > ul {
                       margin-left: 1em;

From 5c0407170c30c3f57d958b4982f63875450203d0 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 20:43:49 +0200
Subject: [PATCH 25/47] removed logs

---
 backend/server/websocket_manager.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/backend/server/websocket_manager.py b/backend/server/websocket_manager.py
index 413374172..4bcb5016e 100644
--- a/backend/server/websocket_manager.py
+++ b/backend/server/websocket_manager.py
@@ -70,8 +70,6 @@ async def start_streaming(self, task, report_type, report_source, source_urls, t
 
     async def chat(self, message, websocket):
         """Chat with the agent based message diff"""
-        print("Chat message: ", message)
-        print("Chat agent: ", self.chat_agent)
         if self.chat_agent:
             await self.chat_agent.chat(message, websocket)
         else:

From b964d4b85359609eb18942f439ca779bcdd355a6 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 21:16:54 +0200
Subject: [PATCH 26/47] added block for all agent logs

---
 frontend/nextjs/app/globals.css               |  18 +++
 frontend/nextjs/app/page.tsx                  | 127 +++++++++---------
 .../components/ResearchBlocks/Answer.tsx      |   3 +-
 .../components/ResearchBlocks/LogMessage.tsx  |  85 ++++++------
 4 files changed, 130 insertions(+), 103 deletions(-)

diff --git a/frontend/nextjs/app/globals.css b/frontend/nextjs/app/globals.css
index 8f699f64c..257f25952 100644
--- a/frontend/nextjs/app/globals.css
+++ b/frontend/nextjs/app/globals.css
@@ -102,3 +102,21 @@ body {
 body {
   margin: 0px !important;
 }
+
+/* Add these styles for the scrollbar */
+.scrollbar-thin {
+  scrollbar-width: thin;
+}
+
+.scrollbar-thumb-gray-600::-webkit-scrollbar-thumb {
+  background-color: #4B5563;
+  border-radius: 6px;
+}
+
+.scrollbar-track-gray-300::-webkit-scrollbar-track {
+  background-color: #D1D5DB;
+}
+
+.scrollbar-thin::-webkit-scrollbar {
+  width: 6px;
+}
diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index fe719a8e8..94de8333d 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -19,6 +19,7 @@ import findDifferences from '../helpers/findDifferences';
 import HumanFeedback from "@/components/HumanFeedback";
 import LoadingDots from "@/components/LoadingDots";
 import { Data, ChatBoxSettings, QuestionData } from '../types/data';
+import Image from "next/image";
 
 export default function Home() {
   const [promptValue, setPromptValue] = useState("");
@@ -40,7 +41,7 @@ export default function Home() {
   const heartbeatInterval = useRef<number>();
   const [showHumanFeedback, setShowHumanFeedback] = useState(false);
   const [questionForHuman, setQuestionForHuman] = useState<true | false>(false);
-  
+  const [allLogs, setAllLogs] = useState<any[]>([]);
 
   useEffect(() => {
     if (chatContainerRef.current) {
@@ -307,81 +308,58 @@ export default function Home() {
     return groupedData;
   };
 
-  const renderComponentsInOrder = () => {
+  // Remove logs processing from renderComponentsInOrder and add this useEffect
+  useEffect(() => {
     const groupedData = preprocessOrderedData(orderedData);
-    console.log('orderedData in renderComponentsInOrder: ', groupedData)
-    let statusReports = ["agent_generated","starting_research","planning_research"]
-
-    return groupedData.map((data, index) => {
+    const statusReports = ["agent_generated", "starting_research", "planning_research"];
+    
+    const newLogs: any[] = [];
+    groupedData.forEach((data) => {
       if (data.type === 'accordionBlock') {
-        const uniqueKey = `accordionBlock-${index}`;
         const logs = data.items.map((item:any, subIndex:any) => ({
           header: item.content,
           text: item.output,
           metadata: item.metadata,
           key: `${item.type}-${item.content}-${subIndex}`,
         }));
+        newLogs.push(...logs);
+      } else if (statusReports.includes(data.content)) {
+        newLogs.push({
+          header: data.content,
+          text: data.output,
+          metadata: data.metadata,
+          key: `${data.type}-${data.content}`,
+        });
+      }
+    });
+    setAllLogs(newLogs);
+  }, [orderedData]); // Only run when orderedData changes
 
-        return (
-          <LogMessage key={uniqueKey} logs={logs} />
-        )
-
-      } else if (data.type === 'sourceBlock') {
-        const uniqueKey = `sourceBlock-${index}`;
-        return <Sources key={uniqueKey} sources={data.items}/>;
+  const renderComponentsInOrder = () => {
+    const groupedData = preprocessOrderedData(orderedData);
+    console.log('orderedData in renderComponentsInOrder: ', groupedData);
+    
+    return groupedData.map((data, index) => {
+      if (data.type === 'sourceBlock') {
+        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
       } else if (data.type === 'reportBlock') {
-        const uniqueKey = `reportBlock-${index}`;
-        return <Answer key={uniqueKey} answer={data.content} />;
+        return <Answer key={`reportBlock-${index}`} answer={data.content} />;
       } else if (data.type === 'langgraphButton') {
-        const uniqueKey = `langgraphButton-${index}`;
+        return null;
+      } else if (data.type === 'question') {
+        return <Question key={`question-${index}`} question={data.content} />;
+      } else if (data.type === 'chat') {
+        return <Answer key={`chat-${index}`} answer={data.content} />;
+      } else if (data.content === 'subqueries') {
         return (
-          <div key={uniqueKey}></div>
-          // <div key={uniqueKey} className="flex justify-center py-4">
-          //   <a
-          //     href={data.link}
-          //     target="_blank"
-          //     rel="noopener noreferrer"
-          //     className="px-4 py-2 text-white bg-[#0DB7ED] rounded"
-          //   >
-          //     View the full Langgraph logs here
-          //   </a>
-          // </div>
+          <SubQuestions
+            key={`subqueries-${index}`}
+            metadata={data.metadata}
+            handleClickSuggestion={handleClickSuggestion}
+          />
         );
-      } else if (data.type === 'question') {
-        const uniqueKey = `question-${index}`;
-        return <Question key={uniqueKey} question={data.content} />;
-      } else if (data.type === 'chat'){
-        const uniqueKey = `chat-${index}`;
-        return <Answer key={uniqueKey} answer={data.content} />;
-      } else {
-        const { type, content, metadata, output } = data;
-        const uniqueKey = `${type}-${content}-${index}`;
-
-        if (content === 'subqueries') {
-          return (
-            <SubQuestions
-              key={uniqueKey}
-              metadata={data.metadata}
-              handleClickSuggestion={handleClickSuggestion}
-            />
-          );
-        } else if (type === 'path') {
-          return <AccessReport key={uniqueKey} accessData={output} report={answer} />;
-        } else if (statusReports.includes(data.content)) {
-          const uniqueKey = `accordionBlock-${index}`;
-          const logs = [{
-            header: data.content,
-            text: data.output,
-            metadata: data.metadata,
-            key: `${data.type}-${data.content}`,
-          }]
-
-          return <LogMessage key={uniqueKey} logs={logs} />;
-        } else {
-          return null;
-        }
       }
-    });
+    }).filter(Boolean);
   };
 
   return (
@@ -400,7 +378,30 @@ export default function Home() {
           <div className="flex h-full w-full grow flex-col justify-between">
             <div className="container w-full space-y-2">
               <div className="container space-y-2 task-components">
-                {renderComponentsInOrder()}
+                {/* Show everything except the final answer first */}
+                {renderComponentsInOrder().filter(component => 
+                  !component?.props?.answer || component?.props?.answer?.length < 100
+                )}
+                
+                {/* Show logs */}
+                {orderedData.length > 0 && (
+                  <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
+                    <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
+                      <Image src="/img/white-books.svg" alt="logs" width={24} height={24} />
+                      <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
+                        logs:
+                      </h3>
+                    </div>
+                    <div className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+                      <LogMessage logs={allLogs} />
+                    </div>
+                  </div>
+                )}
+
+                {/* Show the final answer last */}
+                {renderComponentsInOrder().filter(component => 
+                  component?.props?.answer && component?.props?.answer?.length >= 100
+                )}
               </div>
 
               {showHumanFeedback && (
diff --git a/frontend/nextjs/components/ResearchBlocks/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
index a8fe03372..bfde24185 100644
--- a/frontend/nextjs/components/ResearchBlocks/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -124,11 +124,10 @@ export default function Answer({ answer }: { answer: string }) {
                     ol > li {
                       padding: 0;
                       margin: 0;
-                      line-height: 0;
                     }
                     ul > li {
                       margin-bottom: 0.2em;
-                      line-height: 0.5em;
+                      line-height: 1.2em;
                     }
                     ul > li > ul {
                       margin-left: 1em;
diff --git a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
index 620cda98c..b6a156ef3 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
@@ -4,6 +4,7 @@ import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
 import ImagesCarousel from '../Task/ImagesCarousel';
+import Image from "next/image";
 
 type ProcessedData = {
   field: string;
@@ -27,23 +28,30 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
 
   useEffect(() => {
     const processLogs = async () => {
+      if (!logs) return;
+      
       const newLogs = await Promise.all(
         logs.map(async (log) => {
-          if (log.header === 'differences') {
-            const data = JSON.parse(log.text).data;
-            const processedData = await Promise.all(
-              Object.keys(data).map(async (field) => {
-                const fieldValue = data[field].after || data[field].before;
-                if (!plainTextFields.includes(field)) {
-                  const htmlContent = await markdownToHtml(fieldValue);
-                  return { field, htmlContent, isMarkdown: true };
-                }
-                return { field, htmlContent: fieldValue, isMarkdown: false };
-              })
-            );
-            return { ...log, processedData };
+          try {
+            if (log.header === 'differences' && log.text) {
+              const data = JSON.parse(log.text).data;
+              const processedData = await Promise.all(
+                Object.keys(data).map(async (field) => {
+                  const fieldValue = data[field].after || data[field].before;
+                  if (!plainTextFields.includes(field)) {
+                    const htmlContent = await markdownToHtml(fieldValue);
+                    return { field, htmlContent, isMarkdown: true };
+                  }
+                  return { field, htmlContent: fieldValue, isMarkdown: false };
+                })
+              );
+              return { ...log, processedData };
+            }
+            return log;
+          } catch (error) {
+            console.error('Error processing log:', error);
+            return log;
           }
-          return log;
         })
       );
       setProcessedLogs(newLogs);
@@ -53,30 +61,31 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
   }, [logs]);
 
   return (
-          <div className="w-full log-message">
-            {processedLogs.map((log, index) => {
-              if (log.header === 'subquery_context_window' || log.header === 'differences') {
-                return <Accordion key={index} logs={[log]} />;
-              } else if(log.header === "selected_images") {
-                return (
-                  <ImagesCarousel
-                    images={log.metadata}
-                  />
-                )
-              } else if(log.header !== 'scraping_images') {
-                return (
-                  <div
-                    key={index}
-                    className="w-full max-w-4xl mx-auto rounded-lg pt-2 mt-3 pb-2 px-4 bg-gray-900 shadow-md log-message"
-                  >
-                    <p className="py-3 text-base leading-relaxed text-white dark:text-white log-message">
-                      {log.text}
-                    </p>
-                  </div>
-                );
-              }
-            })}
-          </div>
+    <>
+      {processedLogs.map((log, index) => {
+        if (log.header === 'subquery_context_window' || log.header === 'differences') {
+          return <Accordion key={index} logs={[log]} />;
+        } else if(log.header === "selected_images") {
+          return (
+            <ImagesCarousel
+              key={index}
+              images={log.metadata}
+            />
+          );
+        } else if(log.header !== 'scraping_images') {
+          return (
+            <div
+              key={index}
+              className="w-full max-w-4xl mx-auto rounded-lg pt-2 mt-3 pb-2 px-4 bg-gray-900 shadow-md"
+            >
+              <p className="py-3 text-base leading-relaxed text-white dark:text-white">
+                {log.text}
+              </p>
+            </div>
+          );
+        }
+      })}
+    </>
   );
 };
 

From 767db93161d7c18eaf3953bfb3de0d5c8b9423fe Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 21:41:15 +0200
Subject: [PATCH 27/47] restructured images component

---
 frontend/nextjs/app/page.tsx                  | 19 ++----------
 .../ImagesAlbum.jsx}                          |  0
 .../components/ResearchBlocks/LogMessage.tsx  |  4 +--
 .../components/ResearchBlocks/OrderedLogs.tsx | 31 +++++++++++++++++++
 4 files changed, 36 insertions(+), 18 deletions(-)
 rename frontend/nextjs/components/{Task/ImagesCarousel.jsx => ResearchBlocks/ImagesAlbum.jsx} (100%)
 create mode 100644 frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 94de8333d..f87d824b5 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -9,10 +9,8 @@ import InputArea from "@/components/ResearchBlocks/InputArea";
 import Sources from "@/components/ResearchBlocks/Sources";
 import Question from "@/components/ResearchBlocks/Question";
 import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
+import OrderedLogs from "@/components/ResearchBlocks/OrderedLogs";
 import { useRef, useState, useEffect } from "react";
-import AccessReport from '../components/Task/AccessReport';
-import Accordion from '../components/Task/Accordion';
-import LogMessage from '../components/ResearchBlocks/LogMessage';
 
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
@@ -378,24 +376,13 @@ export default function Home() {
           <div className="flex h-full w-full grow flex-col justify-between">
             <div className="container w-full space-y-2">
               <div className="container space-y-2 task-components">
-                {/* Show everything except the final answer first */}
                 {renderComponentsInOrder().filter(component => 
                   !component?.props?.answer || component?.props?.answer?.length < 100
                 )}
                 
-                {/* Show logs */}
+                {/* Replace logs section with new component */}
                 {orderedData.length > 0 && (
-                  <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
-                    <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
-                      <Image src="/img/white-books.svg" alt="logs" width={24} height={24} />
-                      <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
-                        logs:
-                      </h3>
-                    </div>
-                    <div className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
-                      <LogMessage logs={allLogs} />
-                    </div>
-                  </div>
+                  <OrderedLogs logs={allLogs} />
                 )}
 
                 {/* Show the final answer last */}
diff --git a/frontend/nextjs/components/Task/ImagesCarousel.jsx b/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
similarity index 100%
rename from frontend/nextjs/components/Task/ImagesCarousel.jsx
rename to frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
diff --git a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
index b6a156ef3..e91f616fa 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
@@ -3,7 +3,7 @@ import Accordion from '../Task/Accordion';
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
-import ImagesCarousel from '../Task/ImagesCarousel';
+import ImagesAlbum from './ImagesAlbum';
 import Image from "next/image";
 
 type ProcessedData = {
@@ -67,7 +67,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
           return <Accordion key={index} logs={[log]} />;
         } else if(log.header === "selected_images") {
           return (
-            <ImagesCarousel
+            <ImagesAlbum
               key={index}
               images={log.metadata}
             />
diff --git a/frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx b/frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx
new file mode 100644
index 000000000..7c63e4ad2
--- /dev/null
+++ b/frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx
@@ -0,0 +1,31 @@
+import Image from "next/image";
+import LogMessage from './LogMessage';
+
+interface Log {
+  header: string;
+  text: string;
+  metadata: any;
+  key: string;
+}
+
+interface OrderedLogsProps {
+  logs: Log[];
+}
+
+const OrderedLogs = ({ logs }: OrderedLogsProps) => {
+  return (
+    <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
+      <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
+        <Image src="/img/thinking.svg" alt="logs" width={24} height={24} />
+        <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
+          Agent Work:
+        </h3>
+      </div>
+      <div className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+        <LogMessage logs={logs} />
+      </div>
+    </div>
+  );
+};
+
+export default OrderedLogs; 
\ No newline at end of file

From ad591c50a23a6f6b2c63df8e0b5543d78ec0f269 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 21:47:50 +0200
Subject: [PATCH 28/47] updated images to its own block

---
 frontend/nextjs/app/page.tsx                  | 103 +++++++++++-------
 .../{Task => ResearchBlocks}/ImageModal.jsx   |   0
 .../components/ResearchBlocks/ImagesAlbum.jsx |  16 +--
 .../components/ResearchBlocks/LogMessage.tsx  |  10 +-
 4 files changed, 76 insertions(+), 53 deletions(-)
 rename frontend/nextjs/components/{Task => ResearchBlocks}/ImageModal.jsx (100%)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index f87d824b5..1a78d445b 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -10,6 +10,7 @@ import Sources from "@/components/ResearchBlocks/Sources";
 import Question from "@/components/ResearchBlocks/Question";
 import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
 import OrderedLogs from "@/components/ResearchBlocks/OrderedLogs";
+import ImagesAlbum from "@/components/ResearchBlocks/ImagesAlbum";
 import { useRef, useState, useEffect } from "react";
 
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
@@ -204,19 +205,21 @@ export default function Home() {
   };
 
   const preprocessOrderedData = (data:any) => {
-
     const groupedData: any[] = [];
     let currentAccordionGroup:any = null;
     let currentSourceGroup:any = null;
     let currentReportGroup:any = null;
     let finalReportGroup:any = null;
+    let currentImagesGroup:any = null;
     let sourceBlockEncountered = false;
     let lastSubqueriesIndex = -1;
-  
+
     data.forEach((item:any, index:any) => {
       const { type, content, metadata, output, link } = item;
-  
-      if (type === 'report') {
+
+      if (content === 'selected_images') {
+        groupedData.push({ type: 'imagesBlock', metadata });
+      } else if (type === 'report') {
         if (!currentReportGroup) {
           currentReportGroup = { type: 'reportBlock', content: '' };
           groupedData.push(currentReportGroup);
@@ -335,29 +338,65 @@ export default function Home() {
 
   const renderComponentsInOrder = () => {
     const groupedData = preprocessOrderedData(orderedData);
-    console.log('orderedData in renderComponentsInOrder: ', groupedData);
     
-    return groupedData.map((data, index) => {
-      if (data.type === 'sourceBlock') {
-        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
-      } else if (data.type === 'reportBlock') {
-        return <Answer key={`reportBlock-${index}`} answer={data.content} />;
-      } else if (data.type === 'langgraphButton') {
+    // Separate components into categories
+    const imageComponents = groupedData
+      .filter(data => data.type === 'imagesBlock')
+      .map((data, index) => (
+        <div key={`images-${index}`} className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
+          <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
+            <Image src="/img/image.svg" alt="images" width={24} height={24} />
+            <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
+              Selected Images:
+            </h3>
+          </div>
+          <div className="overflow-y-auto max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+            <ImagesAlbum images={data.metadata} />
+          </div>
+        </div>
+      ));
+
+    const reportComponents = groupedData
+      .filter(data => data.type === 'reportBlock')
+      .map((data, index) => (
+        <Answer key={`reportBlock-${index}`} answer={data.content} />
+      ));
+
+    const otherComponents = groupedData
+      .map((data, index) => {
+        if (data.type === 'sourceBlock') {
+          return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
+        } else if (data.type === 'question') {
+          return <Question key={`question-${index}`} question={data.content} />;
+        } else if (data.type === 'chat') {
+          return <Answer key={`chat-${index}`} answer={data.content} />;
+        } else if (data.content === 'subqueries') {
+          return (
+            <SubQuestions
+              key={`subqueries-${index}`}
+              metadata={data.metadata}
+              handleClickSuggestion={handleClickSuggestion}
+            />
+          );
+        }
         return null;
-      } else if (data.type === 'question') {
-        return <Question key={`question-${index}`} question={data.content} />;
-      } else if (data.type === 'chat') {
-        return <Answer key={`chat-${index}`} answer={data.content} />;
-      } else if (data.content === 'subqueries') {
-        return (
-          <SubQuestions
-            key={`subqueries-${index}`}
-            metadata={data.metadata}
-            handleClickSuggestion={handleClickSuggestion}
-          />
-        );
-      }
-    }).filter(Boolean);
+      }).filter(Boolean);
+
+    return (
+      <>
+        {/* Show initial components */}
+        {otherComponents}
+        
+        {/* Show logs section */}
+        {orderedData.length > 0 && <OrderedLogs logs={allLogs} />}
+        
+        {/* Show images if they exist */}
+        {imageComponents}
+        
+        {/* Show the report components last */}
+        {reportComponents}
+      </>
+    );
   };
 
   return (
@@ -376,19 +415,7 @@ export default function Home() {
           <div className="flex h-full w-full grow flex-col justify-between">
             <div className="container w-full space-y-2">
               <div className="container space-y-2 task-components">
-                {renderComponentsInOrder().filter(component => 
-                  !component?.props?.answer || component?.props?.answer?.length < 100
-                )}
-                
-                {/* Replace logs section with new component */}
-                {orderedData.length > 0 && (
-                  <OrderedLogs logs={allLogs} />
-                )}
-
-                {/* Show the final answer last */}
-                {renderComponentsInOrder().filter(component => 
-                  component?.props?.answer && component?.props?.answer?.length >= 100
-                )}
+                {renderComponentsInOrder()}
               </div>
 
               {showHumanFeedback && (
diff --git a/frontend/nextjs/components/Task/ImageModal.jsx b/frontend/nextjs/components/ResearchBlocks/ImageModal.jsx
similarity index 100%
rename from frontend/nextjs/components/Task/ImageModal.jsx
rename to frontend/nextjs/components/ResearchBlocks/ImageModal.jsx
diff --git a/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx b/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
index 93a21b0fa..bece3cf81 100644
--- a/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
+++ b/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
@@ -1,7 +1,7 @@
 import React, { useState, useEffect } from 'react';
-import ImageModal from './ImageModal'; // Import the ImageModal component
+import ImageModal from './ImageModal';
 
-export default function ImagesGrid({ images }) {
+export default function ImagesAlbum({ images }) {
     const [isModalOpen, setIsModalOpen] = useState(false);
     const [selectedImage, setSelectedImage] = useState(null);
     const [selectedIndex, setSelectedIndex] = useState(0);
@@ -40,18 +40,20 @@ export default function ImagesGrid({ images }) {
         setValidImages(filteredImages);
     }, [images]);
 
-    // Hide the entire component if there are no valid images
     if (validImages.length === 0) return null;
 
     return (
-        <div className="w-full mt-5 mb-5">
-            <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+        <div className="w-full h-full min-h-[200px] max-h-[400px]">
+            <div className="grid grid-cols-2 md:grid-cols-4 lg:grid-cols-4 gap-4 pb-4">
                 {validImages.map((image, index) => (
-                    <div key={index} className="relative aspect-square">
+                    <div 
+                        key={index} 
+                        className="relative aspect-square bg-gray-700 rounded-lg overflow-hidden hover:shadow-lg transition-shadow duration-300"
+                    >
                         <img
                             src={image}
                             alt={`Image ${index + 1}`}
-                            className="absolute inset-0 w-full h-full object-cover rounded-lg cursor-pointer hover:opacity-90 transition-opacity"
+                            className="absolute inset-0 w-full h-full object-cover cursor-pointer hover:opacity-90 transition-opacity duration-300"
                             onClick={() => openModal(image, index)}
                             onError={() => handleImageError(image)}
                         />
diff --git a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
index e91f616fa..eb88ab102 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
@@ -65,14 +65,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
       {processedLogs.map((log, index) => {
         if (log.header === 'subquery_context_window' || log.header === 'differences') {
           return <Accordion key={index} logs={[log]} />;
-        } else if(log.header === "selected_images") {
-          return (
-            <ImagesAlbum
-              key={index}
-              images={log.metadata}
-            />
-          );
-        } else if(log.header !== 'scraping_images') {
+        } else if (log.header !== 'selected_images' && log.header !== 'scraping_images') {
           return (
             <div
               key={index}
@@ -84,6 +77,7 @@ const LogMessage: React.FC<LogMessageProps> = ({ logs }) => {
             </div>
           );
         }
+        return null;
       })}
     </>
   );

From d6d1235c03139a3ba41b7fea539d67a8b817b804 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 21:49:28 +0200
Subject: [PATCH 29/47] added sources overflow

---
 .../components/ResearchBlocks/Sources.tsx     | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/frontend/nextjs/components/ResearchBlocks/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
index b715911db..63ed7022b 100644
--- a/frontend/nextjs/components/ResearchBlocks/Sources.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
@@ -14,21 +14,23 @@ export default function Sources({
           sources:{" "}
         </h3>
       </div>
-      <div className="flex w-full max-w-[890px] flex-wrap content-center items-center gap-[15px]">
-        {sources.length > 0 ? (
-          sources.map((source) => (
-            <SourceCard source={source} key={source.url} />
-          ))
-        ) : (
-          <>
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-            <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
-          </>
-        )}
+      <div className="overflow-y-auto max-h-[350px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+        <div className="flex w-full max-w-[890px] flex-wrap content-center items-center gap-[15px] pb-2">
+          {sources.length > 0 ? (
+            sources.map((source) => (
+              <SourceCard source={source} key={source.url} />
+            ))
+          ) : (
+            <>
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+              <div className="h-20 w-[260px] max-w-sm animate-pulse rounded-md bg-gray-300" />
+            </>
+          )}
+        </div>
       </div>
     </div>
   );

From 148fce011aac8fc8a73eedb2ea0993ffdd3f0517 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 21:56:13 +0200
Subject: [PATCH 30/47] refactored image component

---
 frontend/nextjs/app/page.tsx                  | 18 ++++----------
 .../ResearchBlocks/ImageSection.tsx           | 24 +++++++++++++++++++
 .../{OrderedLogs.tsx => LogsSection.tsx}      |  6 ++---
 .../components/ResearchBlocks/Sources.tsx     |  2 +-
 frontend/nextjs/public/img/image.svg          |  7 ++++++
 5 files changed, 39 insertions(+), 18 deletions(-)
 create mode 100644 frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
 rename frontend/nextjs/components/ResearchBlocks/{OrderedLogs.tsx => LogsSection.tsx} (88%)
 create mode 100644 frontend/nextjs/public/img/image.svg

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 1a78d445b..0d1fa2fb6 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -9,8 +9,8 @@ import InputArea from "@/components/ResearchBlocks/InputArea";
 import Sources from "@/components/ResearchBlocks/Sources";
 import Question from "@/components/ResearchBlocks/Question";
 import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
-import OrderedLogs from "@/components/ResearchBlocks/OrderedLogs";
-import ImagesAlbum from "@/components/ResearchBlocks/ImagesAlbum";
+import LogsSection from "@/components/ResearchBlocks/LogsSection";
+import ImageSection from "@/components/ResearchBlocks/ImageSection";
 import { useRef, useState, useEffect } from "react";
 
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
@@ -343,17 +343,7 @@ export default function Home() {
     const imageComponents = groupedData
       .filter(data => data.type === 'imagesBlock')
       .map((data, index) => (
-        <div key={`images-${index}`} className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
-          <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
-            <Image src="/img/image.svg" alt="images" width={24} height={24} />
-            <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
-              Selected Images:
-            </h3>
-          </div>
-          <div className="overflow-y-auto max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
-            <ImagesAlbum images={data.metadata} />
-          </div>
-        </div>
+        <ImageSection key={`images-${index}`} metadata={data.metadata} />
       ));
 
     const reportComponents = groupedData
@@ -388,7 +378,7 @@ export default function Home() {
         {otherComponents}
         
         {/* Show logs section */}
-        {orderedData.length > 0 && <OrderedLogs logs={allLogs} />}
+        {orderedData.length > 0 && <LogsSection logs={allLogs} />}
         
         {/* Show images if they exist */}
         {imageComponents}
diff --git a/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx b/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
new file mode 100644
index 000000000..1bbacd54b
--- /dev/null
+++ b/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
@@ -0,0 +1,24 @@
+import Image from "next/image";
+import ImagesAlbum from './ImagesAlbum';
+
+interface ImageSectionProps {
+  metadata: any;
+}
+
+const ImageSection = ({ metadata }: ImageSectionProps) => {
+  return (
+    <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
+      <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
+        <Image src="/img/image.svg" alt="images" width={24} height={24} />
+        <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
+          Related Images
+        </h3>
+      </div>
+      <div className="overflow-y-auto max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+        <ImagesAlbum images={metadata} />
+      </div>
+    </div>
+  );
+};
+
+export default ImageSection; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
similarity index 88%
rename from frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx
rename to frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
index 7c63e4ad2..83f0e405b 100644
--- a/frontend/nextjs/components/ResearchBlocks/OrderedLogs.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
@@ -12,13 +12,13 @@ interface OrderedLogsProps {
   logs: Log[];
 }
 
-const OrderedLogs = ({ logs }: OrderedLogsProps) => {
+const LogsSection = ({ logs }: OrderedLogsProps) => {
   return (
     <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
         <Image src="/img/thinking.svg" alt="logs" width={24} height={24} />
         <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
-          Agent Work:
+          Agent Work
         </h3>
       </div>
       <div className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
@@ -28,4 +28,4 @@ const OrderedLogs = ({ logs }: OrderedLogsProps) => {
   );
 };
 
-export default OrderedLogs; 
\ No newline at end of file
+export default LogsSection; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchBlocks/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
index 63ed7022b..14b447282 100644
--- a/frontend/nextjs/components/ResearchBlocks/Sources.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
@@ -11,7 +11,7 @@ export default function Sources({
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
         <Image src="/img/white-books.svg" alt="footer" width={24} height={24} />
         <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
-          sources:{" "}
+          sources{" "}
         </h3>
       </div>
       <div className="overflow-y-auto max-h-[350px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
diff --git a/frontend/nextjs/public/img/image.svg b/frontend/nextjs/public/img/image.svg
new file mode 100644
index 000000000..33f6fab74
--- /dev/null
+++ b/frontend/nextjs/public/img/image.svg
@@ -0,0 +1,7 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" stroke="#ffffff">
+
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
+
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
+
<g id="SVGRepo_iconCarrier"> <path d="M14.2639 15.9375L12.5958 14.2834C11.7909 13.4851 11.3884 13.086 10.9266 12.9401C10.5204 12.8118 10.0838 12.8165 9.68048 12.9536C9.22188 13.1095 8.82814 13.5172 8.04068 14.3326L4.04409 18.2801M14.2639 15.9375L14.6053 15.599C15.4112 14.7998 15.8141 14.4002 16.2765 14.2543C16.6831 14.126 17.12 14.1311 17.5236 14.2687C17.9824 14.4251 18.3761 14.8339 19.1634 15.6514L20 16.4934M14.2639 15.9375L18.275 19.9565M18.275 19.9565C17.9176 20 17.4543 20 16.8 20H7.2C6.07989 20 5.51984 20 5.09202 19.782C4.71569 19.5903 4.40973 19.2843 4.21799 18.908C4.12796 18.7313 4.07512 18.5321 4.04409 18.2801M18.275 19.9565C18.5293 19.9256 18.7301 19.8727 18.908 19.782C19.2843 19.5903 19.5903 19.2843 19.782 18.908C20 18.4802 20 17.9201 20 16.8V16.4934M4.04409 18.2801C4 17.9221 4 17.4575 4 16.8V7.2C4 6.0799 4 5.51984 4.21799 5.09202C4.40973 4.71569 4.71569 4.40973 5.09202 4.21799C5.51984 4 6.07989 4 7.2 4H16.8C17.9201 4 18.4802 4 18.908 4.21799C19.2843 4.40973 19.5903 4.71569 19.782 5.09202C20 5.51984 20 6.0799 20 7.2V16.4934M17 8.99989C17 10.1045 16.1046 10.9999 15 10.9999C13.8954 10.9999 13 10.1045 13 8.99989C13 7.89532 13.8954 6.99989 15 6.99989C16.1046 6.99989 17 7.89532 17 8.99989Z" stroke="#ffffff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/> </g>
+
</svg>
\ No newline at end of file

From b752b51c06f814a18d380d552d588c4c0798a555 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Fri, 1 Nov 2024 22:14:34 +0200
Subject: [PATCH 31/47] added access report back

---
 frontend/nextjs/app/page.tsx                     | 3 +++
 frontend/nextjs/components/Task/AccessReport.tsx | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 0d1fa2fb6..49b0536d7 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -11,6 +11,7 @@ import Question from "@/components/ResearchBlocks/Question";
 import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
 import LogsSection from "@/components/ResearchBlocks/LogsSection";
 import ImageSection from "@/components/ResearchBlocks/ImageSection";
+import AccessReport from "@/components/Task/AccessReport";
 import { useRef, useState, useEffect } from "react";
 
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
@@ -385,6 +386,8 @@ export default function Home() {
         
         {/* Show the report components last */}
         {reportComponents}
+
+        {<AccessReport accessData={orderedData} report={answer} />}
       </>
     );
   };
diff --git a/frontend/nextjs/components/Task/AccessReport.tsx b/frontend/nextjs/components/Task/AccessReport.tsx
index 080e5c91c..237f86985 100644
--- a/frontend/nextjs/components/Task/AccessReport.tsx
+++ b/frontend/nextjs/components/Task/AccessReport.tsx
@@ -37,6 +37,12 @@ const AccessReport: React.FC<AccessReportProps> = ({ accessData, report }) => {
         target="_blank">
         Download DocX
       </a>
+      <a id="downloadLink" 
+        href={""} 
+        className="bg-purple-500 text-white active:bg-purple-600 font-bold uppercase text-sm px-6 py-3 rounded shadow hover:shadow-lg outline-none focus:outline-none mr-1 mb-1 ease-linear transition-all duration-150"
+        target="_blank">
+        New Research
+      </a>
     </div>
   );
 }

From 6926de227c1518df474312dbe804c8b9e2fee2dc Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 07:42:55 +0200
Subject: [PATCH 32/47] fixed bug with options

---
 frontend/nextjs/app/page.tsx                          | 10 +++++++---
 frontend/nextjs/components/ResearchBlocks/Sources.tsx |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 49b0536d7..5bc52d7e8 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -353,6 +353,9 @@ export default function Home() {
         <Answer key={`reportBlock-${index}`} answer={data.content} />
       ));
 
+    // Find path data for AccessReport
+    const pathData = groupedData.find(data => data.type === 'path');
+
     const otherComponents = groupedData
       .map((data, index) => {
         if (data.type === 'sourceBlock') {
@@ -383,11 +386,12 @@ export default function Home() {
         
         {/* Show images if they exist */}
         {imageComponents}
-        
-        {/* Show the report components last */}
+
+        {/* Show the report components */}
         {reportComponents}
 
-        {<AccessReport accessData={orderedData} report={answer} />}
+        {/* Show AccessReport after all Answer components */}
+        {pathData && <AccessReport accessData={pathData.output} report={answer} />}
       </>
     );
   };
diff --git a/frontend/nextjs/components/ResearchBlocks/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
index 14b447282..b50f55052 100644
--- a/frontend/nextjs/components/ResearchBlocks/Sources.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
@@ -14,7 +14,7 @@ export default function Sources({
           sources{" "}
         </h3>
       </div>
-      <div className="overflow-y-auto max-h-[350px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+      <div className="overflow-y-auto max-h-[250px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
         <div className="flex w-full max-w-[890px] flex-wrap content-center items-center gap-[15px] pb-2">
           {sources.length > 0 ? (
             sources.map((source) => (

From 35d5eb433e3b5b57a53b1ddadc1487fa94f292f7 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 07:45:21 +0200
Subject: [PATCH 33/47] modified styles

---
 frontend/nextjs/components/ResearchBlocks/Answer.tsx | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frontend/nextjs/components/ResearchBlocks/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
index bfde24185..6d205e27f 100644
--- a/frontend/nextjs/components/ResearchBlocks/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -125,6 +125,10 @@ export default function Answer({ answer }: { answer: string }) {
                       padding: 0;
                       margin: 0;
                     }
+                    ol > li > p {
+                      padding: 0;
+                      margin: 0;
+                    }
                     ul > li {
                       margin-bottom: 0.2em;
                       line-height: 1.2em;

From 18f1d4da501cafdfa899316be08b58c6ac4b659c Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 08:01:29 +0200
Subject: [PATCH 34/47] refactored structure

---
 frontend/nextjs/app/page.tsx                  |  4 +-
 frontend/nextjs/components/Hero.tsx           |  2 +-
 .../{ResearchBlocks => Images}/ImageModal.jsx |  0
 .../ImagesAlbum.jsx                           |  0
 .../ResearchBlocks/ImageSection.tsx           |  2 +-
 .../components/ResearchBlocks/LogsSection.tsx |  2 +-
 .../ResearchBlocks/ResearchResults.tsx        | 66 +++++++++++++++++++
 .../components/ResearchBlocks/Sources.tsx     |  2 +-
 .../{ => elements}/InputArea.tsx              |  2 +-
 .../{ => elements}/LogMessage.tsx             |  4 +-
 .../{ => elements}/SourceCard.tsx             |  0
 .../{ => elements}/SubQuestions.tsx           |  0
 12 files changed, 75 insertions(+), 9 deletions(-)
 rename frontend/nextjs/components/{ResearchBlocks => Images}/ImageModal.jsx (100%)
 rename frontend/nextjs/components/{ResearchBlocks => Images}/ImagesAlbum.jsx (100%)
 create mode 100644 frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx
 rename frontend/nextjs/components/ResearchBlocks/{ => elements}/InputArea.tsx (98%)
 rename frontend/nextjs/components/ResearchBlocks/{ => elements}/LogMessage.tsx (96%)
 rename frontend/nextjs/components/ResearchBlocks/{ => elements}/SourceCard.tsx (100%)
 rename frontend/nextjs/components/ResearchBlocks/{ => elements}/SubQuestions.tsx (100%)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 5bc52d7e8..2e73f327c 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -4,11 +4,11 @@ import Answer from "@/components/ResearchBlocks/Answer";
 import Footer from "@/components/Footer";
 import Header from "@/components/Header";
 import Hero from "@/components/Hero";
-import InputArea from "@/components/ResearchBlocks/InputArea";
+import InputArea from "@/components/ResearchBlocks/elements/InputArea";
 
 import Sources from "@/components/ResearchBlocks/Sources";
 import Question from "@/components/ResearchBlocks/Question";
-import SubQuestions from "@/components/ResearchBlocks/SubQuestions";
+import SubQuestions from "@/components/ResearchBlocks/elements/SubQuestions";
 import LogsSection from "@/components/ResearchBlocks/LogsSection";
 import ImageSection from "@/components/ResearchBlocks/ImageSection";
 import AccessReport from "@/components/Task/AccessReport";
diff --git a/frontend/nextjs/components/Hero.tsx b/frontend/nextjs/components/Hero.tsx
index 9eea1721c..f6d81f1bd 100644
--- a/frontend/nextjs/components/Hero.tsx
+++ b/frontend/nextjs/components/Hero.tsx
@@ -1,6 +1,6 @@
 import Image from "next/image";
 import { FC } from "react";
-import InputArea from "./ResearchBlocks/InputArea";
+import InputArea from "./ResearchBlocks/elements/InputArea";
 
 type THeroProps = {
   promptValue: string;
diff --git a/frontend/nextjs/components/ResearchBlocks/ImageModal.jsx b/frontend/nextjs/components/Images/ImageModal.jsx
similarity index 100%
rename from frontend/nextjs/components/ResearchBlocks/ImageModal.jsx
rename to frontend/nextjs/components/Images/ImageModal.jsx
diff --git a/frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx b/frontend/nextjs/components/Images/ImagesAlbum.jsx
similarity index 100%
rename from frontend/nextjs/components/ResearchBlocks/ImagesAlbum.jsx
rename to frontend/nextjs/components/Images/ImagesAlbum.jsx
diff --git a/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx b/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
index 1bbacd54b..5c6123551 100644
--- a/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/ImageSection.tsx
@@ -1,5 +1,5 @@
 import Image from "next/image";
-import ImagesAlbum from './ImagesAlbum';
+import ImagesAlbum from '../Images/ImagesAlbum';
 
 interface ImageSectionProps {
   metadata: any;
diff --git a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
index 83f0e405b..76c555f4a 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
@@ -1,5 +1,5 @@
 import Image from "next/image";
-import LogMessage from './LogMessage';
+import LogMessage from './elements/LogMessage';
 
 interface Log {
   header: string;
diff --git a/frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx b/frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx
new file mode 100644
index 000000000..8ba58b40c
--- /dev/null
+++ b/frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx
@@ -0,0 +1,66 @@
+import React from 'react';
+import Answer from './Answer';
+import Sources from './Sources';
+import Question from './Question';
+import SubQuestions from './elements/SubQuestions';
+import LogsSection from './LogsSection';
+import ImageSection from './ImageSection';
+import AccessReport from '../Task/AccessReport';
+
+interface ResearchResultsProps {
+  groupedData: any[];
+  allLogs: any[];
+  answer: string;
+  handleClickSuggestion: (value: string) => void;
+}
+
+export const ResearchResults: React.FC<ResearchResultsProps> = ({
+  groupedData,
+  allLogs,
+  answer,
+  handleClickSuggestion,
+}) => {
+  const imageComponents = groupedData
+    .filter(data => data.type === 'imagesBlock')
+    .map((data, index) => (
+      <ImageSection key={`images-${index}`} metadata={data.metadata} />
+    ));
+
+  const reportComponents = groupedData
+    .filter(data => data.type === 'reportBlock')
+    .map((data, index) => (
+      <Answer key={`reportBlock-${index}`} answer={data.content} />
+    ));
+
+  const pathData = groupedData.find(data => data.type === 'path');
+
+  const otherComponents = groupedData
+    .map((data, index) => {
+      if (data.type === 'sourceBlock') {
+        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
+      } else if (data.type === 'question') {
+        return <Question key={`question-${index}`} question={data.content} />;
+      } else if (data.type === 'chat') {
+        return <Answer key={`chat-${index}`} answer={data.content} />;
+      } else if (data.content === 'subqueries') {
+        return (
+          <SubQuestions
+            key={`subqueries-${index}`}
+            metadata={data.metadata}
+            handleClickSuggestion={handleClickSuggestion}
+          />
+        );
+      }
+      return null;
+    }).filter(Boolean);
+
+  return (
+    <>
+      {otherComponents}
+      {groupedData.length > 0 && <LogsSection logs={allLogs} />}
+      {imageComponents}
+      {reportComponents}
+      {pathData && <AccessReport accessData={pathData.output} report={answer} />}
+    </>
+  );
+}; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchBlocks/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
index b50f55052..c2bf478e5 100644
--- a/frontend/nextjs/components/ResearchBlocks/Sources.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
@@ -1,5 +1,5 @@
 import Image from "next/image";
-import SourceCard from "./SourceCard";
+import SourceCard from "./elements/SourceCard";
 
 export default function Sources({
   sources,
diff --git a/frontend/nextjs/components/ResearchBlocks/InputArea.tsx b/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
similarity index 98%
rename from frontend/nextjs/components/ResearchBlocks/InputArea.tsx
rename to frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
index 2df7fe50a..58c26459d 100644
--- a/frontend/nextjs/components/ResearchBlocks/InputArea.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
@@ -1,6 +1,6 @@
 import Image from "next/image";
 import { FC, useRef } from "react";
-import TypeAnimation from "../TypeAnimation";
+import TypeAnimation from "../../TypeAnimation";
 
 type TInputAreaProps = {
   promptValue: string;
diff --git a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx b/frontend/nextjs/components/ResearchBlocks/elements/LogMessage.tsx
similarity index 96%
rename from frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
rename to frontend/nextjs/components/ResearchBlocks/elements/LogMessage.tsx
index eb88ab102..a291e5f85 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogMessage.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/elements/LogMessage.tsx
@@ -1,9 +1,9 @@
 // LogMessage.tsx
-import Accordion from '../Task/Accordion';
+import Accordion from '../../Task/Accordion';
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
-import ImagesAlbum from './ImagesAlbum';
+import ImagesAlbum from '../../Images/ImagesAlbum';
 import Image from "next/image";
 
 type ProcessedData = {
diff --git a/frontend/nextjs/components/ResearchBlocks/SourceCard.tsx b/frontend/nextjs/components/ResearchBlocks/elements/SourceCard.tsx
similarity index 100%
rename from frontend/nextjs/components/ResearchBlocks/SourceCard.tsx
rename to frontend/nextjs/components/ResearchBlocks/elements/SourceCard.tsx
diff --git a/frontend/nextjs/components/ResearchBlocks/SubQuestions.tsx b/frontend/nextjs/components/ResearchBlocks/elements/SubQuestions.tsx
similarity index 100%
rename from frontend/nextjs/components/ResearchBlocks/SubQuestions.tsx
rename to frontend/nextjs/components/ResearchBlocks/elements/SubQuestions.tsx

From ce9fe7f2cdd5146a6eba754df626e73e7ddf1232 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 08:51:58 +0200
Subject: [PATCH 35/47] fixed ordering of all

---
 frontend/nextjs/app/page.tsx                  | 108 +++----
 .../nextjs/components/Research/Research.tsx   |  78 +++++
 .../components/ResearchBlocks/Answer.tsx      | 271 +++++++++---------
 3 files changed, 273 insertions(+), 184 deletions(-)
 create mode 100644 frontend/nextjs/components/Research/Research.tsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 2e73f327c..dbe4effb1 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -43,12 +43,6 @@ export default function Home() {
   const [questionForHuman, setQuestionForHuman] = useState<true | false>(false);
   const [allLogs, setAllLogs] = useState<any[]>([]);
 
-  useEffect(() => {
-    if (chatContainerRef.current) {
-      chatContainerRef.current.scrollIntoView({ behavior: 'smooth' });
-    }
-  }, [orderedData]);
-
   const startResearch = (chatBoxSettings:any) => {
     const storedConfig = localStorage.getItem('apiVariables');
     const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
@@ -218,14 +212,16 @@ export default function Home() {
     data.forEach((item:any, index:any) => {
       const { type, content, metadata, output, link } = item;
 
-      if (content === 'selected_images') {
-        groupedData.push({ type: 'imagesBlock', metadata });
+      if (type === 'question') {
+        groupedData.push({ type: 'question', content });
       } else if (type === 'report') {
         if (!currentReportGroup) {
           currentReportGroup = { type: 'reportBlock', content: '' };
           groupedData.push(currentReportGroup);
         }
         currentReportGroup.content += output;
+      } else if (content === 'selected_images') {
+        groupedData.push({ type: 'imagesBlock', metadata });
       } else if (type === 'logs' && content === 'research_report') {
         if (!finalReportGroup) {
           finalReportGroup = { type: 'reportBlock', content: '' };
@@ -234,8 +230,6 @@ export default function Home() {
         finalReportGroup.content += output.report;
       } else if (type === 'langgraphButton') {
         groupedData.push({ type: 'langgraphButton', link });
-      } else if (type === 'question') {
-        groupedData.push({ type: 'question', content });
       } else if (type == 'chat'){
         groupedData.push({ type: 'chat', content: content });
       }
@@ -340,58 +334,76 @@ export default function Home() {
   const renderComponentsInOrder = () => {
     const groupedData = preprocessOrderedData(orderedData);
     
-    // Separate components into categories
-    const imageComponents = groupedData
-      .filter(data => data.type === 'imagesBlock')
-      .map((data, index) => (
-        <ImageSection key={`images-${index}`} metadata={data.metadata} />
-      ));
-
-    const reportComponents = groupedData
-      .filter(data => data.type === 'reportBlock')
-      .map((data, index) => (
-        <Answer key={`reportBlock-${index}`} answer={data.content} />
-      ));
-
     // Find path data for AccessReport
     const pathData = groupedData.find(data => data.type === 'path');
 
-    const otherComponents = groupedData
+    // Get the initial question (first question in the data)
+    const initialQuestion = groupedData
+      .find(data => data.type === 'question');
+
+    // Remove initial question from subsequent chat interactions
+    const chatComponents = groupedData
+      .filter(data => {
+        if (data.type === 'question' && data === initialQuestion) {
+          return false;
+        }
+        return (data.type === 'question' || data.type === 'chat');
+      })
       .map((data, index) => {
-        if (data.type === 'sourceBlock') {
-          return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
-        } else if (data.type === 'question') {
+        if (data.type === 'question') {
           return <Question key={`question-${index}`} question={data.content} />;
-        } else if (data.type === 'chat') {
+        } else {
           return <Answer key={`chat-${index}`} answer={data.content} />;
-        } else if (data.content === 'subqueries') {
-          return (
-            <SubQuestions
-              key={`subqueries-${index}`}
-              metadata={data.metadata}
-              handleClickSuggestion={handleClickSuggestion}
-            />
-          );
         }
-        return null;
-      }).filter(Boolean);
+      });
+
+    // Rest of the component separations remain the same
+    const sourceComponents = groupedData
+      .filter(data => data.type === 'sourceBlock')
+      .map((data, index) => (
+        <Sources key={`sourceBlock-${index}`} sources={data.items}/>
+      ));
+
+    const imageComponents = groupedData
+      .filter(data => data.type === 'imagesBlock')
+      .map((data, index) => (
+        <ImageSection key={`images-${index}`} metadata={data.metadata} />
+      ));
+
+    const initialReport = groupedData
+      .find(data => data.type === 'reportBlock');
+
+    const subqueriesComponent = groupedData
+      .find(data => data.content === 'subqueries');
 
     return (
       <>
-        {/* Show initial components */}
-        {otherComponents}
-        
-        {/* Show logs section */}
+        {/* Initial question */}
+        {initialQuestion && <Question question={initialQuestion.content} />}
+
+        {/* Subqueries if they exist */}
+        {subqueriesComponent && (
+          <SubQuestions
+            metadata={subqueriesComponent.metadata}
+            handleClickSuggestion={handleClickSuggestion}
+          />
+        )}
+
+        {/* 0. Agent work logs */}
         {orderedData.length > 0 && <LogsSection logs={allLogs} />}
-        
-        {/* Show images if they exist */}
-        {imageComponents}
 
-        {/* Show the report components */}
-        {reportComponents}
+        {/* 1. Sources */}
+        {sourceComponents}
+
+        {/* 2. Images */}
+        {imageComponents}
 
-        {/* Show AccessReport after all Answer components */}
+        {/* 3. Initial Answer Report */}
+        {initialReport && <Answer answer={initialReport.content} />}
         {pathData && <AccessReport accessData={pathData.output} report={answer} />}
+
+        {/* 4. All subsequent chat interactions */}
+        {chatComponents}
       </>
     );
   };
diff --git a/frontend/nextjs/components/Research/Research.tsx b/frontend/nextjs/components/Research/Research.tsx
new file mode 100644
index 000000000..2941fd927
--- /dev/null
+++ b/frontend/nextjs/components/Research/Research.tsx
@@ -0,0 +1,78 @@
+import { useRef } from 'react';
+import Answer from "../ResearchBlocks/Answer";
+import Sources from "../ResearchBlocks/Sources";
+import Question from "../ResearchBlocks/Question";
+import SubQuestions from "../ResearchBlocks/elements/SubQuestions";
+import LogsSection from "../ResearchBlocks/LogsSection";
+import ImageSection from "../ResearchBlocks/ImageSection";
+import AccessReport from "../Task/AccessReport";
+
+interface GroupedData {
+  type: string;
+  content?: string;
+  metadata?: any;
+  items?: any[];
+  output?: string;
+}
+
+interface ResearchProps {
+  orderedData: any[];
+  allLogs: any[];
+  answer: string;
+  handleClickSuggestion: (value: string) => void;
+  preprocessOrderedData: (data: any[]) => GroupedData[];
+}
+
+export const Research = ({ 
+  orderedData, 
+  allLogs, 
+  answer, 
+  handleClickSuggestion,
+  preprocessOrderedData 
+}: ResearchProps) => {
+  const groupedData = preprocessOrderedData(orderedData);
+  
+  const imageComponents = groupedData
+    .filter((data: GroupedData) => data.type === 'imagesBlock')
+    .map((data: GroupedData, index: number) => (
+      <ImageSection key={`images-${index}`} metadata={data.metadata} />
+    ));
+
+  const reportComponents = groupedData
+    .filter((data: GroupedData) => data.type === 'reportBlock')
+    .map((data: GroupedData, index: number) => (
+      <Answer key={`reportBlock-${index}`} answer={data.content || ''} />
+    ));
+
+  const pathData = groupedData.find((data: GroupedData) => data.type === 'path');
+
+  const otherComponents = groupedData
+    .map((data: GroupedData, index: number) => {
+      if (data.type === 'sourceBlock') {
+        return <Sources key={`sourceBlock-${index}`} sources={data.items || []}/>;
+      } else if (data.type === 'question') {
+        return <Question key={`question-${index}`} question={data.content || ''} />;
+      } else if (data.type === 'chat') {
+        return <Answer key={`chat-${index}`} answer={data.content || ''} />;
+      } else if (data.content === 'subqueries') {
+        return (
+          <SubQuestions
+            key={`subqueries-${index}`}
+            metadata={data.metadata}
+            handleClickSuggestion={handleClickSuggestion}
+          />
+        );
+      }
+      return null;
+    }).filter(Boolean);
+
+  return (
+    <>
+      {otherComponents}
+      {orderedData.length > 0 && <LogsSection logs={allLogs} />}
+      {imageComponents}
+      {reportComponents}
+      {pathData && <AccessReport accessData={pathData.output} report={answer} />}
+    </>
+  );
+}; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchBlocks/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
index 6d205e27f..5e8672077 100644
--- a/frontend/nextjs/components/ResearchBlocks/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -19,153 +19,152 @@ export default function Answer({ answer }: { answer: string }) {
     }
   }
 
-  const [htmlContent, setHtmlContent] = useState('');
+    const [htmlContent, setHtmlContent] = useState('');
 
-  useEffect(() => {
-    markdownToHtml(answer).then((html) => setHtmlContent(html));
-  }, [answer]);
-  
-  return (
-    <div className="container flex h-auto w-full shrink-0 gap-4 bg-gray-900 shadow-md rounded-lg border border-solid border-[#C2C2C2] p-5">
-      <div className="w-full">
-        <div className="flex items-center justify-between pb-3">
-          {answer && (
-            <div className="flex items-center gap-3">
-              {/* <Image
-                src="/img/link.svg"
-                alt="footer"
-                width={20}
-                height={20}
-                className="cursor-pointer"
-              /> */}
-              <button
-                onClick={() => {
-                  navigator.clipboard.writeText(answer.trim());
-                  toast("Answer copied to clipboard", {
-                    icon: "✂️",
-                  });
-                }}
-              >
-
-                <Image
-                  src="/img/copy-white.svg"
+    useEffect(() => {
+      markdownToHtml(answer).then((html) => setHtmlContent(html));
+    }, [answer]);
+    
+    return (
+      <div className="container flex h-auto w-full shrink-0 gap-4 bg-gray-900 shadow-md rounded-lg border border-solid border-[#C2C2C2] p-5">
+        <div className="w-full">
+          <div className="flex items-center justify-between pb-3">
+            {answer && (
+              <div className="flex items-center gap-3">
+                {/* <Image
+                  src="/img/link.svg"
                   alt="footer"
                   width={20}
                   height={20}
-                  className="cursor-pointer text-white"
-                />
-              </button>
-              {/* <Image
-                src="/img/share.svg"
-                alt="footer"
-                width={20}
-                height={20}
-                className="cursor-pointer"
-              /> */}
-            </div>
-          )}
-        </div>
-        <div className="flex flex-wrap content-center items-center gap-[15px]">
-          <div className="w-full whitespace-pre-wrap text-base font-light leading-[152.5%] text-white log-message">
-            {answer ? (
-              <div className="answer-container">
-                <div className="markdown-content" dangerouslySetInnerHTML={{ __html: htmlContent }} />
-                <style jsx>{`
-                .answer-container {
-                    font-family: Georgia, 'Times New Roman', Times, serif;
-                    
-                  }
+                  className="cursor-pointer"
+                /> */}
+                <button
+                  onClick={() => {
+                    navigator.clipboard.writeText(answer.trim());
+                    toast("Answer copied to clipboard", {
+                      icon: "✂️",
+                    });
+                  }}
+                >
 
-                  .markdown-content {
-                    /* Reset margins and paddings */
-                    margin: 0;
-                    padding: 0;
-                    /* Override existing styles for headings */
-                    h1, h2, h3, h4, h5, h6 {
-                      font-size: 22px; font-weight: 400; padding: 0;
-                      line-height: 1.2;
-                    }
-                    /* Optionally add more specific styling */
-                    h1 {
-                      font-size: 2.5em;
-                      color: white;
-                    }
-                    h2 {
-                      font-size: 2em;
-                      color: white;
-                    }
-                    h3 {
-                      font-size: 1.5em;
-                      color: white;
-                    }
-                    h4 {
-                      font-size: 1.2em;
-                      color: white;
-                    }
-                    p {
-                      font-size: 18px;
-                      line-height: 1.5;
-                    }
-                    a {
-                      color: rgb(168 85 247 / var(--tw-bg-opacity));
-                      text-decoration: underline;
-                      font-weight: 500;
-                      target: _blank;
+                  <Image
+                    src="/img/copy-white.svg"
+                    alt="footer"
+                    width={20}
+                    height={20}
+                    className="cursor-pointer text-white"
+                  />
+                </button>
+                {/* <Image
+                  src="/img/share.svg"
+                  alt="footer"
+                  width={20}
+                  height={20}
+                  className="cursor-pointer"
+                /> */}
+              </div>
+            )}
+          </div>
+          <div className="flex flex-wrap content-center items-center gap-[15px]">
+            <div className="w-full whitespace-pre-wrap text-base font-light leading-[152.5%] text-white log-message">
+              {answer ? (
+                <div className="answer-container">
+                  <div className="markdown-content" dangerouslySetInnerHTML={{ __html: htmlContent }} />
+                  <style jsx>{`
+                  .answer-container {
+                      font-family: Georgia, 'Times New Roman', Times, serif;
+                      
                     }
 
-                    /* Table of Contents Styling */
-                    ul, ol {
-                      list-style-type: none;
-                      padding-left: 0;
-                      margin-bottom: 0.2em;
-                      font-size: 18px;
-                    }
-                    
-                    ol > li {
-                      padding: 0;
+                    .markdown-content {
+                      /* Reset margins and paddings */
                       margin: 0;
-                    }
-                    ol > li > p {
                       padding: 0;
-                      margin: 0;
-                    }
-                    ul > li {
-                      margin-bottom: 0.2em;
-                      line-height: 1.2em;
-                    }
-                    ul > li > ul {
-                      margin-left: 1em;
-                      list-style-type: disc;
-                    }
-                    ul > li > ul > li {
-                      margin-bottom: 0.2em;
-                    }
-                    ul > li > ul > li > ul {
-                      margin-left: 1em;
-                      list-style-type: circle;
-                    }
-                    ul > li > ul > li > ul > li {
-                      margin-bottom: 0.2em;
+                      /* Override existing styles for headings */
+                      h1, h2, h3, h4, h5, h6 {
+                        font-size: 22px; font-weight: 400; padding: 0;
+                        line-height: 1.2;
+                      }
+                      /* Optionally add more specific styling */
+                      h1 {
+                        font-size: 2.5em;
+                        color: white;
+                      }
+                      h2 {
+                        font-size: 2em;
+                        color: white;
+                      }
+                      h3 {
+                        font-size: 1.5em;
+                        color: white;
+                      }
+                      h4 {
+                        font-size: 1.2em;
+                        color: white;
+                      }
+                      p {
+                        font-size: 18px;
+                        line-height: 1.5;
+                      }
+                      a {
+                        color: rgb(168 85 247 / var(--tw-bg-opacity));
+                        text-decoration: underline;
+                        font-weight: 500;
+                        target: _blank;
+                      }
+
+                      /* Table of Contents Styling */
+                      ul, ol {
+                        list-style-type: none;
+                        margin-bottom: 0.2em;
+                        font-size: 18px;
+                      }
+                      
+                      ol > li {
+                        padding: 0;
+                        margin: 0;
+                      }
+                      ol > li > p {
+                        padding: 0;
+                        margin: 0;
+                      }
+                      ul > li {
+                        padding: 0;
+                        margin: 0;
+                      }
+                      ul > li > ul {
+                        margin-left: 1em;
+                        list-style-type: disc;
+                      }
+                      ul > li > ul > li {
+                        margin-bottom: 0.2em;
+                      }
+                      ul > li > ul > li > ul {
+                        margin-left: 1em;
+                        list-style-type: circle;
+                      }
+                      ul > li > ul > li > ul > li {
+                        margin-bottom: 0.2em;
+                      }
                     }
-                  }
-                `}</style>
-              </div>
-            ) : (
-              <div className="flex w-full flex-col gap-2">
-                <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
-                <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
-                <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
-                <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
-              </div>
-            )}
+                  `}</style>
+                </div>
+              ) : (
+                <div className="flex w-full flex-col gap-2">
+                  <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
+                  <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
+                  <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
+                  <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
+                </div>
+              )}
+            </div>
           </div>
         </div>
+        <Toaster
+          position="top-center"
+          reverseOrder={false}
+          toastOptions={{ duration: 2000 }}
+        />
       </div>
-      <Toaster
-        position="top-center"
-        reverseOrder={false}
-        toastOptions={{ duration: 2000 }}
-      />
-    </div>
-  );
+    );
 }

From 7ea736167ffb845ee698275ea6b9df1eb1861a5b Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 08:58:03 +0200
Subject: [PATCH 36/47] finalized experience

---
 frontend/nextjs/app/page.tsx                  |  6 +-
 .../nextjs/components/Research/Research.tsx   | 78 -------------------
 .../components/ResearchBlocks/LogsSection.tsx | 15 +++-
 gpt_researcher/scraper/utils.py               |  2 +-
 4 files changed, 18 insertions(+), 83 deletions(-)
 delete mode 100644 frontend/nextjs/components/Research/Research.tsx

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index dbe4effb1..1230eb1a6 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -389,12 +389,12 @@ export default function Home() {
           />
         )}
 
-        {/* 0. Agent work logs */}
-        {orderedData.length > 0 && <LogsSection logs={allLogs} />}
-
         {/* 1. Sources */}
         {sourceComponents}
 
+        {/* 0. Agent work logs */}
+        {orderedData.length > 0 && <LogsSection logs={allLogs} />}
+
         {/* 2. Images */}
         {imageComponents}
 
diff --git a/frontend/nextjs/components/Research/Research.tsx b/frontend/nextjs/components/Research/Research.tsx
deleted file mode 100644
index 2941fd927..000000000
--- a/frontend/nextjs/components/Research/Research.tsx
+++ /dev/null
@@ -1,78 +0,0 @@
-import { useRef } from 'react';
-import Answer from "../ResearchBlocks/Answer";
-import Sources from "../ResearchBlocks/Sources";
-import Question from "../ResearchBlocks/Question";
-import SubQuestions from "../ResearchBlocks/elements/SubQuestions";
-import LogsSection from "../ResearchBlocks/LogsSection";
-import ImageSection from "../ResearchBlocks/ImageSection";
-import AccessReport from "../Task/AccessReport";
-
-interface GroupedData {
-  type: string;
-  content?: string;
-  metadata?: any;
-  items?: any[];
-  output?: string;
-}
-
-interface ResearchProps {
-  orderedData: any[];
-  allLogs: any[];
-  answer: string;
-  handleClickSuggestion: (value: string) => void;
-  preprocessOrderedData: (data: any[]) => GroupedData[];
-}
-
-export const Research = ({ 
-  orderedData, 
-  allLogs, 
-  answer, 
-  handleClickSuggestion,
-  preprocessOrderedData 
-}: ResearchProps) => {
-  const groupedData = preprocessOrderedData(orderedData);
-  
-  const imageComponents = groupedData
-    .filter((data: GroupedData) => data.type === 'imagesBlock')
-    .map((data: GroupedData, index: number) => (
-      <ImageSection key={`images-${index}`} metadata={data.metadata} />
-    ));
-
-  const reportComponents = groupedData
-    .filter((data: GroupedData) => data.type === 'reportBlock')
-    .map((data: GroupedData, index: number) => (
-      <Answer key={`reportBlock-${index}`} answer={data.content || ''} />
-    ));
-
-  const pathData = groupedData.find((data: GroupedData) => data.type === 'path');
-
-  const otherComponents = groupedData
-    .map((data: GroupedData, index: number) => {
-      if (data.type === 'sourceBlock') {
-        return <Sources key={`sourceBlock-${index}`} sources={data.items || []}/>;
-      } else if (data.type === 'question') {
-        return <Question key={`question-${index}`} question={data.content || ''} />;
-      } else if (data.type === 'chat') {
-        return <Answer key={`chat-${index}`} answer={data.content || ''} />;
-      } else if (data.content === 'subqueries') {
-        return (
-          <SubQuestions
-            key={`subqueries-${index}`}
-            metadata={data.metadata}
-            handleClickSuggestion={handleClickSuggestion}
-          />
-        );
-      }
-      return null;
-    }).filter(Boolean);
-
-  return (
-    <>
-      {otherComponents}
-      {orderedData.length > 0 && <LogsSection logs={allLogs} />}
-      {imageComponents}
-      {reportComponents}
-      {pathData && <AccessReport accessData={pathData.output} report={answer} />}
-    </>
-  );
-}; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
index 76c555f4a..7780c929a 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
@@ -1,5 +1,6 @@
 import Image from "next/image";
 import LogMessage from './elements/LogMessage';
+import { useEffect, useRef } from 'react';
 
 interface Log {
   header: string;
@@ -13,6 +14,15 @@ interface OrderedLogsProps {
 }
 
 const LogsSection = ({ logs }: OrderedLogsProps) => {
+  const logsContainerRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    // Scroll to bottom whenever logs change
+    if (logsContainerRef.current) {
+      logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight;
+    }
+  }, [logs]); // Dependency on logs array ensures this runs when new logs are added
+
   return (
     <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
@@ -21,7 +31,10 @@ const LogsSection = ({ logs }: OrderedLogsProps) => {
           Agent Work
         </h3>
       </div>
-      <div className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300">
+      <div 
+        ref={logsContainerRef}
+        className="overflow-y-auto min-h-[200px] max-h-[500px] scrollbar-thin scrollbar-thumb-gray-600 scrollbar-track-gray-300"
+      >
         <LogMessage logs={logs} />
       </div>
     </div>
diff --git a/gpt_researcher/scraper/utils.py b/gpt_researcher/scraper/utils.py
index 44f579066..35338a90e 100644
--- a/gpt_researcher/scraper/utils.py
+++ b/gpt_researcher/scraper/utils.py
@@ -29,7 +29,7 @@ def get_relevant_images(soup: BeautifulSoup, url: str) -> list:
                             score = 2  # Lower score
                         elif width >= 800 or height >= 500:
                             score = 1  # Lowest score
-                        elif width >= 400 or height >= 200:
+                        elif width >= 500 or height >= 300:
                             score = 0  # Lowest score
                         else:
                             continue  # Skip small images

From 29228fb83b6de7683a465424a019b75188df36ed Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 09:15:30 +0200
Subject: [PATCH 37/47] updated markdown

---
 .../components/ResearchBlocks/Answer.tsx      | 100 +------------
 frontend/nextjs/styles/markdown.css           | 131 ++++++++++++++++++
 2 files changed, 133 insertions(+), 98 deletions(-)
 create mode 100644 frontend/nextjs/styles/markdown.css

diff --git a/frontend/nextjs/components/ResearchBlocks/Answer.tsx b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
index 5e8672077..1a1a5ad15 100644
--- a/frontend/nextjs/components/ResearchBlocks/Answer.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Answer.tsx
@@ -1,11 +1,10 @@
 import Image from "next/image";
 import { Toaster, toast } from "react-hot-toast";
-
 import { useEffect, useState } from 'react';
 import { remark } from 'remark';
 import html from 'remark-html';
 import { Compatible } from "vfile";
-
+import '@/styles/markdown.css';
 
 export default function Answer({ answer }: { answer: string }) {
   async function markdownToHtml(markdown: Compatible | undefined) {
@@ -31,13 +30,6 @@ export default function Answer({ answer }: { answer: string }) {
           <div className="flex items-center justify-between pb-3">
             {answer && (
               <div className="flex items-center gap-3">
-                {/* <Image
-                  src="/img/link.svg"
-                  alt="footer"
-                  width={20}
-                  height={20}
-                  className="cursor-pointer"
-                /> */}
                 <button
                   onClick={() => {
                     navigator.clipboard.writeText(answer.trim());
@@ -46,7 +38,6 @@ export default function Answer({ answer }: { answer: string }) {
                     });
                   }}
                 >
-
                   <Image
                     src="/img/copy-white.svg"
                     alt="footer"
@@ -55,100 +46,13 @@ export default function Answer({ answer }: { answer: string }) {
                     className="cursor-pointer text-white"
                   />
                 </button>
-                {/* <Image
-                  src="/img/share.svg"
-                  alt="footer"
-                  width={20}
-                  height={20}
-                  className="cursor-pointer"
-                /> */}
               </div>
             )}
           </div>
           <div className="flex flex-wrap content-center items-center gap-[15px]">
             <div className="w-full whitespace-pre-wrap text-base font-light leading-[152.5%] text-white log-message">
               {answer ? (
-                <div className="answer-container">
-                  <div className="markdown-content" dangerouslySetInnerHTML={{ __html: htmlContent }} />
-                  <style jsx>{`
-                  .answer-container {
-                      font-family: Georgia, 'Times New Roman', Times, serif;
-                      
-                    }
-
-                    .markdown-content {
-                      /* Reset margins and paddings */
-                      margin: 0;
-                      padding: 0;
-                      /* Override existing styles for headings */
-                      h1, h2, h3, h4, h5, h6 {
-                        font-size: 22px; font-weight: 400; padding: 0;
-                        line-height: 1.2;
-                      }
-                      /* Optionally add more specific styling */
-                      h1 {
-                        font-size: 2.5em;
-                        color: white;
-                      }
-                      h2 {
-                        font-size: 2em;
-                        color: white;
-                      }
-                      h3 {
-                        font-size: 1.5em;
-                        color: white;
-                      }
-                      h4 {
-                        font-size: 1.2em;
-                        color: white;
-                      }
-                      p {
-                        font-size: 18px;
-                        line-height: 1.5;
-                      }
-                      a {
-                        color: rgb(168 85 247 / var(--tw-bg-opacity));
-                        text-decoration: underline;
-                        font-weight: 500;
-                        target: _blank;
-                      }
-
-                      /* Table of Contents Styling */
-                      ul, ol {
-                        list-style-type: none;
-                        margin-bottom: 0.2em;
-                        font-size: 18px;
-                      }
-                      
-                      ol > li {
-                        padding: 0;
-                        margin: 0;
-                      }
-                      ol > li > p {
-                        padding: 0;
-                        margin: 0;
-                      }
-                      ul > li {
-                        padding: 0;
-                        margin: 0;
-                      }
-                      ul > li > ul {
-                        margin-left: 1em;
-                        list-style-type: disc;
-                      }
-                      ul > li > ul > li {
-                        margin-bottom: 0.2em;
-                      }
-                      ul > li > ul > li > ul {
-                        margin-left: 1em;
-                        list-style-type: circle;
-                      }
-                      ul > li > ul > li > ul > li {
-                        margin-bottom: 0.2em;
-                      }
-                    }
-                  `}</style>
-                </div>
+                <div className="markdown-content" dangerouslySetInnerHTML={{ __html: htmlContent }} />
               ) : (
                 <div className="flex w-full flex-col gap-2">
                   <div className="h-6 w-full animate-pulse rounded-md bg-gray-300" />
diff --git a/frontend/nextjs/styles/markdown.css b/frontend/nextjs/styles/markdown.css
new file mode 100644
index 000000000..329fc50c0
--- /dev/null
+++ b/frontend/nextjs/styles/markdown.css
@@ -0,0 +1,131 @@
+.markdown-content {
+  /* Base styles */
+  color: white;
+  font-family: Georgia, 'Times New Roman', Times, serif;
+  font-size: 18px;
+  line-height: 1.6;
+
+  /* Headings */
+  h1, h2, h3, h4, h5, h6 {
+    line-height: 1.2;
+    font-weight: 500;
+  }
+
+  h1 { font-size: 2.5em; }
+  h2 { font-size: 2em; }
+  h3 { font-size: 1.5em; }
+  h4 { font-size: 1.2em; }
+  h5 { font-size: 1.1em; }
+  h6 { font-size: 1em; }
+
+  /* Paragraphs and spacing */
+  p {
+    font-size: 18px;
+    line-height: 1.6;
+  }
+
+  /* Lists */
+  ul, ol {
+    margin: 0 0 0 2em;
+    padding: 0;
+  }
+
+  ul {
+    list-style-type: disc;
+  }
+
+  ol {
+    list-style-type: decimal;
+  }
+
+  li {
+    display: list-item;
+    
+    > ul, > ol {
+      margin: 0 0 0 1.5em;
+    }
+  }
+
+  /* Numbered lists specific styling */
+  ol {
+    counter-reset: item;
+    list-style-position: outside;
+    
+    > li {
+      counter-increment: item;
+      
+      &::marker {
+        color: white;
+      }
+    }
+  }
+
+  /* Nested lists with proper numbering */
+  ol ol {
+    counter-reset: subitem;
+    
+    > li {
+      counter-increment: subitem;
+    }
+  }
+
+  /* Links */
+  a {
+    color: rgb(168 85 247);
+    text-decoration: underline;
+    font-weight: 500;
+    
+    &:hover {
+      opacity: 0.8;
+    }
+  }
+
+  /* Code blocks */
+  pre {
+    background-color: #1e1e1e;
+    padding: 1em;
+    border-radius: 4px;
+    overflow-x: auto;
+    margin: 1em 0;
+  }
+
+  code {
+    font-family: 'Courier New', Courier, monospace;
+    font-size: 0.9em;
+    padding: 0 0.4em;
+    background-color: #1e1e1e;
+    border-radius: 3px;
+  }
+
+  /* Blockquotes */
+  blockquote {
+    border-left: 4px solid rgb(168 85 247);
+    margin: 0;
+    padding-left: 1em;
+    font-style: italic;
+  }
+
+  /* Tables */
+  table {
+    border-collapse: collapse;
+    width: 100%;
+    margin: 1em 0;
+  }
+
+  th, td {
+    border: 1px solid #444;
+    padding: 0.5em;
+    text-align: left;
+  }
+
+  th {
+    background-color: #333;
+  }
+
+  /* Horizontal rule */
+  hr {
+    border: 0;
+    border-top: 1px solid #444;
+    margin: 2em 0;
+  }
+} 
\ No newline at end of file

From c9f9b6a261b5ed76d9fe11da1a62fd7c9611e2f2 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 13:23:35 +0200
Subject: [PATCH 38/47] refactored page.tsx to be more modular

---
 frontend/nextjs/app/page.tsx                  | 342 ++----------------
 .../nextjs/components/ResearchResults.tsx     |  76 ++++
 frontend/nextjs/hooks/useWebSocket.ts         |  69 ++++
 frontend/nextjs/styles/markdown.css           |   5 +
 frontend/nextjs/utils/dataProcessing.ts       | 103 ++++++
 5 files changed, 291 insertions(+), 304 deletions(-)
 create mode 100644 frontend/nextjs/components/ResearchResults.tsx
 create mode 100644 frontend/nextjs/hooks/useWebSocket.ts
 create mode 100644 frontend/nextjs/utils/dataProcessing.ts

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 1230eb1a6..90dc3f065 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -1,25 +1,19 @@
 "use client";
 
-import Answer from "@/components/ResearchBlocks/Answer";
-import Footer from "@/components/Footer";
-import Header from "@/components/Header";
-import Hero from "@/components/Hero";
-import InputArea from "@/components/ResearchBlocks/elements/InputArea";
-
-import Sources from "@/components/ResearchBlocks/Sources";
-import Question from "@/components/ResearchBlocks/Question";
-import SubQuestions from "@/components/ResearchBlocks/elements/SubQuestions";
-import LogsSection from "@/components/ResearchBlocks/LogsSection";
-import ImageSection from "@/components/ResearchBlocks/ImageSection";
-import AccessReport from "@/components/Task/AccessReport";
 import { useRef, useState, useEffect } from "react";
-
+import { useWebSocket } from '@/hooks/useWebSocket';
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
+import { Data, ChatBoxSettings, QuestionData } from '../types/data';
+import { preprocessOrderedData } from '../utils/dataProcessing';
+import { ResearchResults } from '../components/ResearchResults';
+
+import Header from "@/components/Header";
+import Hero from "@/components/Hero";
+import Footer from "@/components/Footer";
+import InputArea from "@/components/ResearchBlocks/elements/InputArea";
 import HumanFeedback from "@/components/HumanFeedback";
 import LoadingDots from "@/components/LoadingDots";
-import { Data, ChatBoxSettings, QuestionData } from '../types/data';
-import Image from "next/image";
 
 export default function Home() {
   const [promptValue, setPromptValue] = useState("");
@@ -31,87 +25,22 @@ export default function Home() {
     report_type: 'research_report', 
     tone: 'Objective' 
   });
-  const chatContainerRef = useRef<HTMLDivElement>(null);
-
   const [question, setQuestion] = useState("");
-  const [sources, setSources] = useState<{ name: string; url: string }[]>([]);
-  const [similarQuestions, setSimilarQuestions] = useState<string[]>([]);
-  const [socket, setSocket] = useState<WebSocket | null>(null);
   const [orderedData, setOrderedData] = useState<Data[]>([]);
-  const heartbeatInterval = useRef<number>();
   const [showHumanFeedback, setShowHumanFeedback] = useState(false);
   const [questionForHuman, setQuestionForHuman] = useState<true | false>(false);
   const [allLogs, setAllLogs] = useState<any[]>([]);
+  const chatContainerRef = useRef<HTMLDivElement>(null);
 
-  const startResearch = (chatBoxSettings:any) => {
-    const storedConfig = localStorage.getItem('apiVariables');
-    const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
-    const headers = {
-      'retriever': apiVariables.RETRIEVER,
-      'langchain_api_key': apiVariables.LANGCHAIN_API_KEY,
-      'openai_api_key': apiVariables.OPENAI_API_KEY,
-      'tavily_api_key': apiVariables.TAVILY_API_KEY,
-      'google_api_key': apiVariables.GOOGLE_API_KEY,
-      'google_cx_key': apiVariables.GOOGLE_CX_KEY,
-      'bing_api_key': apiVariables.BING_API_KEY,
-      'searchapi_api_key': apiVariables.SEARCHAPI_API_KEY,
-      'serpapi_api_key': apiVariables.SERPAPI_API_KEY,
-      'serper_api_key': apiVariables.SERPER_API_KEY,
-      'searx_url': apiVariables.SEARX_URL
-    };
-
-    if (!socket) {
-      if (typeof window !== 'undefined') {
-        const { protocol, pathname } = window.location;
-        let { host } = window.location;
-        host = host.includes('localhost') ? 'localhost:8000' : host;
-        const ws_uri = `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}${pathname}ws`;
-
-        const newSocket = new WebSocket(ws_uri);
-        setSocket(newSocket);
-
-        newSocket.onmessage = (event) => {
-          const data = JSON.parse(event.data);
-          console.log('websocket data caught in frontend: ', data);
-
-          if (data.type === 'human_feedback' && data.content === 'request') {
-            console.log('triggered human feedback condition')
-            setQuestionForHuman(data.output)
-            setShowHumanFeedback(true);
-          } else {
-            const contentAndType = `${data.content}-${data.type}`;
-            setOrderedData((prevOrder) => [...prevOrder, { ...data, contentAndType }]);
-
-            if (data.type === 'report') {
-              setAnswer((prev:any) => prev + data.output);
-            } else if (data.type === 'path' || data.type === 'chat') {
-              setLoading(false);
-            }
-          }
-          
-        };
-
-        newSocket.onopen = () => {
-          const { task, report_type, report_source, tone } = chatBoxSettings;
-          let data = "start " + JSON.stringify({ task: promptValue, report_type, report_source, tone, headers });
-          newSocket.send(data);
-        };
-
-        newSocket.onclose = () => {
-          clearInterval(heartbeatInterval.current);
-          setSocket(null);
-        };
-      }
-    } else {
-      const { task, report_type, report_source, tone } = chatBoxSettings;
-      let data = "start " + JSON.stringify({ task: promptValue, report_type, report_source, tone, headers });
-      socket.send(data);
-    }
-  };
+  const { socket, initializeWebSocket } = useWebSocket(
+    setOrderedData,
+    setAnswer,
+    setLoading,
+    setShowHumanFeedback,
+    setQuestionForHuman
+  );
 
-  // Add this function to handle feedback submission
   const handleFeedbackSubmit = (feedback: string | null) => {
-    console.log('user feedback is passed to handleFeedbackSubmit: ', feedback);
     if (socket) {
       socket.send(JSON.stringify({ type: 'human_feedback', content: feedback }));
     }
@@ -126,14 +55,10 @@ export default function Home() {
       setPromptValue("");
       setAnswer("");
 
-      const questionData: QuestionData = { 
-        type: 'question', 
-        content: message 
-      };
+      const questionData: QuestionData = { type: 'question', content: message };
       setOrderedData(prevOrder => [...prevOrder, questionData]);
       
-      const data = `chat${JSON.stringify({ message })}`;
-      socket.send(data);
+      socket.send(`chat${JSON.stringify({ message })}`);
     }
   };
 
@@ -143,31 +68,19 @@ export default function Home() {
     setQuestion(newQuestion);
     setPromptValue("");
     setAnswer("");
+    setOrderedData((prevOrder) => [...prevOrder, { type: 'question', content: newQuestion }]);
 
-    // Add the new question to orderedData
-    setOrderedData((prevOrder:any) => [...prevOrder, { type: 'question', content: newQuestion }]);
-
-    const { report_type, report_source, tone } = chatBoxSettings;
-
-    // Retrieve LANGGRAPH_HOST_URL from local storage or state
     const storedConfig = localStorage.getItem('apiVariables');
     const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
     const langgraphHostUrl = apiVariables.LANGGRAPH_HOST_URL;
 
-    if (report_type === 'multi_agents' && langgraphHostUrl) {
-
-      let { streamResponse, host, thread_id } = await startLanggraphResearch(newQuestion, report_source, langgraphHostUrl);
-
+    if (chatBoxSettings.report_type === 'multi_agents' && langgraphHostUrl) {
+      let { streamResponse, host, thread_id } = await startLanggraphResearch(newQuestion, chatBoxSettings.report_source, langgraphHostUrl);
       const langsmithGuiLink = `https://smith.langchain.com/studio/thread/${thread_id}?baseUrl=${host}`;
-
-      console.log('langsmith-gui-link in page.tsx', langsmithGuiLink);
-      // Add the Langgraph button to orderedData
       setOrderedData((prevOrder) => [...prevOrder, { type: 'langgraphButton', link: langsmithGuiLink }]);
 
       let previousChunk = null;
-
       for await (const chunk of streamResponse) {
-        console.log(chunk);
         if (chunk.data.report != null && chunk.data.report != "Full report content here") {
           setOrderedData((prevOrder) => [...prevOrder, { ...chunk.data, output: chunk.data.report, type: 'report' }]);
           setLoading(false);
@@ -178,7 +91,7 @@ export default function Home() {
         previousChunk = chunk;
       }
     } else {
-      startResearch(chatBoxSettings);
+      initializeWebSocket(newQuestion, chatBoxSettings);
     }
   };
 
@@ -187,8 +100,6 @@ export default function Home() {
     setPromptValue("");
     setQuestion("");
     setAnswer("");
-    setSources([]);
-    setSimilarQuestions([]);
   };
 
   const handleClickSuggestion = (value: string) => {
@@ -199,214 +110,32 @@ export default function Home() {
     }
   };
 
-  const preprocessOrderedData = (data:any) => {
-    const groupedData: any[] = [];
-    let currentAccordionGroup:any = null;
-    let currentSourceGroup:any = null;
-    let currentReportGroup:any = null;
-    let finalReportGroup:any = null;
-    let currentImagesGroup:any = null;
-    let sourceBlockEncountered = false;
-    let lastSubqueriesIndex = -1;
-
-    data.forEach((item:any, index:any) => {
-      const { type, content, metadata, output, link } = item;
-
-      if (type === 'question') {
-        groupedData.push({ type: 'question', content });
-      } else if (type === 'report') {
-        if (!currentReportGroup) {
-          currentReportGroup = { type: 'reportBlock', content: '' };
-          groupedData.push(currentReportGroup);
-        }
-        currentReportGroup.content += output;
-      } else if (content === 'selected_images') {
-        groupedData.push({ type: 'imagesBlock', metadata });
-      } else if (type === 'logs' && content === 'research_report') {
-        if (!finalReportGroup) {
-          finalReportGroup = { type: 'reportBlock', content: '' };
-          groupedData.push(finalReportGroup);
-        }
-        finalReportGroup.content += output.report;
-      } else if (type === 'langgraphButton') {
-        groupedData.push({ type: 'langgraphButton', link });
-      } else if (type == 'chat'){
-        groupedData.push({ type: 'chat', content: content });
-      }
-      else {
-        if (currentReportGroup) {
-          currentReportGroup = null;
-        }
-  
-        if (content === 'subqueries') {
-          if (currentAccordionGroup) {
-            currentAccordionGroup = null;
-          }
-          if (currentSourceGroup) {
-            groupedData.push(currentSourceGroup);
-            currentSourceGroup = null;
-          }
-          groupedData.push(item);
-          lastSubqueriesIndex = groupedData.length - 1;
-        } else if (type === 'sourceBlock') {
-          currentSourceGroup = item;
-          if (lastSubqueriesIndex !== -1) {
-            groupedData.splice(lastSubqueriesIndex + 1, 0, currentSourceGroup);
-            lastSubqueriesIndex = -1;
-          } else {
-            groupedData.push(currentSourceGroup);
-          }
-          sourceBlockEncountered = true;
-          currentSourceGroup = null;
-        } else if (content === 'added_source_url') {
-          if (!currentSourceGroup) {
-            currentSourceGroup = { type: 'sourceBlock', items: [] };
-            if (lastSubqueriesIndex !== -1) {
-              groupedData.splice(lastSubqueriesIndex + 1, 0, currentSourceGroup);
-              lastSubqueriesIndex = -1;
-            } else {
-              groupedData.push(currentSourceGroup);
-            }
-            sourceBlockEncountered = true;
-          }
-          let hostname = "";
-          try {
-            if (typeof metadata === 'string') {
-              hostname = new URL(metadata).hostname.replace('www.', '');
-            }
-          } catch (e) {
-            console.error(`Invalid URL: ${metadata}`, e);
-            hostname = "unknown"; // Default or fallback value
-          }
-          currentSourceGroup.items.push({ name: hostname, url: metadata });
-        } else if (type !== 'path' && content !== '') {
-          if (sourceBlockEncountered) {
-            if (!currentAccordionGroup) {
-              currentAccordionGroup = { type: 'accordionBlock', items: [] };
-              groupedData.push(currentAccordionGroup);
-            }
-            currentAccordionGroup.items.push(item);
-          } else {
-            groupedData.push(item);
-          }
-        } else {
-          if (currentAccordionGroup) {
-            currentAccordionGroup = null;
-          }
-          if (currentSourceGroup) {
-            currentSourceGroup = null;
-          }
-          groupedData.push(item);
-        }
-      }
-    });
-  
-    return groupedData;
-  };
-
-  // Remove logs processing from renderComponentsInOrder and add this useEffect
   useEffect(() => {
     const groupedData = preprocessOrderedData(orderedData);
     const statusReports = ["agent_generated", "starting_research", "planning_research"];
     
-    const newLogs: any[] = [];
-    groupedData.forEach((data) => {
+    const newLogs = groupedData.reduce((acc: any[], data) => {
       if (data.type === 'accordionBlock') {
-        const logs = data.items.map((item:any, subIndex:any) => ({
+        const logs = data.items.map((item: any, subIndex: any) => ({
           header: item.content,
           text: item.output,
           metadata: item.metadata,
           key: `${item.type}-${item.content}-${subIndex}`,
         }));
-        newLogs.push(...logs);
+        return [...acc, ...logs];
       } else if (statusReports.includes(data.content)) {
-        newLogs.push({
+        return [...acc, {
           header: data.content,
           text: data.output,
           metadata: data.metadata,
           key: `${data.type}-${data.content}`,
-        });
+        }];
       }
-    });
-    setAllLogs(newLogs);
-  }, [orderedData]); // Only run when orderedData changes
-
-  const renderComponentsInOrder = () => {
-    const groupedData = preprocessOrderedData(orderedData);
+      return acc;
+    }, []);
     
-    // Find path data for AccessReport
-    const pathData = groupedData.find(data => data.type === 'path');
-
-    // Get the initial question (first question in the data)
-    const initialQuestion = groupedData
-      .find(data => data.type === 'question');
-
-    // Remove initial question from subsequent chat interactions
-    const chatComponents = groupedData
-      .filter(data => {
-        if (data.type === 'question' && data === initialQuestion) {
-          return false;
-        }
-        return (data.type === 'question' || data.type === 'chat');
-      })
-      .map((data, index) => {
-        if (data.type === 'question') {
-          return <Question key={`question-${index}`} question={data.content} />;
-        } else {
-          return <Answer key={`chat-${index}`} answer={data.content} />;
-        }
-      });
-
-    // Rest of the component separations remain the same
-    const sourceComponents = groupedData
-      .filter(data => data.type === 'sourceBlock')
-      .map((data, index) => (
-        <Sources key={`sourceBlock-${index}`} sources={data.items}/>
-      ));
-
-    const imageComponents = groupedData
-      .filter(data => data.type === 'imagesBlock')
-      .map((data, index) => (
-        <ImageSection key={`images-${index}`} metadata={data.metadata} />
-      ));
-
-    const initialReport = groupedData
-      .find(data => data.type === 'reportBlock');
-
-    const subqueriesComponent = groupedData
-      .find(data => data.content === 'subqueries');
-
-    return (
-      <>
-        {/* Initial question */}
-        {initialQuestion && <Question question={initialQuestion.content} />}
-
-        {/* Subqueries if they exist */}
-        {subqueriesComponent && (
-          <SubQuestions
-            metadata={subqueriesComponent.metadata}
-            handleClickSuggestion={handleClickSuggestion}
-          />
-        )}
-
-        {/* 1. Sources */}
-        {sourceComponents}
-
-        {/* 0. Agent work logs */}
-        {orderedData.length > 0 && <LogsSection logs={allLogs} />}
-
-        {/* 2. Images */}
-        {imageComponents}
-
-        {/* 3. Initial Answer Report */}
-        {initialReport && <Answer answer={initialReport.content} />}
-        {pathData && <AccessReport accessData={pathData.output} report={answer} />}
-
-        {/* 4. All subsequent chat interactions */}
-        {chatComponents}
-      </>
-    );
-  };
+    setAllLogs(newLogs);
+  }, [orderedData]);
 
   return (
     <>
@@ -424,7 +153,12 @@ export default function Home() {
           <div className="flex h-full w-full grow flex-col justify-between">
             <div className="container w-full space-y-2">
               <div className="container space-y-2 task-components">
-                {renderComponentsInOrder()}
+                <ResearchResults
+                  orderedData={orderedData}
+                  answer={answer}
+                  allLogs={allLogs}
+                  handleClickSuggestion={handleClickSuggestion}
+                />
               </div>
 
               {showHumanFeedback && (
diff --git a/frontend/nextjs/components/ResearchResults.tsx b/frontend/nextjs/components/ResearchResults.tsx
new file mode 100644
index 000000000..499d28f66
--- /dev/null
+++ b/frontend/nextjs/components/ResearchResults.tsx
@@ -0,0 +1,76 @@
+import React from 'react';
+import Question from './ResearchBlocks/Question';
+import Answer from './ResearchBlocks/Answer';
+import Sources from './ResearchBlocks/Sources';
+import ImageSection from './ResearchBlocks/ImageSection';
+import SubQuestions from './ResearchBlocks/elements/SubQuestions';
+import LogsSection from './ResearchBlocks/LogsSection';
+import AccessReport from './Task/AccessReport';
+import { preprocessOrderedData } from '../utils/dataProcessing';
+import { Data } from '../types/data';
+
+interface ResearchResultsProps {
+  orderedData: Data[];
+  answer: string;
+  allLogs: any[];
+  handleClickSuggestion: (value: string) => void;
+}
+
+export const ResearchResults: React.FC<ResearchResultsProps> = ({
+  orderedData,
+  answer,
+  allLogs,
+  handleClickSuggestion
+}) => {
+  const groupedData = preprocessOrderedData(orderedData);
+  const pathData = groupedData.find(data => data.type === 'path');
+  const initialQuestion = groupedData.find(data => data.type === 'question');
+
+  const chatComponents = groupedData
+    .filter(data => {
+      if (data.type === 'question' && data === initialQuestion) {
+        return false;
+      }
+      return (data.type === 'question' || data.type === 'chat');
+    })
+    .map((data, index) => {
+      if (data.type === 'question') {
+        return <Question key={`question-${index}`} question={data.content} />;
+      } else {
+        return <Answer key={`chat-${index}`} answer={data.content} />;
+      }
+    });
+
+  const sourceComponents = groupedData
+    .filter(data => data.type === 'sourceBlock')
+    .map((data, index) => (
+      <Sources key={`sourceBlock-${index}`} sources={data.items}/>
+    ));
+
+  const imageComponents = groupedData
+    .filter(data => data.type === 'imagesBlock')
+    .map((data, index) => (
+      <ImageSection key={`images-${index}`} metadata={data.metadata} />
+    ));
+
+  const initialReport = groupedData.find(data => data.type === 'reportBlock');
+  const subqueriesComponent = groupedData.find(data => data.content === 'subqueries');
+
+  return (
+    <>
+      {initialQuestion && <Question question={initialQuestion.content} />}
+      {subqueriesComponent && (
+        <SubQuestions
+          metadata={subqueriesComponent.metadata}
+          handleClickSuggestion={handleClickSuggestion}
+        />
+      )}
+      {sourceComponents}
+      {orderedData.length > 0 && <LogsSection logs={allLogs} />}
+      {imageComponents}
+      {initialReport && <Answer answer={initialReport.content} />}
+      {pathData && <AccessReport accessData={pathData.output} report={answer} />}
+      {chatComponents}
+    </>
+  );
+}; 
\ No newline at end of file
diff --git a/frontend/nextjs/hooks/useWebSocket.ts b/frontend/nextjs/hooks/useWebSocket.ts
new file mode 100644
index 000000000..96772a445
--- /dev/null
+++ b/frontend/nextjs/hooks/useWebSocket.ts
@@ -0,0 +1,69 @@
+import { useRef, useState } from 'react';
+import { Data, ChatBoxSettings, QuestionData } from '../types/data';
+
+export const useWebSocket = (setOrderedData: React.Dispatch<React.SetStateAction<Data[]>>, setAnswer: React.Dispatch<React.SetStateAction<string>>, setLoading: React.Dispatch<React.SetStateAction<boolean>>, setShowHumanFeedback: React.Dispatch<React.SetStateAction<boolean>>, setQuestionForHuman: React.Dispatch<React.SetStateAction<boolean | true>>) => {
+  const [socket, setSocket] = useState<WebSocket | null>(null);
+  const heartbeatInterval = useRef<number>();
+
+  const initializeWebSocket = (promptValue: string, chatBoxSettings: ChatBoxSettings) => {
+    const storedConfig = localStorage.getItem('apiVariables');
+    const apiVariables = storedConfig ? JSON.parse(storedConfig) : {};
+    const headers = {
+      'retriever': apiVariables.RETRIEVER,
+      'langchain_api_key': apiVariables.LANGCHAIN_API_KEY,
+      'openai_api_key': apiVariables.OPENAI_API_KEY,
+      'tavily_api_key': apiVariables.TAVILY_API_KEY,
+      'google_api_key': apiVariables.GOOGLE_API_KEY,
+      'google_cx_key': apiVariables.GOOGLE_CX_KEY,
+      'bing_api_key': apiVariables.BING_API_KEY,
+      'searchapi_api_key': apiVariables.SEARCHAPI_API_KEY,
+      'serpapi_api_key': apiVariables.SERPAPI_API_KEY,
+      'serper_api_key': apiVariables.SERPER_API_KEY,
+      'searx_url': apiVariables.SEARX_URL
+    };
+
+    if (!socket && typeof window !== 'undefined') {
+      const { protocol, pathname } = window.location;
+      let { host } = window.location;
+      host = host.includes('localhost') ? 'localhost:8000' : host;
+      const ws_uri = `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}${pathname}ws`;
+
+      const newSocket = new WebSocket(ws_uri);
+      setSocket(newSocket);
+
+      newSocket.onmessage = (event) => {
+        const data = JSON.parse(event.data);
+        if (data.type === 'human_feedback' && data.content === 'request') {
+          setQuestionForHuman(data.output);
+          setShowHumanFeedback(true);
+        } else {
+          const contentAndType = `${data.content}-${data.type}`;
+          setOrderedData((prevOrder) => [...prevOrder, { ...data, contentAndType }]);
+
+          if (data.type === 'report') {
+            setAnswer((prev: any) => prev + data.output);
+          } else if (data.type === 'path' || data.type === 'chat') {
+            setLoading(false);
+          }
+        }
+      };
+
+      newSocket.onopen = () => {
+        const { report_type, report_source, tone } = chatBoxSettings;
+        let data = "start " + JSON.stringify({ task: promptValue, report_type, report_source, tone, headers });
+        newSocket.send(data);
+      };
+
+      newSocket.onclose = () => {
+        clearInterval(heartbeatInterval.current);
+        setSocket(null);
+      };
+    } else if (socket) {
+      const { report_type, report_source, tone } = chatBoxSettings;
+      let data = "start " + JSON.stringify({ task: promptValue, report_type, report_source, tone, headers });
+      socket.send(data);
+    }
+  };
+
+  return { socket, setSocket, initializeWebSocket };
+};
\ No newline at end of file
diff --git a/frontend/nextjs/styles/markdown.css b/frontend/nextjs/styles/markdown.css
index 329fc50c0..26094e4e5 100644
--- a/frontend/nextjs/styles/markdown.css
+++ b/frontend/nextjs/styles/markdown.css
@@ -44,6 +44,11 @@
     > ul, > ol {
       margin: 0 0 0 1.5em;
     }
+
+    /* Add this new rule to handle paragraphs within list items */
+    > p {
+      margin: 0;
+    }
   }
 
   /* Numbered lists specific styling */
diff --git a/frontend/nextjs/utils/dataProcessing.ts b/frontend/nextjs/utils/dataProcessing.ts
new file mode 100644
index 000000000..005fd711e
--- /dev/null
+++ b/frontend/nextjs/utils/dataProcessing.ts
@@ -0,0 +1,103 @@
+import { Data } from '../types/data';
+
+export const preprocessOrderedData = (data: Data[]) => {
+  const groupedData: any[] = [];
+  let currentAccordionGroup: any = null;
+  let currentSourceGroup: any = null;
+  let currentReportGroup: any = null;
+  let finalReportGroup: any = null;
+  let sourceBlockEncountered = false;
+  let lastSubqueriesIndex = -1;
+
+  data.forEach((item: any) => {
+    const { type, content, metadata, output, link } = item;
+
+    if (type === 'question') {
+      groupedData.push({ type: 'question', content });
+    } else if (type === 'report') {
+      if (!currentReportGroup) {
+        currentReportGroup = { type: 'reportBlock', content: '' };
+        groupedData.push(currentReportGroup);
+      }
+      currentReportGroup.content += output;
+    } else if (content === 'selected_images') {
+      groupedData.push({ type: 'imagesBlock', metadata });
+    } else if (type === 'logs' && content === 'research_report') {
+      if (!finalReportGroup) {
+        finalReportGroup = { type: 'reportBlock', content: '' };
+        groupedData.push(finalReportGroup);
+      }
+      finalReportGroup.content += output.report;
+    } else if (type === 'langgraphButton') {
+      groupedData.push({ type: 'langgraphButton', link });
+    } else if (type === 'chat') {
+      groupedData.push({ type: 'chat', content: content });
+    } else {
+      if (currentReportGroup) {
+        currentReportGroup = null;
+      }
+
+      if (content === 'subqueries') {
+        if (currentAccordionGroup) {
+          currentAccordionGroup = null;
+        }
+        if (currentSourceGroup) {
+          groupedData.push(currentSourceGroup);
+          currentSourceGroup = null;
+        }
+        groupedData.push(item);
+        lastSubqueriesIndex = groupedData.length - 1;
+      } else if (type === 'sourceBlock') {
+        currentSourceGroup = item;
+        if (lastSubqueriesIndex !== -1) {
+          groupedData.splice(lastSubqueriesIndex + 1, 0, currentSourceGroup);
+          lastSubqueriesIndex = -1;
+        } else {
+          groupedData.push(currentSourceGroup);
+        }
+        sourceBlockEncountered = true;
+        currentSourceGroup = null;
+      } else if (content === 'added_source_url') {
+        if (!currentSourceGroup) {
+          currentSourceGroup = { type: 'sourceBlock', items: [] };
+          if (lastSubqueriesIndex !== -1) {
+            groupedData.splice(lastSubqueriesIndex + 1, 0, currentSourceGroup);
+            lastSubqueriesIndex = -1;
+          } else {
+            groupedData.push(currentSourceGroup);
+          }
+          sourceBlockEncountered = true;
+        }
+        let hostname = "";
+        try {
+          if (typeof metadata === 'string') {
+            hostname = new URL(metadata).hostname.replace('www.', '');
+          }
+        } catch (e) {
+          hostname = "unknown";
+        }
+        currentSourceGroup.items.push({ name: hostname, url: metadata });
+      } else if (type !== 'path' && content !== '') {
+        if (sourceBlockEncountered) {
+          if (!currentAccordionGroup) {
+            currentAccordionGroup = { type: 'accordionBlock', items: [] };
+            groupedData.push(currentAccordionGroup);
+          }
+          currentAccordionGroup.items.push(item);
+        } else {
+          groupedData.push(item);
+        }
+      } else {
+        if (currentAccordionGroup) {
+          currentAccordionGroup = null;
+        }
+        if (currentSourceGroup) {
+          currentSourceGroup = null;
+        }
+        groupedData.push(item);
+      }
+    }
+  });
+
+  return groupedData;
+}; 
\ No newline at end of file

From b845b772c3dde94dbbae17666c2ef5b5ae866c08 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 13:46:32 +0200
Subject: [PATCH 39/47] added floating icons in research progress

---
 frontend/nextjs/app/page.tsx                  | 63 ++++++++++++++++++-
 frontend/nextjs/components/Header.tsx         | 38 ++++++++++-
 .../{Task => ResearchBlocks}/AccessReport.tsx |  0
 ...ResearchResults.tsx => ResearchLayout.tsx} |  2 +-
 .../ResearchBlocks/elements/InputArea.tsx     |  7 +++
 .../nextjs/components/ResearchResults.tsx     |  2 +-
 .../nextjs/components/Settings/ChatBox.tsx    |  2 +-
 gpt_researcher/config/variables/default.py    |  4 +-
 8 files changed, 109 insertions(+), 9 deletions(-)
 rename frontend/nextjs/components/{Task => ResearchBlocks}/AccessReport.tsx (100%)
 rename frontend/nextjs/components/ResearchBlocks/{ResearchResults.tsx => ResearchLayout.tsx} (97%)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index 90dc3f065..c15426b80 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -31,6 +31,7 @@ export default function Home() {
   const [questionForHuman, setQuestionForHuman] = useState<true | false>(false);
   const [allLogs, setAllLogs] = useState<any[]>([]);
   const chatContainerRef = useRef<HTMLDivElement>(null);
+  const [isStopped, setIsStopped] = useState(false);
 
   const { socket, initializeWebSocket } = useWebSocket(
     setOrderedData,
@@ -110,11 +111,60 @@ export default function Home() {
     }
   };
 
+  /**
+   * Handles stopping the current research
+   * - Closes WebSocket connection
+   * - Stops loading state
+   * - Marks research as stopped
+   * - Preserves current results
+   */
+  const handleStopResearch = () => {
+    if (socket) {
+      socket.close();
+    }
+    setLoading(false);
+    setIsStopped(true);
+  };
+
+  /**
+   * Handles starting a new research
+   * - Clears all previous research data and states
+   * - Resets UI to initial state
+   * - Closes any existing WebSocket connections
+   */
+  const handleStartNewResearch = () => {
+    // Reset UI states
+    setShowResult(false);
+    setPromptValue("");
+    setIsStopped(false);
+    
+    // Clear previous research data
+    setQuestion("");
+    setAnswer("");
+    setOrderedData([]);
+    setAllLogs([]);
+    
+    // Reset feedback states
+    setShowHumanFeedback(false);
+    setQuestionForHuman(false);
+    
+    // Clean up connections
+    if (socket) {
+      socket.close();
+    }
+    setLoading(false);
+  };
+
+  /**
+   * Processes ordered data into logs for display
+   * Updates whenever orderedData changes
+   */
   useEffect(() => {
     const groupedData = preprocessOrderedData(orderedData);
     const statusReports = ["agent_generated", "starting_research", "planning_research"];
     
     const newLogs = groupedData.reduce((acc: any[], data) => {
+      // Process accordion blocks (grouped data)
       if (data.type === 'accordionBlock') {
         const logs = data.items.map((item: any, subIndex: any) => ({
           header: item.content,
@@ -123,7 +173,9 @@ export default function Home() {
           key: `${item.type}-${item.content}-${subIndex}`,
         }));
         return [...acc, ...logs];
-      } else if (statusReports.includes(data.content)) {
+      } 
+      // Process status reports
+      else if (statusReports.includes(data.content)) {
         return [...acc, {
           header: data.content,
           text: data.output,
@@ -139,7 +191,13 @@ export default function Home() {
 
   return (
     <>
-      <Header />
+      <Header 
+        loading={loading}
+        isStopped={isStopped}
+        showResult={showResult}
+        onStop={handleStopResearch}
+        onNewResearch={handleStartNewResearch}
+      />
       <main className="min-h-[100vh] pt-[120px]">
         {!showResult && (
           <Hero
@@ -182,6 +240,7 @@ export default function Home() {
                   handleSecondary={handleDisplayResult}
                   disabled={loading}
                   reset={reset}
+                  isStopped={isStopped}
                 />
               )}
             </div>
diff --git a/frontend/nextjs/components/Header.tsx b/frontend/nextjs/components/Header.tsx
index 42646445b..30227fae1 100644
--- a/frontend/nextjs/components/Header.tsx
+++ b/frontend/nextjs/components/Header.tsx
@@ -1,11 +1,23 @@
 import Image from "next/image";
 
-const Header = () => {
+interface HeaderProps {
+  loading?: boolean;      // Indicates if research is currently in progress
+  isStopped?: boolean;    // Indicates if research was manually stopped
+  showResult?: boolean;   // Controls if research results are being displayed
+  onStop?: () => void;    // Handler for stopping ongoing research
+  onNewResearch?: () => void;  // Handler for starting fresh research
+}
+
+const Header = ({ loading, isStopped, showResult, onStop, onNewResearch }: HeaderProps) => {
   return (
     <div className="fixed top-0 left-0 right-0 z-50">
+      {/* Original gradient background with blur effect */}
       <div className="absolute inset-0 backdrop-blur-sm bg-gradient-to-b to-transparent"></div>
+      
+      {/* Header container */}
       <div className="container relative h-[60px] px-4 lg:h-[80px] lg:px-0 pt-4 pb-4">
-        <div className="flex justify-center items-center h-full">
+        <div className="flex flex-col items-center">
+          {/* Logo/Home link */}
           <a href="/">
             <Image
               src="/img/gptr-logo.png"
@@ -15,6 +27,28 @@ const Header = () => {
               className="lg:h-16 lg:w-16"
             />
           </a>
+          
+          {/* Action buttons container */}
+          <div className="flex gap-2 mt-2 transition-all duration-300 ease-in-out">
+            {/* Stop button - shown only during active research */}
+            {loading && !isStopped && (
+              <button
+                onClick={onStop}
+                className="flex items-center justify-center px-6 h-8 text-sm text-white bg-red-500 rounded-full hover:bg-red-600 transform hover:scale-105 transition-all duration-200 shadow-lg whitespace-nowrap"
+              >
+                Stop
+              </button>
+            )}
+            {/* New Research button - shown after stopping or completing research */}
+            {(isStopped || !loading) && showResult && (
+              <button
+                onClick={onNewResearch}
+                className="flex items-center justify-center px-6 h-8 text-sm text-white bg-[rgb(168,85,247)] rounded-full hover:bg-[rgb(147,51,234)] transform hover:scale-105 transition-all duration-200 shadow-lg whitespace-nowrap"
+              >
+                New Research
+              </button>
+            )}
+          </div>
         </div>
       </div>
     </div>
diff --git a/frontend/nextjs/components/Task/AccessReport.tsx b/frontend/nextjs/components/ResearchBlocks/AccessReport.tsx
similarity index 100%
rename from frontend/nextjs/components/Task/AccessReport.tsx
rename to frontend/nextjs/components/ResearchBlocks/AccessReport.tsx
diff --git a/frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
similarity index 97%
rename from frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx
rename to frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
index 8ba58b40c..dd8e41a50 100644
--- a/frontend/nextjs/components/ResearchBlocks/ResearchResults.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
@@ -5,7 +5,7 @@ import Question from './Question';
 import SubQuestions from './elements/SubQuestions';
 import LogsSection from './LogsSection';
 import ImageSection from './ImageSection';
-import AccessReport from '../Task/AccessReport';
+import AccessReport from './AccessReport';
 
 interface ResearchResultsProps {
   groupedData: any[];
diff --git a/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx b/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
index 58c26459d..5a5cb024d 100644
--- a/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/elements/InputArea.tsx
@@ -9,6 +9,7 @@ type TInputAreaProps = {
   handleSecondary?: (query: string) => void;
   disabled?: boolean;
   reset?: () => void;
+  isStopped?: boolean;
 };
 
 // Debounce function to limit the rate at which a function can fire
@@ -31,7 +32,13 @@ const InputArea: FC<TInputAreaProps> = ({
   handleSecondary,
   disabled,
   reset,
+  isStopped,
 }) => {
+  // Only show input if not stopped
+  if (isStopped) {
+    return null;
+  }
+
   const placeholder = handleSecondary
     ? "Any questions about this report?"
     : "What would you like to research next?";
diff --git a/frontend/nextjs/components/ResearchResults.tsx b/frontend/nextjs/components/ResearchResults.tsx
index 499d28f66..6a8576de1 100644
--- a/frontend/nextjs/components/ResearchResults.tsx
+++ b/frontend/nextjs/components/ResearchResults.tsx
@@ -5,7 +5,7 @@ import Sources from './ResearchBlocks/Sources';
 import ImageSection from './ResearchBlocks/ImageSection';
 import SubQuestions from './ResearchBlocks/elements/SubQuestions';
 import LogsSection from './ResearchBlocks/LogsSection';
-import AccessReport from './Task/AccessReport';
+import AccessReport from './ResearchBlocks/AccessReport';
 import { preprocessOrderedData } from '../utils/dataProcessing';
 import { Data } from '../types/data';
 
diff --git a/frontend/nextjs/components/Settings/ChatBox.tsx b/frontend/nextjs/components/Settings/ChatBox.tsx
index 4df28afcb..5870b1dfe 100644
--- a/frontend/nextjs/components/Settings/ChatBox.tsx
+++ b/frontend/nextjs/components/Settings/ChatBox.tsx
@@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react';
 import ResearchForm from '../Task/ResearchForm';
 import Report from '../Task/Report';
 import AgentLogs from '../Task/AgentLogs';
-import AccessReport from '../Task/AccessReport';
+import AccessReport from '../ResearchBlocks/AccessReport';
 
 interface ChatBoxSettings {
   report_source: string;
diff --git a/gpt_researcher/config/variables/default.py b/gpt_researcher/config/variables/default.py
index 034275c30..38f71c51a 100644
--- a/gpt_researcher/config/variables/default.py
+++ b/gpt_researcher/config/variables/default.py
@@ -3,7 +3,7 @@
 DEFAULT_CONFIG: BaseConfig = {
     "RETRIEVER": "tavily",
     "EMBEDDING": "openai:text-embedding-3-small",
-    "SIMILARITY_THRESHOLD": 0.42,
+    "SIMILARITY_THRESHOLD": 0.52,
     "FAST_LLM": "openai:gpt-4o-mini",
     "SMART_LLM": "openai:gpt-4o-2024-08-06",
     "STRATEGIC_LLM": "openai:o1-preview",
@@ -16,7 +16,7 @@
     "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0",
     "MAX_SEARCH_RESULTS_PER_QUERY": 5,
     "MEMORY_BACKEND": "local",
-    "TOTAL_WORDS": 900,
+    "TOTAL_WORDS": 1200,
     "REPORT_FORMAT": "APA",
     "MAX_ITERATIONS": 4,
     "AGENT_ROLE": None,

From c76b689d0a09b315b198aca069e070dcb94b7a41 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 13:50:37 +0200
Subject: [PATCH 40/47] modified

---
 .../components/ResearchBlocks/ResearchLayout.tsx      | 11 ++++++++---
 gpt_researcher/config/variables/default.py            |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
index dd8e41a50..c6358d2ff 100644
--- a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
@@ -20,6 +20,12 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
   answer,
   handleClickSuggestion,
 }) => {
+  const sourcesComponents = groupedData
+    .filter(data => data.type === 'sourceBlock')
+    .map((data, index) => (
+      <Sources key={`sourceBlock-${index}`} sources={data.items}/>
+    ));
+
   const imageComponents = groupedData
     .filter(data => data.type === 'imagesBlock')
     .map((data, index) => (
@@ -36,9 +42,7 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
 
   const otherComponents = groupedData
     .map((data, index) => {
-      if (data.type === 'sourceBlock') {
-        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
-      } else if (data.type === 'question') {
+      if (data.type === 'question') {
         return <Question key={`question-${index}`} question={data.content} />;
       } else if (data.type === 'chat') {
         return <Answer key={`chat-${index}`} answer={data.content} />;
@@ -58,6 +62,7 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
     <>
       {otherComponents}
       {groupedData.length > 0 && <LogsSection logs={allLogs} />}
+      {sourcesComponents}
       {imageComponents}
       {reportComponents}
       {pathData && <AccessReport accessData={pathData.output} report={answer} />}
diff --git a/gpt_researcher/config/variables/default.py b/gpt_researcher/config/variables/default.py
index 38f71c51a..c0b2b8088 100644
--- a/gpt_researcher/config/variables/default.py
+++ b/gpt_researcher/config/variables/default.py
@@ -16,7 +16,7 @@
     "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0",
     "MAX_SEARCH_RESULTS_PER_QUERY": 5,
     "MEMORY_BACKEND": "local",
-    "TOTAL_WORDS": 1200,
+    "TOTAL_WORDS": 1000,
     "REPORT_FORMAT": "APA",
     "MAX_ITERATIONS": 4,
     "AGENT_ROLE": None,

From fe335c9e4582744a0cb73d2763b2b5ac960b2aca Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 13:52:49 +0200
Subject: [PATCH 41/47] reverted last commit

---
 .../components/ResearchBlocks/ResearchLayout.tsx      | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
index c6358d2ff..dd8e41a50 100644
--- a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
@@ -20,12 +20,6 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
   answer,
   handleClickSuggestion,
 }) => {
-  const sourcesComponents = groupedData
-    .filter(data => data.type === 'sourceBlock')
-    .map((data, index) => (
-      <Sources key={`sourceBlock-${index}`} sources={data.items}/>
-    ));
-
   const imageComponents = groupedData
     .filter(data => data.type === 'imagesBlock')
     .map((data, index) => (
@@ -42,7 +36,9 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
 
   const otherComponents = groupedData
     .map((data, index) => {
-      if (data.type === 'question') {
+      if (data.type === 'sourceBlock') {
+        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
+      } else if (data.type === 'question') {
         return <Question key={`question-${index}`} question={data.content} />;
       } else if (data.type === 'chat') {
         return <Answer key={`chat-${index}`} answer={data.content} />;
@@ -62,7 +58,6 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
     <>
       {otherComponents}
       {groupedData.length > 0 && <LogsSection logs={allLogs} />}
-      {sourcesComponents}
       {imageComponents}
       {reportComponents}
       {pathData && <AccessReport accessData={pathData.output} report={answer} />}

From d9062abab5859f5121ad57c121b3525731276472 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 13:54:36 +0200
Subject: [PATCH 42/47] finalized feature

---
 frontend/nextjs/components/ResearchBlocks/AccessReport.tsx | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/frontend/nextjs/components/ResearchBlocks/AccessReport.tsx b/frontend/nextjs/components/ResearchBlocks/AccessReport.tsx
index 237f86985..080e5c91c 100644
--- a/frontend/nextjs/components/ResearchBlocks/AccessReport.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/AccessReport.tsx
@@ -37,12 +37,6 @@ const AccessReport: React.FC<AccessReportProps> = ({ accessData, report }) => {
         target="_blank">
         Download DocX
       </a>
-      <a id="downloadLink" 
-        href={""} 
-        className="bg-purple-500 text-white active:bg-purple-600 font-bold uppercase text-sm px-6 py-3 rounded shadow hover:shadow-lg outline-none focus:outline-none mr-1 mb-1 ease-linear transition-all duration-150"
-        target="_blank">
-        New Research
-      </a>
     </div>
   );
 }

From 9e8a6e89c8631e50b528aacf1e7f8dbded6a962b Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 14:09:15 +0200
Subject: [PATCH 43/47] fixed markdown issues with list items

---
 .../components/ResearchBlocks/LogsSection.tsx |  2 +-
 .../components/ResearchBlocks/Sources.tsx     |  2 +-
 frontend/nextjs/public/img/browser.svg        |  7 +++++
 frontend/nextjs/public/img/chat.svg           |  9 ++++++
 frontend/nextjs/styles/markdown.css           | 31 +++++++++++++++++--
 5 files changed, 46 insertions(+), 5 deletions(-)
 create mode 100644 frontend/nextjs/public/img/browser.svg
 create mode 100644 frontend/nextjs/public/img/chat.svg

diff --git a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
index 7780c929a..7fd018ec2 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
@@ -26,7 +26,7 @@ const LogsSection = ({ logs }: OrderedLogsProps) => {
   return (
     <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
-        <Image src="/img/thinking.svg" alt="logs" width={24} height={24} />
+        <Image src="/img/chat.svg" alt="logs" width={24} height={24} />
         <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
           Agent Work
         </h3>
diff --git a/frontend/nextjs/components/ResearchBlocks/Sources.tsx b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
index c2bf478e5..485539d2c 100644
--- a/frontend/nextjs/components/ResearchBlocks/Sources.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/Sources.tsx
@@ -9,7 +9,7 @@ export default function Sources({
   return (
     <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
-        <Image src="/img/white-books.svg" alt="footer" width={24} height={24} />
+        <Image src="/img/browser.svg" alt="footer" width={24} height={24} />
         <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
           sources{" "}
         </h3>
diff --git a/frontend/nextjs/public/img/browser.svg b/frontend/nextjs/public/img/browser.svg
new file mode 100644
index 000000000..2fc00d577
--- /dev/null
+++ b/frontend/nextjs/public/img/browser.svg
@@ -0,0 +1,7 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
+<svg height="800px" width="800px" version="1.1" id="_x32_" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 512 512" xml:space="preserve" fill="#000000">
+
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
+
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
+
<g id="SVGRepo_iconCarrier"> <style type="text/css"> .st0{fill:#ffffff;} </style> <g> <path class="st0" d="M181.374,369.211c19.336,18.703,45.727,30.187,74.563,30.187c1.133,0,2.281-0.015,3.422-0.054 c29.023-0.906,55.07-13.305,73.774-32.656c18.703-19.321,30.187-45.719,30.187-74.555c0-1.133-0.015-2.274-0.062-3.414l-9,0.281 l9-0.289c-0.906-29.023-13.297-55.062-32.641-73.766c-19.336-18.703-45.726-30.188-74.57-30.18c-1.141,0-2.281,0.016-3.422,0.047 c-29.016,0.906-55.062,13.305-73.766,32.656c-18.703,19.321-30.187,45.719-30.187,74.562c0,1.133,0.023,2.282,0.062,3.414 C149.64,324.469,162.031,350.508,181.374,369.211z M320.164,354.164c-8.226,8.508-18.133,15.367-29.195,20.078 c4.594-6.618,8.515-14.43,11.758-23.18h20.312C322.102,352.118,321.148,353.156,320.164,354.164z M331.586,339.594h-25.203 c3.477-12.656,5.594-26.781,6.023-41.781h32.68C344.133,313.078,339.32,327.321,331.586,339.594z M331.602,244.562 c7.703,12.219,12.5,26.438,13.484,41.782h-32.711c-0.437-14.977-2.5-29.125-5.969-41.782H331.602z M318.086,227.906 c1.719,1.656,3.344,3.406,4.922,5.187h-20.304c-1.719-4.648-3.594-9.078-5.703-13.156c-1.844-3.57-3.836-6.914-5.969-10 C301.109,214.25,310.281,220.359,318.086,227.906z M261.734,202.992c2.484,0.149,4.945,0.398,7.374,0.75 c1.008,0.664,2.016,1.375,3.031,2.195c6.86,5.594,13.321,15.023,18.313,27.156h-28.718V202.992z M261.734,244.562H294.5 c3.679,12.336,5.976,26.562,6.445,41.782h-39.21V244.562z M261.734,297.813h39.171c-0.468,15.234-2.781,29.453-6.461,41.781h-32.71 V297.813z M261.734,351.062h28.687c-1.141,2.766-2.328,5.43-3.602,7.898c-4.32,8.375-9.398,14.946-14.68,19.258 c-0.992,0.812-1.984,1.507-2.976,2.164c-2.446,0.359-4.93,0.617-7.43,0.781V351.062z M180.39,339.594 c-7.695-12.203-12.5-26.43-13.477-41.781h32.711c0.438,14.977,2.5,29.125,5.969,41.781H180.39z M193.906,356.25 c-1.718-1.657-3.343-3.398-4.922-5.188h20.305c1.726,4.657,3.602,9.086,5.695,13.164c1.859,3.57,3.844,6.914,5.985,9.993 C210.874,369.906,201.718,363.797,193.906,356.25z M250.266,381.172c-2.485-0.156-4.954-0.398-7.375-0.758 c-1.016-0.657-2.016-1.375-3.031-2.195c-6.867-5.578-13.321-15.016-18.313-27.157h28.719V381.172z M250.266,339.594H217.5 c-3.688-12.328-5.985-26.562-6.446-41.781h39.211V339.594z M250.266,286.344h-39.172c0.469-15.226,2.766-29.438,6.453-41.782 h32.719V286.344z M225.179,225.203c4.321-8.375,9.391-14.946,14.68-19.266c0.992-0.805,1.984-1.508,2.984-2.156 c2.438-0.367,4.922-0.626,7.422-0.782v30.094h-28.688C222.71,230.344,223.906,227.664,225.179,225.203z M191.828,229.992 c8.226-8.5,18.141-15.367,29.203-20.078c-4.594,6.617-8.531,14.43-11.766,23.179h-20.305 C189.89,232.039,190.843,231,191.828,229.992z M180.406,244.562h25.203c-3.469,12.672-5.594,26.797-6.023,41.782h-32.68 C167.859,271.086,172.672,256.844,180.406,244.562z"/> <path class="st0" d="M464,0H48C21.492,0,0,21.492,0,48v416c0,26.508,21.492,48,48,48h416c26.507,0,48-21.492,48-48V48 C512,21.492,490.507,0,464,0z M444.664,35c10.492,0,19,8.508,19,19s-8.508,19-19,19s-19-8.508-19-19S434.172,35,444.664,35z M374.164,35c10.492,0,19,8.508,19,19s-8.508,19-19,19s-19-8.508-19-19S363.672,35,374.164,35z M303.664,35 c10.492,0,19,8.508,19,19s-8.508,19-19,19s-19-8.508-19-19S293.172,35,303.664,35z M472,464c0,4.406-3.586,8-8,8H48 c-4.414,0-8-3.594-8-8V104h432V464z"/> </g> </g>
+
</svg>
\ No newline at end of file
diff --git a/frontend/nextjs/public/img/chat.svg b/frontend/nextjs/public/img/chat.svg
new file mode 100644
index 000000000..c1651b5b2
--- /dev/null
+++ b/frontend/nextjs/public/img/chat.svg
@@ -0,0 +1,9 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
+<svg fill="#ffffff" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg" stroke="#ffffff">
+
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
+
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
+
<g id="SVGRepo_iconCarrier">
+
<path d="M22.365 8.729c.9 0 1.635-.735 1.635-1.635s-.735-1.636-1.635-1.636-1.636.735-1.636 1.636.723 1.635 1.636 1.635m-4.907 5.452a3.27 3.27 0 1 0 0-6.542 3.27 3.27 0 0 0 0 6.542m0 8.722c2.105 0 3.816-1.711 3.816-3.829s-1.711-3.816-3.829-3.816a3.82 3.82 0 0 0-3.816 3.816 3.825 3.825 0 0 0 3.829 3.83M6.542 14.18a6.542 6.542 0 1 0 0-13.084 6.542 6.542 0 1 0 0 13.084"/>
+
</g>
+
</svg>
\ No newline at end of file
diff --git a/frontend/nextjs/styles/markdown.css b/frontend/nextjs/styles/markdown.css
index 26094e4e5..51c99f5eb 100644
--- a/frontend/nextjs/styles/markdown.css
+++ b/frontend/nextjs/styles/markdown.css
@@ -26,7 +26,7 @@
 
   /* Lists */
   ul, ol {
-    margin: 0 0 0 2em;
+    margin: 0 0 0 1em;
     padding: 0;
   }
 
@@ -39,15 +39,40 @@
   }
 
   li {
-    display: list-item;
+    display: grid;
+    grid-template-columns: auto 1fr;
+    grid-gap: 0.5em;
+    padding: 0;
     
     > ul, > ol {
       margin: 0 0 0 1.5em;
     }
 
-    /* Add this new rule to handle paragraphs within list items */
+    /* Remove the marker styling since we're using grid */
+    &::marker {
+      display: none;
+    }
+
+    /* Add a pseudo-element for the marker */
+    &::before {
+      content: '•'; /* For unordered lists */
+      justify-self: end;
+    }
+
+    /* Updated list item paragraph styling */
     > p {
       margin: 0;
+      padding: 0;
+      grid-column: 2;
+    }
+  }
+
+  /* Handle ordered lists */
+  ol > li {
+    counter-increment: list;
+    
+    &::before {
+      content: counter(list) '.'; /* For ordered lists */
     }
   }
 

From 230140785207601c6c2eaf27dbd5c610259e3fc2 Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 14:11:38 +0200
Subject: [PATCH 44/47] finalized icons

---
 frontend/nextjs/components/ResearchBlocks/LogsSection.tsx | 2 +-
 frontend/nextjs/public/img/chat-check.svg                 | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 frontend/nextjs/public/img/chat-check.svg

diff --git a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
index 7fd018ec2..2a5f0b447 100644
--- a/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
+++ b/frontend/nextjs/components/ResearchBlocks/LogsSection.tsx
@@ -26,7 +26,7 @@ const LogsSection = ({ logs }: OrderedLogsProps) => {
   return (
     <div className="container h-auto w-full shrink-0 rounded-lg border border-solid border-[#C2C2C2] bg-gray-800 shadow-md p-5">
       <div className="flex items-start gap-4 pb-3 lg:pb-3.5">
-        <Image src="/img/chat.svg" alt="logs" width={24} height={24} />
+        <Image src="/img/chat-check.svg" alt="logs" width={24} height={24} />
         <h3 className="text-base font-bold uppercase leading-[152.5%] text-white">
           Agent Work
         </h3>
diff --git a/frontend/nextjs/public/img/chat-check.svg b/frontend/nextjs/public/img/chat-check.svg
new file mode 100644
index 000000000..b0e16373f
--- /dev/null
+++ b/frontend/nextjs/public/img/chat-check.svg
@@ -0,0 +1,7 @@
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
+
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
+
<g id="SVGRepo_iconCarrier"> <path d="M14 19C17.7712 19 19.6569 19 20.8284 17.8284C22 16.6569 22 14.7712 22 11C22 7.22876 22 5.34315 20.8284 4.17157C19.6569 3 17.7712 3 14 3H10C6.22876 3 4.34315 3 3.17157 4.17157C2 5.34315 2 7.22876 2 11C2 14.7712 2 16.6569 3.17157 17.8284C3.82475 18.4816 4.69989 18.7706 6 18.8985" stroke="#ffffff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/> <path d="M9 11L11.25 13L15 9" stroke="#ffffff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/> <path d="M14 19C12.7635 19 11.4022 19.4992 10.1586 20.145C8.16119 21.1821 7.16249 21.7007 6.67035 21.3703C6.1782 21.0398 6.27135 20.0151 6.45766 17.9657L6.5 17.5" stroke="#ffffff" stroke-width="2" stroke-linecap="round"/> </g>
+
</svg>
\ No newline at end of file

From 5c7df270428b1ad87936b2b953a84b125d4e080f Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 15:23:08 +0200
Subject: [PATCH 45/47] updated threshold and revised sources layout

---
 .../ResearchBlocks/ResearchLayout.tsx         | 66 -------------------
 .../nextjs/components/ResearchResults.tsx     |  2 +-
 gpt_researcher/config/variables/default.py    |  2 +-
 3 files changed, 2 insertions(+), 68 deletions(-)
 delete mode 100644 frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx

diff --git a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx b/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
deleted file mode 100644
index dd8e41a50..000000000
--- a/frontend/nextjs/components/ResearchBlocks/ResearchLayout.tsx
+++ /dev/null
@@ -1,66 +0,0 @@
-import React from 'react';
-import Answer from './Answer';
-import Sources from './Sources';
-import Question from './Question';
-import SubQuestions from './elements/SubQuestions';
-import LogsSection from './LogsSection';
-import ImageSection from './ImageSection';
-import AccessReport from './AccessReport';
-
-interface ResearchResultsProps {
-  groupedData: any[];
-  allLogs: any[];
-  answer: string;
-  handleClickSuggestion: (value: string) => void;
-}
-
-export const ResearchResults: React.FC<ResearchResultsProps> = ({
-  groupedData,
-  allLogs,
-  answer,
-  handleClickSuggestion,
-}) => {
-  const imageComponents = groupedData
-    .filter(data => data.type === 'imagesBlock')
-    .map((data, index) => (
-      <ImageSection key={`images-${index}`} metadata={data.metadata} />
-    ));
-
-  const reportComponents = groupedData
-    .filter(data => data.type === 'reportBlock')
-    .map((data, index) => (
-      <Answer key={`reportBlock-${index}`} answer={data.content} />
-    ));
-
-  const pathData = groupedData.find(data => data.type === 'path');
-
-  const otherComponents = groupedData
-    .map((data, index) => {
-      if (data.type === 'sourceBlock') {
-        return <Sources key={`sourceBlock-${index}`} sources={data.items}/>;
-      } else if (data.type === 'question') {
-        return <Question key={`question-${index}`} question={data.content} />;
-      } else if (data.type === 'chat') {
-        return <Answer key={`chat-${index}`} answer={data.content} />;
-      } else if (data.content === 'subqueries') {
-        return (
-          <SubQuestions
-            key={`subqueries-${index}`}
-            metadata={data.metadata}
-            handleClickSuggestion={handleClickSuggestion}
-          />
-        );
-      }
-      return null;
-    }).filter(Boolean);
-
-  return (
-    <>
-      {otherComponents}
-      {groupedData.length > 0 && <LogsSection logs={allLogs} />}
-      {imageComponents}
-      {reportComponents}
-      {pathData && <AccessReport accessData={pathData.output} report={answer} />}
-    </>
-  );
-}; 
\ No newline at end of file
diff --git a/frontend/nextjs/components/ResearchResults.tsx b/frontend/nextjs/components/ResearchResults.tsx
index 6a8576de1..fe6b4c71b 100644
--- a/frontend/nextjs/components/ResearchResults.tsx
+++ b/frontend/nextjs/components/ResearchResults.tsx
@@ -65,8 +65,8 @@ export const ResearchResults: React.FC<ResearchResultsProps> = ({
           handleClickSuggestion={handleClickSuggestion}
         />
       )}
-      {sourceComponents}
       {orderedData.length > 0 && <LogsSection logs={allLogs} />}
+      {sourceComponents}
       {imageComponents}
       {initialReport && <Answer answer={initialReport.content} />}
       {pathData && <AccessReport accessData={pathData.output} report={answer} />}
diff --git a/gpt_researcher/config/variables/default.py b/gpt_researcher/config/variables/default.py
index c0b2b8088..f6d848ecd 100644
--- a/gpt_researcher/config/variables/default.py
+++ b/gpt_researcher/config/variables/default.py
@@ -3,7 +3,7 @@
 DEFAULT_CONFIG: BaseConfig = {
     "RETRIEVER": "tavily",
     "EMBEDDING": "openai:text-embedding-3-small",
-    "SIMILARITY_THRESHOLD": 0.52,
+    "SIMILARITY_THRESHOLD": 0.6,
     "FAST_LLM": "openai:gpt-4o-mini",
     "SMART_LLM": "openai:gpt-4o-2024-08-06",
     "STRATEGIC_LLM": "openai:o1-preview",

From 277441753e375236e5abd4283119923c713d70fa Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 15:31:20 +0200
Subject: [PATCH 46/47] added scrol down hint

---
 frontend/nextjs/app/page.tsx | 67 ++++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/frontend/nextjs/app/page.tsx b/frontend/nextjs/app/page.tsx
index c15426b80..e5bd5fd5e 100644
--- a/frontend/nextjs/app/page.tsx
+++ b/frontend/nextjs/app/page.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useRef, useState, useEffect } from "react";
+import { useRef, useState, useEffect, useCallback } from "react";
 import { useWebSocket } from '@/hooks/useWebSocket';
 import { startLanggraphResearch } from '../components/Langgraph/Langgraph';
 import findDifferences from '../helpers/findDifferences';
@@ -32,6 +32,8 @@ export default function Home() {
   const [allLogs, setAllLogs] = useState<any[]>([]);
   const chatContainerRef = useRef<HTMLDivElement>(null);
   const [isStopped, setIsStopped] = useState(false);
+  const [showScrollButton, setShowScrollButton] = useState(false);
+  const mainContentRef = useRef<HTMLDivElement>(null);
 
   const { socket, initializeWebSocket } = useWebSocket(
     setOrderedData,
@@ -189,6 +191,46 @@ export default function Home() {
     setAllLogs(newLogs);
   }, [orderedData]);
 
+  const handleScroll = useCallback(() => {
+    // Calculate if we're near bottom (within 100px)
+    const scrollPosition = window.scrollY + window.innerHeight;
+    const nearBottom = scrollPosition >= document.documentElement.scrollHeight - 100;
+    
+    // Show button if we're not near bottom and page is scrollable
+    const isPageScrollable = document.documentElement.scrollHeight > window.innerHeight;
+    setShowScrollButton(isPageScrollable && !nearBottom);
+  }, []);
+
+  // Add ResizeObserver to watch for content changes
+  useEffect(() => {
+    const resizeObserver = new ResizeObserver(() => {
+      handleScroll();
+    });
+
+    if (mainContentRef.current) {
+      resizeObserver.observe(mainContentRef.current);
+    }
+
+    window.addEventListener('scroll', handleScroll);
+    window.addEventListener('resize', handleScroll);
+    
+    return () => {
+      if (mainContentRef.current) {
+        resizeObserver.unobserve(mainContentRef.current);
+      }
+      resizeObserver.disconnect();
+      window.removeEventListener('scroll', handleScroll);
+      window.removeEventListener('resize', handleScroll);
+    };
+  }, [handleScroll]);
+
+  const scrollToBottom = () => {
+    window.scrollTo({
+      top: document.documentElement.scrollHeight,
+      behavior: 'smooth'
+    });
+  };
+
   return (
     <>
       <Header 
@@ -198,7 +240,7 @@ export default function Home() {
         onStop={handleStopResearch}
         onNewResearch={handleStartNewResearch}
       />
-      <main className="min-h-[100vh] pt-[120px]">
+      <main ref={mainContentRef} className="min-h-[100vh] pt-[120px]">
         {!showResult && (
           <Hero
             promptValue={promptValue}
@@ -247,6 +289,27 @@ export default function Home() {
           </div>
         )}
       </main>
+      {showScrollButton && showResult && (
+        <button
+          onClick={scrollToBottom}
+          className="fixed bottom-8 right-8 flex items-center justify-center w-12 h-12 text-white bg-[rgb(168,85,247)] rounded-full hover:bg-[rgb(147,51,234)] transform hover:scale-105 transition-all duration-200 shadow-lg z-50"
+        >
+          <svg 
+            xmlns="http://www.w3.org/2000/svg" 
+            className="h-6 w-6" 
+            fill="none" 
+            viewBox="0 0 24 24" 
+            stroke="currentColor"
+          >
+            <path 
+              strokeLinecap="round" 
+              strokeLinejoin="round" 
+              strokeWidth={2} 
+              d="M19 14l-7 7m0 0l-7-7m7 7V3" 
+            />
+          </svg>
+        </button>
+      )}
       <Footer setChatBoxSettings={setChatBoxSettings} chatBoxSettings={chatBoxSettings} />
     </>
   );

From ae6ba0e957788a23a73cd34e4c89c7c30642700a Mon Sep 17 00:00:00 2001
From: Assaf Elovic <assaf.elovic@gmail.com>
Date: Sat, 2 Nov 2024 15:33:01 +0200
Subject: [PATCH 47/47] removed grid from report line items

---
 frontend/nextjs/styles/markdown.css | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/frontend/nextjs/styles/markdown.css b/frontend/nextjs/styles/markdown.css
index 51c99f5eb..54268922a 100644
--- a/frontend/nextjs/styles/markdown.css
+++ b/frontend/nextjs/styles/markdown.css
@@ -39,9 +39,6 @@
   }
 
   li {
-    display: grid;
-    grid-template-columns: auto 1fr;
-    grid-gap: 0.5em;
     padding: 0;
     
     > ul, > ol {
@@ -54,8 +51,7 @@
     }
 
     /* Add a pseudo-element for the marker */
-    &::before {
-      content: '•'; /* For unordered lists */
+    &::before { /* For unordered lists */
       justify-self: end;
     }
 
@@ -67,15 +63,6 @@
     }
   }
 
-  /* Handle ordered lists */
-  ol > li {
-    counter-increment: list;
-    
-    &::before {
-      content: counter(list) '.'; /* For ordered lists */
-    }
-  }
-
   /* Numbered lists specific styling */
   ol {
     counter-reset: item;