Merge branch 'develop' into discriptionbox

Author: AmitChauhan63390

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,90 @@
+name: Bug Report
+description: Report a bug to help us improve
+title: "[Bug]: "
+labels: [bug]
+assignees: []
+
+body:
+  - type: dropdown
+    id: environment
+    attributes:
+      label: Where are you using the app?
+      options:
+        - Cloud (Hosted by Us)
+        - Self-Hosted (OSS) with Docker
+        - Self-Hosted (OSS) without Docker
+    validations:
+      required: true
+
+  - type: input
+    id: app_version
+    attributes:
+      label: App Version
+      description: Enter the version number you are using (if known).
+      placeholder: "e.g., v1.2.3"
+    validations:
+      required: false
+
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Which browser are you using?
+      placeholder: "e.g., Chrome 124, Firefox 115, Safari 17"
+    validations:
+      required: true
+
+  - type: input
+    id: operating_system
+    attributes:
+      label: Operating System
+      description: Your operating system and version.
+      placeholder: "e.g., Windows 11, macOS Sonoma, Ubuntu 22.04"
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: How can we reproduce the problem?
+      placeholder: |
+        1. Go to '...'
+        2. Click on '...'
+        3. Scroll down to '...'
+        4. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen instead?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual_behavior
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant Logs or Screenshots
+      description: Please paste any logs, screenshots, or console errors if available.
+      placeholder: "Paste logs or upload screenshots."
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional_context
+    attributes:
+      label: Additional Context
+      description: Anything else we should know?
+    validations:
+      required: false

ENVEXAMPLE

@@ -7,6 +7,8 @@ DB_PASSWORD=postgres                    # PostgreSQL password
 DB_HOST=postgres                        # Host for PostgreSQL in Docker
 DB_PORT=5432                            # Port for PostgreSQL (default: 5432)
 ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0      # Key for encrypting sensitive data (passwords and proxies)
+SESSION_SECRET=maxun_session # A strong, random string used to sign session cookies. Recommended to define your own session secret to avoid session hijacking.
+
 MINIO_ENDPOINT=minio                    # MinIO endpoint in Docker
 MINIO_PORT=9000                         # Port for MinIO (default: 9000)
 MINIO_CONSOLE_PORT=9001                 # Web UI Port for MinIO (default: 9001)

README.md

@@ -98,6 +98,7 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 | `DB_HOST`             | Yes       | Host address where the Postgres database server is running.                                  | Database connection will fail.                               |
 | `DB_PORT`             | Yes       | Port number used to connect to the Postgres database server.                                 | Database connection will fail.                               |
 | `ENCRYPTION_KEY`      | Yes       | Key used for encrypting sensitive data (proxies, passwords).                                 | Encryption functionality will not work.                      |
+| `SESSION_SECRET`      | No       | A strong, random string used to sign session cookies                                          | Uses default secret. Recommended to define your own session secret to avoid session hijacking.  |
 | `MINIO_ENDPOINT`      | Yes       | Endpoint URL for MinIO, to store Robot Run Screenshots.                                      | Connection to MinIO storage will fail.                       |
 | `MINIO_PORT`          | Yes       | Port number for MinIO service.                                                               | Connection to MinIO storage will fail.                       |
 | `MINIO_CONSOLE_PORT`          | No       | Port number for MinIO WebUI service. Needed for Docker setup.                         | Cannot access MinIO Web UI. |
@@ -107,7 +108,6 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 | `GOOGLE_REDIRECT_URI` | No       | Redirect URI for handling Google OAuth responses.                                            | Google login will not work.                                  |
 | `AIRTABLE_CLIENT_ID` | No       | Client ID for Airtable, used for Airtable integration authentication.                         | Airtable login will not work.  |
 | `AIRTABLE_REDIRECT_URI` | No    | Redirect URI for handling Airtable OAuth responses.                                           | Airtable login will not work.  |
-
 | `MAXUN_TELEMETRY`     | No        | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. |
 
 

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   postgres:
     image: postgres:13

index.html

@@ -6,7 +6,7 @@
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="Web site created using Vite"
+      content="Maxun is an open-source no-code web data extraction platform. Train a robot in 2 minutes to extract data on auto-pilot!"
     />
     <link rel="icon" type="image/png" href="src/assets/maxunlogo.png">
     <title>Maxun | Open Source No Code Web Data Extraction Platform</title>

maxun-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",

maxun-core/src/interpret.ts

@@ -37,14 +37,16 @@ declare global {
  * Defines optional intepreter options (passed in constructor)
  */
 interface InterpreterOptions {
+  mode?: string;
   maxRepeats: number;
   maxConcurrency: number;
   serializableCallback: (output: any) => (void | Promise<void>);
   binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
   debug: boolean;
   debugChannel: Partial<{
-    activeId: Function,
-    debugMessage: Function,
+    activeId: (id: number) => void,
+    debugMessage: (msg: string) => void,
+    setActionType: (type: string) => void,
   }>
 }
 
@@ -377,12 +379,20 @@ export default class Interpreter extends EventEmitter {
      */
     const wawActions: Record<CustomFunctions, (...args: any[]) => void> = {
       screenshot: async (params: PageScreenshotOptions) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('screenshot');
+        }
+
         const screenshotBuffer = await page.screenshot({
           ...params, path: undefined,
         });
         await this.options.binaryCallback(screenshotBuffer, 'image/png');
       },
       enqueueLinks: async (selector: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('enqueueLinks');
+        }
+
         const links: string[] = await page.locator(selector)
           .evaluateAll(
             // @ts-ignore
@@ -409,55 +419,61 @@ export default class Interpreter extends EventEmitter {
         await page.close();
       },
       scrape: async (selector?: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrape');
+        }
+
         await this.ensureScriptsLoaded(page);
 
         const scrapeResults: Record<string, string>[] = await page.evaluate((s) => window.scrape(s ?? null), selector);
         await this.options.serializableCallback(scrapeResults);
       },
 
       scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; shadow: string}>) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeSchema');
+        }
+
+        if (this.options.mode && this.options.mode === 'editor') {
+          await this.options.serializableCallback({});
+          return;
+        }
+      
         await this.ensureScriptsLoaded(page);
 
         const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
 
-        const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
-        newResults.forEach((result) => {
-          Object.entries(result).forEach(([key, value]) => {
-              const keyExists = this.cumulativeResults.some(
-                  (item) => key in item && item[key] !== undefined
-              );
-  
-              if (!keyExists) {
-                  this.cumulativeResults.push({ [key]: value });
-              }
-          });
+        if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
+          this.cumulativeResults = [];
+        }
+      
+        if (this.cumulativeResults.length === 0) {
+          this.cumulativeResults.push({});
+        }
+      
+        const mergedResult = this.cumulativeResults[0];
+        const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
+        
+        Object.entries(resultToProcess).forEach(([key, value]) => {
+          if (value !== undefined) {
+            mergedResult[key] = value;
+          }
         });
-
-        const mergedResult: Record<string, string>[] = [
-          Object.fromEntries( 
-            Object.entries(
-              this.cumulativeResults.reduce((acc, curr) => {
-                Object.entries(curr).forEach(([key, value]) => {
-                  // If the key doesn't exist or the current value is not undefined, add/update it
-                  if (value !== undefined) {
-                    acc[key] = value;
-                  }
-                });
-                return acc;
-              }, {})
-            )
-          )
-        ];
-
-        // Log cumulative results after each action
-        console.log("CUMULATIVE results:", this.cumulativeResults);
-        console.log("MERGED results:", mergedResult);
-
-        await this.options.serializableCallback(mergedResult);
-        // await this.options.serializableCallback(scrapeResult);
+      
+        console.log("Updated merged result:", mergedResult);
+        await this.options.serializableCallback([mergedResult]);
       },
 
       scrapeList: async (config: { listSelector: string, fields: any, limit?: number, pagination: any }) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeList');
+        }
+
+        if (this.options.mode && this.options.mode === 'editor') {
+          await this.options.serializableCallback({});
+          return;
+        }
+
         await this.ensureScriptsLoaded(page);
         if (!config.pagination) {
           const scrapeResults: Record<string, any>[] = await page.evaluate((cfg) => window.scrapeList(cfg), config);
@@ -469,6 +485,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       scrapeListAuto: async (config: { listSelector: string }) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scrapeListAuto');
+        }
+
         await this.ensureScriptsLoaded(page);
 
         const scrapeResults: { selector: string, innerText: string }[] = await page.evaluate((listSelector) => {
@@ -479,6 +499,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       scroll: async (pages?: number) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('scroll');
+        }
+
         await page.evaluate(async (pagesInternal) => {
           for (let i = 1; i <= (pagesInternal ?? 1); i += 1) {
             // @ts-ignore
@@ -488,6 +512,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       script: async (code: string) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('script');
+        }
+
         const AsyncFunction: FunctionConstructor = Object.getPrototypeOf(
           async () => { },
         ).constructor;
@@ -496,6 +524,10 @@ export default class Interpreter extends EventEmitter {
       },
 
       flag: async () => new Promise((res) => {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType('flag');
+        }
+
         this.emit('flag', page, res);
       }),
     };
@@ -526,6 +558,10 @@ export default class Interpreter extends EventEmitter {
         const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
         await wawActions[step.action as CustomFunctions](...(params ?? []));
       } else {
+        if (this.options.debugChannel?.setActionType) {
+          this.options.debugChannel.setActionType(String(step.action));
+        }
+
         // Implements the dot notation for the "method name" in the workflow
         const levels = String(step.action).split('.');
         const methodName = levels[levels.length - 1];

nginx.conf

@@ -1,45 +1,24 @@
 server {
     listen 80;
+    server_name _;
 
+    root /var/www/maxun;
+    index index.html;
+
+    # Serve the frontend
     location / {
-        root /usr/share/nginx/html;
         try_files $uri $uri/ /index.html;
     }
-
-    location /api {
-        proxy_pass http://localhost:8080;
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'upgrade';
-        proxy_set_header Host $host;
-        proxy_cache_bypass $http_upgrade;
-        
-        # Add timeout configurations
-        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
-        proxy_read_timeout 60s;
-        
-        # Add error handling
-        proxy_intercept_errors on;
-        error_page 502 503 504 /50x.html;
-    }
-
-    location ~ ^/(record|workflow|storage|auth|integration|proxy|api-docs) {
-        proxy_pass http://localhost:8080;
+    
+    # Proxy for backend 
+    location  ^/(auth|storage|record|workflow|robot|proxy|api-docs|api)(/|$) {
+        proxy_pass http://localhost:8080; # change as per your setup
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection 'keep-alive';  # Ensure connections remain open
         proxy_set_header Connection 'upgrade';
         proxy_set_header Host $host;
-        proxy_cache_bypass $http_upgrade;
-
-        # Timeout configurations
-        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
-        proxy_read_timeout 60s;
-
-        # Error handling for these routes
-        proxy_intercept_errors on;
-        error_page 502 503 504 /50x.html;
+        proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    	proxy_set_header X-Forwarded-Proto $scheme;
     }
-}
+}