Merge pull request #869 from ScrapeGraphAI/pre/beta

Pre/beta

.github/workflows/pylint.yml → .github/workflows/code-quality.yml

@@ -1,20 +1,35 @@
+name: Code Quality Checks
+
 on:
   push:
     paths:
       - 'scrapegraphai/**'
       - '.github/workflows/pylint.yml'
 
 jobs:
-  build:
+  quality:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: Install uv
         uses: astral-sh/setup-uv@v3
+
       - name: Install dependencies
         run: uv sync --frozen
+
+      - name: Run Ruff
+        run: uv run ruff check scrapegraphai
+
+      - name: Run Black
+        run: uv run black --check scrapegraphai
+
+      - name: Run isort
+        run: uv run isort --check-only scrapegraphai
+
       - name: Analysing the code with pylint
         run: uv run poe pylint-ci
+
       - name: Check Pylint score
         run: |
           pylint_score=$(uv run poe pylint-score-ci | grep 'Raw metrics' | awk '{print $4}')
@@ -23,4 +38,4 @@ jobs:
             exit 1
           else
             echo "Pylint score is acceptable."
-          fi
+          fi

.gitignore

@@ -42,3 +42,153 @@ lib/
 # extras
 cache/
 run_smart_scraper.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+pdm.lock
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+.idea/
+
+# VS Code
+.vscode/
+
+# macOS
+.DS_Store
+
+dev.ipynb

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        exclude: mkdocs.yml

CHANGELOG.md

@@ -1,4 +1,59 @@
-## [1.34.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.34.1...v1.34.2) (2025-01-06)
+## [1.35.0-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.35.0-beta.3...v1.35.0-beta.4) (2025-01-06)
+
+
+### Features
+
+* ⏰added graph timeout and fixed model_tokens param ([#810](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/810) [#856](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/856) [#853](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/853)) ([01a331a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/01a331afa5fc6f6d6aea4f1969cbf41f0b25f5e0))
+
+## [1.35.0-beta.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.35.0-beta.2...v1.35.0-beta.3) (2025-01-06)
+
+
+### Features
+
+* serper api search ([1c0141f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/1c0141fd281881e342a113d5a414930d8184146b))
+
+## [1.35.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.35.0-beta.1...v1.35.0-beta.2) (2025-01-06)
+
+
+### Features
+
+* add codequality workflow ([4380afb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/4380afb5c15e7f6057fd44bdbd6bde410bb98378))
+
+## [1.35.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.34.3-beta.1...v1.35.0-beta.1) (2025-01-06)
+
+
+### Features
+
+* ⛏️ enhanced contribution and precommit added ([fcbfe78](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcbfe78983c5c36fe5e4e0659ccfebc7fd9952b4))
+* add timeout and retry_limit in loader_kwargs ([#865](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/865) [#831](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/831)) ([21147c4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/21147c46a53e943dd5f297e6c7c3433edadfbc27))
+
+
+### Bug Fixes
+
+* local html handling ([2a15581](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2a15581865d84021278ec0bf601172f6f8343717))
+
+## [1.34.3-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.34.2...v1.34.3-beta.1) (2025-01-06)
+
+
+### Bug Fixes
+
+* browserbase integration ([752a885](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/752a885f5c521b7141728952d913a5a25650d8e2))
+
+
+### CI
+
+* **release:** 1.34.2-beta.1 [skip ci] ([f383e72](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f383e7283727ad798fe152434eee7e6750c36166)), closes [#861](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/861) [#861](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/861)
+* **release:** 1.34.2-beta.2 [skip ci] ([93fd9d2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/93fd9d29036ce86f6a17f960f691bc6e4b26ea51))
+
+## [1.34.2-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.34.2-beta.1...v1.34.2-beta.2) (2025-01-06)
+
+
+### Bug Fixes
+
+* browserbase integration ([752a885](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/752a885f5c521b7141728952d913a5a25650d8e2))
+
+## [1.34.2-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.34.1...v1.34.2-beta.1) (2025-01-06)
+
 
 
 ### Bug Fixes