Move scraper files to a subfolder

Author: dan-niles

.github/workflows/Publish.yml

@@ -16,18 +16,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Build packages
+        working-directory: scraper
         run: |
           pip install -U pip build
           python -m build --sdist --wheel
 
       - name: Upload to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1.8
         with:
-          packages-dir: dist/
+          packages-dir: scraper/dist/
 
       - name: Build and push Docker image
         uses: openzim/docker-publish-action@v10

.github/workflows/QA.yml

@@ -16,19 +16,23 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Install dependencies
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[lint,check,scripts,test]
 
       - name: Check black formatting
+        working-directory: scraper
         run: inv lint-black
 
       - name: Check ruff
+        working-directory: scraper
         run: inv lint-ruff
 
       - name: Check pyright
-        run: inv check-pyright
+        working-directory: scraper
+        run: inv check-pyright

.github/workflows/Tests.yml

@@ -16,15 +16,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Install dependencies (and project)
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[test,scripts]
 
       - name: Run the tests
+        working-directory: scraper
         run: inv coverage --args "-vvv"
 
       - name: Upload coverage report to codecov
@@ -40,10 +42,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Ensure we can build Python targets
+        working-directory: scraper
         run: |
           pip install -U pip build
           python3 -m build --sdist --wheel

.gitignore

@@ -366,13 +366,13 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/linux,macos,python,database,visualstudiocode,intellij
 
 # JS deps
-src/youtube2zim/templates/assets/chosen/
-src/youtube2zim/templates/assets/jquery.min.js
-src/youtube2zim/templates/assets/ogvjs/
-src/youtube2zim/templates/assets/videojs-ogvjs.js
-src/youtube2zim/templates/assets/videojs/
-src/youtube2zim/templates/assets/polyfills.js
-src/youtube2zim/templates/assets/webp-hero.bundle.js
+scraper/src/youtube2zim/templates/assets/chosen/
+scraper/src/youtube2zim/templates/assets/jquery.min.js
+scraper/src/youtube2zim/templates/assets/ogvjs/
+scraper/src/youtube2zim/templates/assets/videojs-ogvjs.js
+scraper/src/youtube2zim/templates/assets/videojs/
+scraper/src/youtube2zim/templates/assets/polyfills.js
+scraper/src/youtube2zim/templates/assets/webp-hero.bundle.js
 
 # output dir
 output

CHANGELOG

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Move scraper files to `scraper` subfolder and update workflows
+
 ## [2.3.0] - 2024-05-22
 
 ### Added

Dockerfile

@@ -14,24 +14,25 @@ RUN apt-get update \
       pip
 
 # Custom entrypoint
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY scraper/entrypoint.sh /usr/local/bin/entrypoint.sh
 ENTRYPOINT ["entrypoint.sh"]
 RUN mkdir -p /output
 WORKDIR /output
 
 # Copy pyproject.toml and its dependencies
-COPY pyproject.toml README.md openzim.toml /src/
-COPY src/youtube2zim/__about__.py /src/src/youtube2zim/__about__.py
+COPY README.md /src/
+COPY scraper/pyproject.toml scraper/openzim.toml /src/scraper/
+COPY scraper/src/youtube2zim/__about__.py /src/scraper/src/youtube2zim/__about__.py
 
 # Install Python dependencies
-RUN pip install --no-cache-dir /src
+RUN pip install --no-cache-dir /src/scraper
 
 # Copy code + associated artifacts
-COPY src /src/src
+COPY scraper/src /src/scraper/src
 COPY *.md LICENSE CHANGELOG /src/
 
 # Install + cleanup
-RUN pip install --no-cache-dir /src \
- && rm -rf /src
+RUN pip install --no-cache-dir /src/scraper \
+ && rm -rf /src/scraper
 
 CMD ["youtube2zim", "--help"]

README.md

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "youtube2zim"
 requires-python = ">=3.12,<3.13"
 description = "Make ZIM file from a Youtube channel, user or playlist(s)"
-readme = "README.md"
+readme = "../README.md"
 dependencies = [
   "python-slugify==3.0.3",
   "yt-dlp",                      # youtube-dl should be updated as frequently as possible

contrib/reencode_low_quality.py scraper/contrib/reencode_low_quality.py

@@ -33,4 +33,3 @@ msgstr "Page {current}/{total}"
 #: youtube2zim/scraper.py:872
 msgid "Back to top"
 msgstr "Haut"
-

contrib/video_encoding_tester.py scraper/contrib/video_encoding_tester.py

@@ -198,7 +198,7 @@ def handle_single_zim(self):
 
     @staticmethod
     def compute_format(playlist, fmt):
-        return fmt.format(**playlist.__dict__(), **{"period": "{period}"})
+        return fmt.format(**playlist.to_dict(), **{"period": "{period}"})
 
     def fetch_metadata(self):
         """retrieves and loads metadata from --metadata-from"""

entrypoint.sh scraper/entrypoint.sh

@@ -43,7 +43,7 @@ def from_id(cls, playlist_id):
             creator_name=playlist_json["snippet"]["channelTitle"],
         )
 
-    def __dict__(self):
+    def to_dict(self):
         return {
             "playlist_id": self.playlist_id,
             "title": self.title,