update recipes in each deployment

application/lib/mechanize.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python3
 # -*- coding:utf-8 -*-
 #mechanize 的兼容层，让其他recipe可以不修改就可以使用
-from urlopener import UrlOpener as Browser, Request
-
-
+from urlopener import UrlOpener as Browser, UrlRequest as Request

docker/Dockerfile

@@ -5,15 +5,29 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
 USER root
-RUN mkdir -p /usr/kindleear
+RUN mkdir -p /usr/kindleear /data
 WORKDIR /usr/kindleear
-COPY ./config.py ./tools/update_req.py ./
-COPY ./tools/mp3cat/*mp3cat /usr/kindleear/tools/
+COPY ./application/ ./application/
+COPY ./docker/ ./docker/
+COPY ./tools/ ./tools/
+COPY ./main.py ./config.py ./
+RUN python /usr/kindleear/tools/update_req.py docker $TARGETPLATFORM
+
+#update recipes
+RUN apk add --no-cache git && \
+    git clone --depth 1 https://github.com/kovidgoyal/calibre && \
+    rm -f /usr/kindleear/application/recipes/builtin_recipes.* && \
+    mv -f /usr/kindleear/calibre/recipes/*.recipe /usr/kindleear/application/recipes/ && \
+    pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt && \
+    python /usr/kindleear/tools/archive_builtin_recipes.py -q && \
+    rm -f /usr/kindleear/application/recipes/*.recipe
+
+#choose mp3cat by arch
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
         rm -f /usr/kindleear/tools/mp3cat; \
         mv /usr/kindleear/tools/arm64_mp3cat /usr/kindleear/tools/mp3cat; \
     fi
-RUN python update_req.py docker $TARGETPLATFORM
 
 #step 2
 FROM --platform=$TARGETPLATFORM python:3.9.19-alpine
@@ -35,6 +49,7 @@ RUN pip install --upgrade pip && \
     chmod +x /usr/local/bin/mp3cat /usr/local/bin/run_docker.sh
 
 COPY ./application/ ./application/
+COPY --from=builder /usr/kindleear/application/recipes/builtin_recipes.* ./application/recipes/
 
 EXPOSE 8000
 

tools/archive_builtin_recipes.py

@@ -87,7 +87,7 @@ def serialize_collection(mapping_of_recipe_classes):
 </recipe_collection>'''.encode()
 
 #old_recipes_dir: recipes extracted from builtin_recipes.zip
-def serialize_builtin_recipes(recipes_dir, old_recipes_dir):
+def serialize_builtin_recipes(recipes_dir, old_recipes_dir, quiet_print):
     recipe_mapping = {}
     skipped = 0
     for recipe_id, f in iterate_recipe_files(recipes_dir, old_recipes_dir):
@@ -96,7 +96,7 @@ def serialize_builtin_recipes(recipes_dir, old_recipes_dir):
             try:
                 recipe_class = compile_recipe(stream.read())
             except Exception as e:
-                print(f'Failed to compile: {f}, skipped. Error:\n{e}')
+                quiet_print(f'Failed to compile: {f}, skipped. Error:\n{e}')
                 skipped += 1
                 continue
         if recipe_class is not None:
@@ -123,7 +123,8 @@ def newer(targets, sources):
 
     return newest_source > oldest_target
 
-def archive_builtin_recipes(recipes_dir):
+def archive_builtin_recipes(recipes_dir, quiet):
+    quiet_print = lambda x: x if quiet else print
     xml_name = os.path.join(recipes_dir, 'builtin_recipes.xml')
     zip_name = os.path.join(recipes_dir, 'builtin_recipes.zip')
     try:
@@ -133,7 +134,7 @@ def archive_builtin_recipes(recipes_dir):
 
     old_recipes_dir = ''
     if files and os.path.exists(zip_name):
-        print('\tFound builtin_recipes.zip, extracting')
+        quiet_print('\tFound builtin_recipes.zip, extracting')
         old_recipes_dir = tempfile.mkdtemp(prefix='builtin_recipes_')
         with zipfile.ZipFile(zip_name, 'r') as zfile:
             zfile.extractall(old_recipes_dir)
@@ -142,8 +143,8 @@ def archive_builtin_recipes(recipes_dir):
 
     created = []
     if newer(xml_name, files):
-        print('\tCreating builtin_recipes.xml')
-        num, skipped, xml = serialize_builtin_recipes(recipes_dir, old_recipes_dir)
+        quiet_print('\tCreating builtin_recipes.xml')
+        num, skipped, xml = serialize_builtin_recipes(recipes_dir, old_recipes_dir, quiet_print)
         if num:
             with open(xml_name, 'wb') as f:
                 f.write(xml)
@@ -164,7 +165,7 @@ def archive_builtin_recipes(recipes_dir):
     #            files.append(f)
 
     if newer(zip_name, files):
-        print('\tCreating builtin_recipes.zip')
+        quiet_print('\tCreating builtin_recipes.zip')
         with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_STORED) as zf:
             for n in sorted(files, key=os.path.basename):
                 with open(n, 'rb') as f:
@@ -180,13 +181,18 @@ def archive_builtin_recipes(recipes_dir):
     return created
 
 def main():
-    print('\n---------------------------------------------------')
-    print('The script will create the "builtin_recipes.xml" and "builtin_recipes.zip"\n'
-        'from the recipes directory.')
-    print('---------------------------------------------------')
-    confirm = input('Press Y to continue, or N to exit : ')
+    quiet = False
+    if len(sys.argv) > 1 and sys.argv[1] == '-q':
+        confirm = 'y'
+        quiet = True
+    else:
+        print('\n---------------------------------------------------')
+        print('The script will create the "builtin_recipes.xml" and "builtin_recipes.zip"\n'
+            'from the recipes directory.')
+        print('---------------------------------------------------')
+        confirm = input('Press Y to continue, or N to exit : ')
     if confirm.strip().lower() == 'y':
-        created = archive_builtin_recipes(recipesDir)
+        archive_builtin_recipes(recipesDir, quiet)
 
 if __name__ == '__main__':
     sys.exit(main())

tools/gae_deploy.sh

@@ -16,34 +16,52 @@ fi
 SH_DIR="$(dirname "$0")"
 KE_DIR="$(dirname "$SH_DIR")"
 
-if python ${SH_DIR}/update_req.py gae[$@]; then
-  echo "Enabling APIs..."
-  gcloud services enable firestore.googleapis.com datastore.googleapis.com \
-  cloudtasks.googleapis.com cloudscheduler.googleapis.com appenginereporting.googleapis.com \
-  artifactregistry.googleapis.com cloudbuild.googleapis.com cloudtrace.googleapis.com \
-  containerregistry.googleapis.com firebaserules.googleapis.com logging.googleapis.com \
-  pubsub.googleapis.com storage-api.googleapis.com
-  gcloud logging buckets update _Default --location=global --retention-days=7
-
-  gcloud beta app deploy --version=1 ${KE_DIR}/app.yaml ${KE_DIR}/worker.yaml
-  gcloud beta app deploy --quiet --version=1 ${KE_DIR}/cron.yaml
-  gcloud beta app deploy --quiet --version=1 ${KE_DIR}/queue.yaml
-  gcloud beta app deploy --quiet --version=1 ${KE_DIR}/dispatch.yaml
-
-  echo -e "\nDeleting repositories (build cache)..."
-  REPOS=$(gcloud beta artifacts repositories list 2>/dev/null)
-  REPO=$(echo "$REPOS" | grep -oP '(?<=REPOSITORY: ).*')
-  LOC=$(echo "$REPOS" | grep -oP '(?<=LOCATION: ).*')
-  if [[ -n "$REPO" && -n "$LOC" ]]; then
-    gcloud beta artifacts repositories delete $REPO --location=$LOC --quiet
-  else
-    echo "REPO or LOC is empty. Skipping delete command."
-    echo "You can delete manually artifacts in https://console.cloud.google.com/artifacts"
-  fi
-
-  echo "The deployment is completed."
-  echo "The access address is: https://$GOOGLE_CLOUD_PROJECT.appspot.com"
-else
+if ! python3 ${SH_DIR}/update_req.py gae[$@]; then
   echo "Error: Update requirements failed. The deployment is terminated."
   exit 1
 fi
+
+echo -e "\nDownloading the latest recipe collection..."
+rm -rf ${KE_DIR}/calibre
+git clone --depth 1 https://github.com/kovidgoyal/calibre ${KE_DIR}/calibre
+rm -f ${KE_DIR}/application/recipes/builtin_recipes.*
+mv -f ${KE_DIR}/calibre/recipes/*.recipe ${KE_DIR}/application/recipes/
+python3 -m venv ${KE_DIR}/myenv
+source ${KE_DIR}/myenv/bin/activate
+pip3 install --upgrade pip
+pip3 install --no-cache-dir -q -r ${KE_DIR}/requirements.txt
+python3 ${KE_DIR}/tools/archive_builtin_recipes.py -q
+rm -f ${KE_DIR}/application/recipes/*.recipe > /dev/null 2>&1
+deactivate
+rm -rf ${KE_DIR}/myenv
+
+echo -e "\nEnabling APIs..."
+gcloud services enable firestore.googleapis.com datastore.googleapis.com \
+cloudtasks.googleapis.com cloudscheduler.googleapis.com appenginereporting.googleapis.com \
+artifactregistry.googleapis.com cloudbuild.googleapis.com cloudtrace.googleapis.com \
+containerregistry.googleapis.com firebaserules.googleapis.com logging.googleapis.com \
+pubsub.googleapis.com storage-api.googleapis.com
+gcloud logging buckets update _Default --location=global --retention-days=7
+
+if ! gcloud beta app deploy --version=1 ${KE_DIR}/app.yaml ${KE_DIR}/worker.yaml; then
+  echo "The deployment is terminated."
+  exit 1
+fi
+
+gcloud beta app deploy --quiet --version=1 ${KE_DIR}/cron.yaml
+gcloud beta app deploy --quiet --version=1 ${KE_DIR}/queue.yaml
+gcloud beta app deploy --quiet --version=1 ${KE_DIR}/dispatch.yaml
+
+echo -e "\nDeleting repositories (build cache)..."
+REPOS=$(gcloud beta artifacts repositories list 2>/dev/null)
+REPO=$(echo "$REPOS" | grep -oP '(?<=REPOSITORY: ).*')
+LOC=$(echo "$REPOS" | grep -oP '(?<=LOCATION: ).*')
+if [[ -n "$REPO" && -n "$LOC" ]]; then
+  gcloud beta artifacts repositories delete $REPO --location=$LOC --quiet
+else
+  echo "REPO or LOC is empty. Skipping delete command."
+  echo "You can delete manually artifacts in https://console.cloud.google.com/artifacts"
+fi
+
+echo "The deployment is completed."
+echo "The access address is: https://$GOOGLE_CLOUD_PROJECT.appspot.com"