diff --git a/application/lib/mechanize.py b/application/lib/mechanize.py index 23c18389e..561d66ebf 100644 --- a/application/lib/mechanize.py +++ b/application/lib/mechanize.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 # -*- coding:utf-8 -*- #mechanize 的兼容层,让其他recipe可以不修改就可以使用 -from urlopener import UrlOpener as Browser, Request - - +from urlopener import UrlOpener as Browser, UrlRequest as Request diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c43dd607..2165ff330 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,15 +5,29 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 USER root -RUN mkdir -p /usr/kindleear +RUN mkdir -p /usr/kindleear /data WORKDIR /usr/kindleear -COPY ./config.py ./tools/update_req.py ./ -COPY ./tools/mp3cat/*mp3cat /usr/kindleear/tools/ +COPY ./application/ ./application/ +COPY ./docker/ ./docker/ +COPY ./tools/ ./tools/ +COPY ./main.py ./config.py ./ +RUN python /usr/kindleear/tools/update_req.py docker $TARGETPLATFORM + +#update recipes +RUN apk add --no-cache git && \ + git clone --depth 1 https://github.com/kovidgoyal/calibre && \ + rm -f /usr/kindleear/application/recipes/builtin_recipes.* && \ + mv -f /usr/kindleear/calibre/recipes/*.recipe /usr/kindleear/application/recipes/ && \ + pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt && \ + python /usr/kindleear/tools/archive_builtin_recipes.py -q && \ + rm -f /usr/kindleear/application/recipes/*.recipe + +#choose mp3cat by arch RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ rm -f /usr/kindleear/tools/mp3cat; \ mv /usr/kindleear/tools/arm64_mp3cat /usr/kindleear/tools/mp3cat; \ fi -RUN python update_req.py docker $TARGETPLATFORM #step 2 FROM --platform=$TARGETPLATFORM python:3.9.19-alpine @@ -35,6 +49,7 @@ RUN pip install --upgrade pip && \ chmod +x /usr/local/bin/mp3cat /usr/local/bin/run_docker.sh COPY ./application/ ./application/ +COPY --from=builder /usr/kindleear/application/recipes/builtin_recipes.* ./application/recipes/ EXPOSE 8000 diff --git a/tools/archive_builtin_recipes.py b/tools/archive_builtin_recipes.py index 47767d607..5ad6410d7 100644 --- a/tools/archive_builtin_recipes.py +++ b/tools/archive_builtin_recipes.py @@ -87,7 +87,7 @@ def serialize_collection(mapping_of_recipe_classes): '''.encode() #old_recipes_dir: recipes extracted from builtin_recipes.zip -def serialize_builtin_recipes(recipes_dir, old_recipes_dir): +def serialize_builtin_recipes(recipes_dir, old_recipes_dir, quiet_print): recipe_mapping = {} skipped = 0 for recipe_id, f in iterate_recipe_files(recipes_dir, old_recipes_dir): @@ -96,7 +96,7 @@ def serialize_builtin_recipes(recipes_dir, old_recipes_dir): try: recipe_class = compile_recipe(stream.read()) except Exception as e: - print(f'Failed to compile: {f}, skipped. Error:\n{e}') + quiet_print(f'Failed to compile: {f}, skipped. Error:\n{e}') skipped += 1 continue if recipe_class is not None: @@ -123,7 +123,8 @@ def newer(targets, sources): return newest_source > oldest_target -def archive_builtin_recipes(recipes_dir): +def archive_builtin_recipes(recipes_dir, quiet): + quiet_print = lambda x: x if quiet else print xml_name = os.path.join(recipes_dir, 'builtin_recipes.xml') zip_name = os.path.join(recipes_dir, 'builtin_recipes.zip') try: @@ -133,7 +134,7 @@ def archive_builtin_recipes(recipes_dir): old_recipes_dir = '' if files and os.path.exists(zip_name): - print('\tFound builtin_recipes.zip, extracting') + quiet_print('\tFound builtin_recipes.zip, extracting') old_recipes_dir = tempfile.mkdtemp(prefix='builtin_recipes_') with zipfile.ZipFile(zip_name, 'r') as zfile: zfile.extractall(old_recipes_dir) @@ -142,8 +143,8 @@ def archive_builtin_recipes(recipes_dir): created = [] if newer(xml_name, files): - print('\tCreating builtin_recipes.xml') - num, skipped, xml = serialize_builtin_recipes(recipes_dir, old_recipes_dir) + quiet_print('\tCreating builtin_recipes.xml') + num, skipped, xml = serialize_builtin_recipes(recipes_dir, old_recipes_dir, quiet_print) if num: with open(xml_name, 'wb') as f: f.write(xml) @@ -164,7 +165,7 @@ def archive_builtin_recipes(recipes_dir): # files.append(f) if newer(zip_name, files): - print('\tCreating builtin_recipes.zip') + quiet_print('\tCreating builtin_recipes.zip') with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_STORED) as zf: for n in sorted(files, key=os.path.basename): with open(n, 'rb') as f: @@ -180,13 +181,18 @@ def archive_builtin_recipes(recipes_dir): return created def main(): - print('\n---------------------------------------------------') - print('The script will create the "builtin_recipes.xml" and "builtin_recipes.zip"\n' - 'from the recipes directory.') - print('---------------------------------------------------') - confirm = input('Press Y to continue, or N to exit : ') + quiet = False + if len(sys.argv) > 1 and sys.argv[1] == '-q': + confirm = 'y' + quiet = True + else: + print('\n---------------------------------------------------') + print('The script will create the "builtin_recipes.xml" and "builtin_recipes.zip"\n' + 'from the recipes directory.') + print('---------------------------------------------------') + confirm = input('Press Y to continue, or N to exit : ') if confirm.strip().lower() == 'y': - created = archive_builtin_recipes(recipesDir) + archive_builtin_recipes(recipesDir, quiet) if __name__ == '__main__': sys.exit(main()) diff --git a/tools/gae_deploy.sh b/tools/gae_deploy.sh index 7d9717d24..7224892cf 100644 --- a/tools/gae_deploy.sh +++ b/tools/gae_deploy.sh @@ -16,34 +16,52 @@ fi SH_DIR="$(dirname "$0")" KE_DIR="$(dirname "$SH_DIR")" -if python ${SH_DIR}/update_req.py gae[$@]; then - echo "Enabling APIs..." - gcloud services enable firestore.googleapis.com datastore.googleapis.com \ - cloudtasks.googleapis.com cloudscheduler.googleapis.com appenginereporting.googleapis.com \ - artifactregistry.googleapis.com cloudbuild.googleapis.com cloudtrace.googleapis.com \ - containerregistry.googleapis.com firebaserules.googleapis.com logging.googleapis.com \ - pubsub.googleapis.com storage-api.googleapis.com - gcloud logging buckets update _Default --location=global --retention-days=7 - - gcloud beta app deploy --version=1 ${KE_DIR}/app.yaml ${KE_DIR}/worker.yaml - gcloud beta app deploy --quiet --version=1 ${KE_DIR}/cron.yaml - gcloud beta app deploy --quiet --version=1 ${KE_DIR}/queue.yaml - gcloud beta app deploy --quiet --version=1 ${KE_DIR}/dispatch.yaml - - echo -e "\nDeleting repositories (build cache)..." - REPOS=$(gcloud beta artifacts repositories list 2>/dev/null) - REPO=$(echo "$REPOS" | grep -oP '(?<=REPOSITORY: ).*') - LOC=$(echo "$REPOS" | grep -oP '(?<=LOCATION: ).*') - if [[ -n "$REPO" && -n "$LOC" ]]; then - gcloud beta artifacts repositories delete $REPO --location=$LOC --quiet - else - echo "REPO or LOC is empty. Skipping delete command." - echo "You can delete manually artifacts in https://console.cloud.google.com/artifacts" - fi - - echo "The deployment is completed." - echo "The access address is: https://$GOOGLE_CLOUD_PROJECT.appspot.com" -else +if ! python3 ${SH_DIR}/update_req.py gae[$@]; then echo "Error: Update requirements failed. The deployment is terminated." exit 1 fi + +echo -e "\nDownloading the latest recipe collection..." +rm -rf ${KE_DIR}/calibre +git clone --depth 1 https://github.com/kovidgoyal/calibre ${KE_DIR}/calibre +rm -f ${KE_DIR}/application/recipes/builtin_recipes.* +mv -f ${KE_DIR}/calibre/recipes/*.recipe ${KE_DIR}/application/recipes/ +python3 -m venv ${KE_DIR}/myenv +source ${KE_DIR}/myenv/bin/activate +pip3 install --upgrade pip +pip3 install --no-cache-dir -q -r ${KE_DIR}/requirements.txt +python3 ${KE_DIR}/tools/archive_builtin_recipes.py -q +rm -f ${KE_DIR}/application/recipes/*.recipe > /dev/null 2>&1 +deactivate +rm -rf ${KE_DIR}/myenv + +echo -e "\nEnabling APIs..." +gcloud services enable firestore.googleapis.com datastore.googleapis.com \ +cloudtasks.googleapis.com cloudscheduler.googleapis.com appenginereporting.googleapis.com \ +artifactregistry.googleapis.com cloudbuild.googleapis.com cloudtrace.googleapis.com \ +containerregistry.googleapis.com firebaserules.googleapis.com logging.googleapis.com \ +pubsub.googleapis.com storage-api.googleapis.com +gcloud logging buckets update _Default --location=global --retention-days=7 + +if ! gcloud beta app deploy --version=1 ${KE_DIR}/app.yaml ${KE_DIR}/worker.yaml; then + echo "The deployment is terminated." + exit 1 +fi + +gcloud beta app deploy --quiet --version=1 ${KE_DIR}/cron.yaml +gcloud beta app deploy --quiet --version=1 ${KE_DIR}/queue.yaml +gcloud beta app deploy --quiet --version=1 ${KE_DIR}/dispatch.yaml + +echo -e "\nDeleting repositories (build cache)..." +REPOS=$(gcloud beta artifacts repositories list 2>/dev/null) +REPO=$(echo "$REPOS" | grep -oP '(?<=REPOSITORY: ).*') +LOC=$(echo "$REPOS" | grep -oP '(?<=LOCATION: ).*') +if [[ -n "$REPO" && -n "$LOC" ]]; then + gcloud beta artifacts repositories delete $REPO --location=$LOC --quiet +else + echo "REPO or LOC is empty. Skipping delete command." + echo "You can delete manually artifacts in https://console.cloud.google.com/artifacts" +fi + +echo "The deployment is completed." +echo "The access address is: https://$GOOGLE_CLOUD_PROJECT.appspot.com"