From 286299a280b8ff93fa7e1b12462b583ff1418768 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 05:35:19 +0000 Subject: [PATCH 01/11] test --- utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.py b/utils.py index 3f400b1..6266b07 100644 --- a/utils.py +++ b/utils.py @@ -16,9 +16,9 @@ def run(threads, function, files, disable_progress_bar=False): + run_output = [] with tqdm(total=len(files), unit="file", disable=disable_progress_bar) as pbar: - run_output = [] def _run_callback(out): nonlocal run_output @@ -28,7 +28,7 @@ def _run_callback(out): def _error_callback(e): logger.critical( - f"An Error occurred in thread...: {''.join(traceback.format_exception(e))}" + f"An Error occurred in thread {e}...: {''.join(traceback.format_exception(e))}" ) with multiprocessing.Pool(threads) as pool: @@ -43,7 +43,7 @@ def _error_callback(e): pool.close() pool.join() - return run_output + return run_output def get_filelist(path, regex, excluded_files=[]): From 0140da0fffdbe2894a8167c8f9450df92900e4b1 Mon Sep 17 00:00:00 2001 From: Clement <98598399+klementng@users.noreply.github.com> Date: Sat, 13 Jul 2024 13:36:06 +0800 Subject: [PATCH 02/11] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 37d5743..a8152cf 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ name: Docker on: push: - branches: [ "main" ] + branches: [ "main","dev"] # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: From 750611816288c69ed0155ad45aaf0bcd9b4d71a2 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 06:02:33 +0000 Subject: [PATCH 03/11] rewr --- actions.py | 4 +--- main.py | 7 ++++--- postprocessing.py | 7 ++++--- utils.py | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/actions.py b/actions.py index f228111..98a217a 100644 --- a/actions.py +++ b/actions.py @@ -109,7 +109,5 @@ def events_action_delete(ssafile: SSAFile, event: SSAEvent, **kwargs): ssafile.events.remove(event) -def events_misc_remove_miscellaneous_events( - ssafile: SSAFile, **kwargs -): +def events_misc_remove_miscellaneous_events(ssafile: SSAFile, **kwargs): ssafile.remove_miscellaneous_events() diff --git a/main.py b/main.py index 7e8cba0..e467d3f 100755 --- a/main.py +++ b/main.py @@ -19,13 +19,14 @@ def postprocess_subtitles(files, args, pp_args): postprocesser = postprocessing.SubtitleFormatter(pp_args.postprocessing) - run(args.threads, postprocesser.format, files, args.disable_progress_bar) + output = run(args.threads, postprocesser.format, files, args.disable_progress_bar) + output_files = list(chain.from_iterable(output)) if args.exclude_mode == "e+a" and pp_args.exclude_subtitles != None: with open(pp_args.exclude_subtitles, "a") as f: - f.write("\n".join(files)) + f.write("\n".join(output_files)) - return files + return output_files def extract_subtitles(files, args, vid_args): diff --git a/postprocessing.py b/postprocessing.py index aafddbe..7bbb9a8 100644 --- a/postprocessing.py +++ b/postprocessing.py @@ -141,8 +141,9 @@ def format(self, path, save=True): runner = SubtitleRunner(self.workflows[ext]) self.log.debug(f"Formatting subtitle: {path}") - return runner.format(path, save=save) + runner.format(path, save=save) + return path else: - logger.warning("Unsupported format, skipping...") - return None + logger.critical("Unsupported format, skipping...") + raise ValueError(f"File cannot be formatted: {path}") diff --git a/utils.py b/utils.py index 6266b07..426c4c6 100644 --- a/utils.py +++ b/utils.py @@ -34,6 +34,7 @@ def _error_callback(e): with multiprocessing.Pool(threads) as pool: for i in range(pbar.total): + pool.apply_async( function, args=(files[i],), From 464d65480fe01451fb5b6a1528457d636ad18619 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 06:05:41 +0000 Subject: [PATCH 04/11] update pdar on error as well --- utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 426c4c6..2a9cb14 100644 --- a/utils.py +++ b/utils.py @@ -30,7 +30,8 @@ def _error_callback(e): logger.critical( f"An Error occurred in thread {e}...: {''.join(traceback.format_exception(e))}" ) - + pbar.update(1) + with multiprocessing.Pool(threads) as pool: for i in range(pbar.total): From 7fed89964c79e6c53e2b4b2d033fbac60d6d10c6 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 06:37:50 +0000 Subject: [PATCH 05/11] add info --- postprocess.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/postprocess.yml b/postprocess.yml index 1a8a970..0b60dd2 100644 --- a/postprocess.yml +++ b/postprocess.yml @@ -1,3 +1,18 @@ +# DEFINING A TASK: +# Order of operation: +# 1) selectors: Select objects to be processed +# - func(ssafile, **kwargs) -> obj +# 2) filter: filter out object from selectors +# - func(ssafile, item, **kwargs) -> bool +# 3) actions: run actions for each item that was selected +# - func(ssafile, item, **kwargs) -> output +# 4) run misc function +# - func(ssafile,**kwargs) -> output +# +# defining id saves the output of an action to a dict: outputs +# the value can be accessed using: {{outputs['id'][0]['PlayResX']}} +# + ass: tasks: - selectors: From b08637203a5f2bf6c5dee885d713f803e5c23613 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 06:37:57 +0000 Subject: [PATCH 06/11] remove error callback --- utils.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/utils.py b/utils.py index 2a9cb14..604281f 100644 --- a/utils.py +++ b/utils.py @@ -16,6 +16,15 @@ def run(threads, function, files, disable_progress_bar=False): + + def worker(*args, **kwargs): + try: + return function(*args, **kwargs) + except Exception as e: + logger.critical("An error occurred in a worker thread", exc_info=True) + + return None + run_output = [] with tqdm(total=len(files), unit="file", disable=disable_progress_bar) as pbar: @@ -24,24 +33,15 @@ def _run_callback(out): nonlocal run_output nonlocal pbar pbar.update(1) - run_output.append(out) - def _error_callback(e): - logger.critical( - f"An Error occurred in thread {e}...: {''.join(traceback.format_exception(e))}" - ) - pbar.update(1) - + if out is not None: + run_output.append(out) + with multiprocessing.Pool(threads) as pool: for i in range(pbar.total): + pool.apply_async(worker, args=(files[i],), callback=_run_callback) - pool.apply_async( - function, - args=(files[i],), - callback=_run_callback, - error_callback=_error_callback, - ) pool.close() pool.join() From 087994b1f374bcd3111a7568e50ccbeb0cdf8166 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 07:03:30 +0000 Subject: [PATCH 07/11] test --- utils.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/utils.py b/utils.py index 604281f..a4933ff 100644 --- a/utils.py +++ b/utils.py @@ -17,35 +17,31 @@ def run(threads, function, files, disable_progress_bar=False): - def worker(*args, **kwargs): - try: - return function(*args, **kwargs) - except Exception as e: - logger.critical("An error occurred in a worker thread", exc_info=True) - - return None - - run_output = [] + output = [] with tqdm(total=len(files), unit="file", disable=disable_progress_bar) as pbar: - def _run_callback(out): - nonlocal run_output - nonlocal pbar + def _run_callback(result): pbar.update(1) + output.append(result) - if out is not None: - run_output.append(out) + def _error_callback(error): + logger.critical(f"An error occurred in thread....") + traceback.print_exception(type(error), error, error.__traceback__) with multiprocessing.Pool(threads) as pool: - - for i in range(pbar.total): - pool.apply_async(worker, args=(files[i],), callback=_run_callback) + for fp in files: + pool.apply_async( + function, + args=(fp,), + callback=_run_callback, + error_callback=_error_callback, + ) pool.close() pool.join() - return run_output + return output def get_filelist(path, regex, excluded_files=[]): From 5b99759c548351d4d9c07b9f95fabbbfac296ebd Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 07:11:35 +0000 Subject: [PATCH 08/11] test --- postprocessing.py | 3 +-- utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/postprocessing.py b/postprocessing.py index 7bbb9a8..f28e327 100644 --- a/postprocessing.py +++ b/postprocessing.py @@ -145,5 +145,4 @@ def format(self, path, save=True): return path else: - logger.critical("Unsupported format, skipping...") - raise ValueError(f"File cannot be formatted: {path}") + raise ValueError(f"Unsupported format: {path}") diff --git a/utils.py b/utils.py index a4933ff..ef79e31 100644 --- a/utils.py +++ b/utils.py @@ -35,7 +35,7 @@ def _error_callback(error): function, args=(fp,), callback=_run_callback, - error_callback=_error_callback, + # error_callback=_error_callback, ) pool.close() From 8b279cbfdde8b70141f58ba330efe01e3d50cb95 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 07:34:23 +0000 Subject: [PATCH 09/11] test --- utils.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/utils.py b/utils.py index ef79e31..c314f5c 100644 --- a/utils.py +++ b/utils.py @@ -8,6 +8,7 @@ import time import traceback from itertools import chain +import functools from tqdm_loggable.auto import tqdm from tqdm_loggable.tqdm_logging import tqdm_logging @@ -21,18 +22,30 @@ def run(threads, function, files, disable_progress_bar=False): with tqdm(total=len(files), unit="file", disable=disable_progress_bar) as pbar: + def worker(func): + + @functools.wraps(func) + def wrapped_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except: + logger.critical(f"An error occurred in thread....", exc_info=True) + + return wrapped_func + def _run_callback(result): pbar.update(1) output.append(result) - def _error_callback(error): - logger.critical(f"An error occurred in thread....") - traceback.print_exception(type(error), error, error.__traceback__) + # def _error_callback(error): + # pbar.update(1) + # logger.critical(f"An error occurred in thread....") + # traceback.print_exception(type(error), error, error.__traceback__) with multiprocessing.Pool(threads) as pool: for fp in files: pool.apply_async( - function, + worker(function), args=(fp,), callback=_run_callback, # error_callback=_error_callback, From 68ab4fcc174c4cd32a5b27cd208e9e2d7732b2d8 Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 07:45:17 +0000 Subject: [PATCH 10/11] ewr --- utils.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/utils.py b/utils.py index c314f5c..c626ce8 100644 --- a/utils.py +++ b/utils.py @@ -18,20 +18,9 @@ def run(threads, function, files, disable_progress_bar=False): - output = [] - with tqdm(total=len(files), unit="file", disable=disable_progress_bar) as pbar: - def worker(func): - - @functools.wraps(func) - def wrapped_func(*args, **kwargs): - try: - return func(*args, **kwargs) - except: - logger.critical(f"An error occurred in thread....", exc_info=True) - - return wrapped_func + output = [] def _run_callback(result): pbar.update(1) @@ -45,7 +34,7 @@ def _run_callback(result): with multiprocessing.Pool(threads) as pool: for fp in files: pool.apply_async( - worker(function), + function, args=(fp,), callback=_run_callback, # error_callback=_error_callback, @@ -54,7 +43,7 @@ def _run_callback(result): pool.close() pool.join() - return output + return output def get_filelist(path, regex, excluded_files=[]): From ff0d2a0e7c0d912f4120e21c71a57aa95c763bdf Mon Sep 17 00:00:00 2001 From: Clement Date: Sat, 13 Jul 2024 08:00:11 +0000 Subject: [PATCH 11/11] ters --- requirements.txt | 2 +- utils.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 70ded7f..b1f1e8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ babelfish==0.6.1 cachetools==5.3.3 pgsrip==0.1.11 -pysubs2==1.7.2 +pysubs2==1.6.1 PyYAML==6.0.1 tqdm_loggable==0.2 watchdog==4.0.1 \ No newline at end of file diff --git a/utils.py b/utils.py index c626ce8..803db1b 100644 --- a/utils.py +++ b/utils.py @@ -23,13 +23,19 @@ def run(threads, function, files, disable_progress_bar=False): output = [] def _run_callback(result): + nonlocal pbar + nonlocal output + pbar.update(1) output.append(result) - # def _error_callback(error): - # pbar.update(1) - # logger.critical(f"An error occurred in thread....") - # traceback.print_exception(type(error), error, error.__traceback__) + def _error_callback(error): + nonlocal pbar + nonlocal output + + pbar.update(1) + logger.critical(f"An error occurred in thread....") + traceback.print_exception(type(error), error, error.__traceback__) with multiprocessing.Pool(threads) as pool: for fp in files: @@ -37,7 +43,7 @@ def _run_callback(result): function, args=(fp,), callback=_run_callback, - # error_callback=_error_callback, + error_callback=_error_callback, ) pool.close()