Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Logging scripts micha #441

Merged
merged 22 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
e421fc0
modif du fichier env_linux.yml : version python 10
MichaBiriuchinskii Jun 16, 2023
2a6d7a0
adding logger for annotations.py
MichaBiriuchinskii Jun 22, 2023
e7d1b15
annotations + cmdline update
MichaBiriuchinskii Jun 23, 2023
69204bc
setting up logger config for annotations.py and cmdline.py
LoannPeurey Jun 23, 2023
5babfee
.gitignore to ignore ._*
MichaBiriuchinskii Jun 23, 2023
d1cc29c
removing the exemple Critical errors
MichaBiriuchinskii Jun 23, 2023
6d26d5a
modifs of the cmsline
MichaBiriuchinskii Jun 26, 2023
a7c416c
Logging: annotations.py completed
MichaBiriuchinskii Jul 12, 2023
9056a56
Loggin: cmdline.py
MichaBiriuchinskii Jul 12, 2023
40d0bc8
Logging: converters.py completed
MichaBiriuchinskii Jul 12, 2023
3f5c8a4
Logging: processors.py completed
MichaBiriuchinskii Jul 12, 2023
667c4c9
Logging: samplers.py completed
MichaBiriuchinskii Jul 13, 2023
74d5cc0
Logging: zooniverse.py completed
MichaBiriuchinskii Jul 13, 2023
79fc1ec
Converters.py was modified my mistake which caused the test-fail
MichaBiriuchinskii Jul 13, 2023
70545d7
test to identify failures in cli calls only on girhub testing
LoannPeurey Jul 18, 2023
afb1326
Revert "test to identify failures in cli calls only on girhub testing"
LoannPeurey Jul 19, 2023
6e038df
ignore pycharm project folders
LoannPeurey Jul 19, 2023
0df00a1
add colorlog to dependencies
LoannPeurey Jul 19, 2023
bf00954
use .items as .iteritems will get deprecated
LoannPeurey Nov 13, 2023
5ac4432
allow pandas up to 1.5.3
LoannPeurey Nov 13, 2023
6e19847
CHANGELOG
LoannPeurey Nov 13, 2023
9eb90f3
Merge branch 'master' into logging_scripts_micha
LoannPeurey Nov 13, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/output
.DS_Store
._*

# Byte-compiled / optimized / DLL files
__pycache__/
Expand Down Expand Up @@ -80,6 +81,9 @@ instance/
# Sphinx documentation
docs/_build/

# PyCharm
/.idea/

# PyBuilder
target/

Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.

### Changed

- the output of the CLI in the terminal is now handled by the logger module and not by print statements
- validating a dataset now results in warnings for broken symlinks and no errors anymore (#425)
- validation with recordings existing but for which mediainfo can't read the sample rate no longer fail but outputs a warning.

Expand Down
44 changes: 21 additions & 23 deletions ChildProject/annotations.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import sys
import traceback
from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
import logging

from . import __version__
from .projects import ChildProject
Expand All @@ -16,6 +17,11 @@
from .utils import Segment, intersect_ranges, path_is_parent, TimeInterval, series_to_datetime, find_lines_involved_in_overlap


# Create a logger for the module (file)
logger_annotations = logging.getLogger(__name__)
# messages are propagated to the higher level logger (ChildProject), used in cmdline.py
logger_annotations.propagate = True

class AnnotationManager:
INDEX_COLUMNS = [
IndexColumn(
Expand Down Expand Up @@ -353,12 +359,7 @@ def read(self) -> Tuple[List[str], List[str]]:
return errors, warnings

def validate_annotation(self, annotation: dict) -> Tuple[List[str], List[str]]:
print(
"validating {} from {}...".format(
annotation["annotation_filename"], annotation["set"]
)
)

logger_annotations.info("Validating %s from %s...", annotation["annotation_filename"], annotation["set"])
segments = IndexTable(
"segments",
path=os.path.join(
Expand Down Expand Up @@ -490,11 +491,12 @@ def _import_annotation(
if self.annotations[(self.annotations['set'] == annotation['set']) &
(self.annotations['annotation_filename'] == annotation_filename)].shape[0] > 0:
if overwrite_existing:
print(f"Warning: annotation file {output_filename} will be overwritten")
logger_annotations.warning("Annotation file %s will be overwritten", output_filename)

else:
error_filename = output_filename.replace('\\','/')
annotation["error"] = f"annotation file {error_filename} already exists, to reimport it, use the overwrite_existing flag"
print(f"Error: {annotation['error']}")
logger_annotations.error("Error: %s", annotation['error'])
annotation["imported_at"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return annotation

Expand All @@ -509,7 +511,8 @@ def _import_annotation(
if ovl_annots.shape[0] > 0:
array_tup = list(ovl_annots[['set','recording_filename','range_onset', 'range_offset']].itertuples(index=False, name=None))
annotation["error"] = f"importation for set <{annotation['set']}> recording <{annotation['recording_filename']}> from {annotation['range_onset']} to {annotation['range_offset']} cannot continue because it overlaps with these existing annotation lines: {array_tup}"
print(f"Error: {annotation['error']}")
logger_annotations.error("Error: %s", annotation['error'])
#(f"Error: {annotation['error']}")
annotation["imported_at"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return annotation

Expand Down Expand Up @@ -541,10 +544,7 @@ def _import_annotation(
)
except:
annotation["error"] = traceback.format_exc()
print(
"an error occured while processing '{}'".format(path), file=sys.stderr
)
print(traceback.format_exc(), file=sys.stderr)
logger_annotations.error("An error occurred while processing '%s'", path, exc_info=True)

if df is None or not isinstance(df, pd.DataFrame):
annotation["imported_at"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Expand Down Expand Up @@ -654,9 +654,7 @@ def import_annotations(

builtin = input_processed[input_processed["format"].isin(converters.keys())]
if not builtin["format"].map(lambda f: converters[f].THREAD_SAFE).all():
print(
"warning: some of the converters do not support multithread importation; running on 1 thread"
)
logger_annotations.warning("warning: some of the converters do not support multithread importation; running on 1 thread")
threads = 1

#if the input to import has overlaps in it, raise an error immediately, nothing will be imported
Expand Down Expand Up @@ -688,14 +686,15 @@ def import_annotations(
axis=1,
inplace=True,
)

if 'error' in imported.columns:
errors = imported[~imported["error"].isnull()]
imported = imported[imported["error"].isnull()]
#when errors occur, separate them in a different csv in extra
if errors.shape[0] > 0:
output = os.path.join(self.project.path, "extra","errors_import_{}.csv".format(datetime.datetime.now().strftime("%Y%m%d-%H%M%S")))
errors.to_csv(output, index=False)
print(f"Errors summary exported to {output}")
logger_annotations.info("Errors summary exported to %s", output)
else:
errors = None

Expand All @@ -709,7 +708,7 @@ def import_annotations(
sets = set(input_processed['set'].unique())
outdated_sets = self._check_for_outdated_merged_sets(sets= sets)
for warning in outdated_sets:
print("warning: {}".format(warning))
logger_annotations.warning("warning: %s", warning)

return (imported, errors)

Expand Down Expand Up @@ -763,19 +762,18 @@ def remove_set(self, annotation_set: str, recursive: bool = False):
path = os.path.join(
self.project.path, "annotations", annotation_set, "converted"
)

try:
rmtree(path)
except:
print("could not delete '{}', as it does not exist (yet?)".format(path))
logger_annotations.info("could not delete '%s', as it does not exist (yet?)", path)
pass

self.annotations = self.annotations[self.annotations["set"] != annotation_set]
self.write()

outdated_sets = self._check_for_outdated_merged_sets(sets= {annotation_set})
for warning in outdated_sets:
print("warning: {}".format(warning))
logger_annotations.warning("warning: %s", warning)

def rename_set(
self,
Expand Down Expand Up @@ -1419,7 +1417,7 @@ def get_within_ranges(
)

if missing_data == "warn":
print(f"warning: {error_message}")
logger_annotations.warning("warning: %s", error_message)
else:
raise Exception(error_message)

Expand Down
Loading
Loading