monitoring: better time stamps

Co-Authored-by: Peter Weber <[email protected]>

rero_ils/modules/contributions/tasks.py

@@ -111,5 +111,5 @@ def update_contributions(pids=None, dbcommit=True, reindex=True, verbose=False,
             error_pids.append(pid)
         log[msg] += 1
     if timestamp:
-        set_timestamp('update_contributions', msg=log)
+        set_timestamp('update_contributions', **log)
     return log, error_pids

rero_ils/modules/documents/tasks.py

@@ -21,6 +21,8 @@
 
 from celery import shared_task
 
+from rero_ils.modules.utils import set_timestamp
+
 from .utils_mef import ReplaceMefIdentifiedByContribution, \
     ReplaceMefIdentifiedBySubjects
 from ..contributions.api import Contribution
@@ -85,6 +87,8 @@ def replace_idby_contribution(verbose=False, details=False, timestamp=True):
         replace_contribution.set_timestamp()
     if details:
         replace_contribution.print_details()
+    set_timestamp('replace_idby_contribution',
+                  count=replace_contribution.counts_len)
     return replace_contribution.counts_len
 
 
@@ -103,4 +107,5 @@ def replace_idby_subjects(verbose=False, details=False, timestamp=True):
         replace_subjects.set_timestamp()
     if details:
         replace_subjects.print_details()
+    set_timestamp('replace_idby_subjects', count=replace_subjects.counts_len)
     return replace_subjects.counts_len

rero_ils/modules/ebooks/tasks.py

@@ -25,7 +25,7 @@
 
 from .utils import create_document_holding, update_document_holding
 from ..documents.api import Document, DocumentsSearch
-from ..utils import do_bulk_index, get_schema_for_resource
+from ..utils import do_bulk_index, get_schema_for_resource, set_timestamp
 
 
 @shared_task(ignore_result=True)
@@ -76,6 +76,8 @@ def create_records(records):
     current_app.logger.info(
         f'create_records: {n_updated} updated, {n_created} new'
     )
+    set_timestamp('ebooks_create_records', created=n_created,
+                  updated=n_updated)
     return n_created, n_updated
 
 
@@ -106,4 +108,5 @@ def delete_records(records):
         except Exception as err:
             current_app.logger.error(f'EBOOKS DELETE RECORDS: {err} {record}')
     current_app.logger.info(f'delete_records: {count}')
+    set_timestamp('ebooks_delete_records', deleted=count)
     return count

rero_ils/modules/holdings/tasks.py

@@ -36,11 +36,13 @@ def delete_standard_holdings_having_no_items():
         .filter('term', items_count=0) \
         .source('pid')
 
+    deleted = 0
     for hit in es_query.scan():
         record = Holding.get_record_by_pid(hit.pid)
         record.delete(force=False, dbcommit=True, delindex=True)
+        deleted += 1
 
     current_app.logger.debug("Ending delete_standard_holdings_having_no_items")
     msg = f'Number of removed holdings: {es_query.count()}'
-    set_timestamp('holdings-deletion', msg=msg)
+    set_timestamp('holdings-deletion', deleted=deleted)
     return msg

rero_ils/modules/items/tasks.py

@@ -74,10 +74,9 @@ def clean_obsolete_temporary_item_types():
     clean the temporary item_type informations. Update item into database to
     commit change
     """
-    current_app.logger.debug("Starting clean_obsolete_temporary_item_types()"
-                             " tasks ...")
-    msg = 'Nothing to do'
+    counter = 0
     for item in Item.get_items_with_obsolete_temporary_item_type():
+        counter += 1
         # logger information
         tmp_itty_data = item['temporary_item_type']
         tmp_itty = extracted_data_from_ref(tmp_itty_data['$ref'], 'record')
@@ -95,8 +94,9 @@ def clean_obsolete_temporary_item_types():
         # remove the obsolete data
         del item['temporary_item_type']
         item.replace(data=item, dbcommit=True, reindex=True)
-    set_timestamp('clean_obsolete_temporary_item_types', msg=msg)
-    current_app.logger.debug("Ending clean_obsolete_temporary_item_types()")
+    count = {'deleted': counter}
+    set_timestamp('clean_obsolete_temporary_item_types', **count)
+    return count
 
 
 @shared_task

rero_ils/modules/loans/tasks.py

@@ -40,9 +40,8 @@ def loan_anonymizer(dbcommit=True, reindex=True):
             loan.anonymize(loan, dbcommit=dbcommit, reindex=reindex)
             counter += 1
 
-    msg = f'number_of_loans_anonymized: {counter}'
-    set_timestamp('anonymize-loans', msg=msg)
-    return msg
+    set_timestamp('anonymize-loans', count=counter)
+    return counter
 
 
 @shared_task(ignore_result=True)
@@ -65,4 +64,6 @@ def cancel_expired_request_task(tstamp=None):
         )
         if actions.get('cancel', {}).get('pid') == loan.pid:
             total_cancelled_loans += 1
+    set_timestamp('cancel-expired-request-task', total=total_loans_counter,
+                  cancelled=total_cancelled_loans)
     return total_loans_counter, total_cancelled_loans

tests/ui/loans/test_loans_api.py

@@ -229,8 +229,8 @@ def test_anonymizer_job(
 
     # So a this time, if we run the `loan_anonymizer` task, none loan cannot
     # be anonymized --> return should be 0
-    msg = loan_anonymizer(dbcommit=True, reindex=True)
-    assert msg == 'number_of_loans_anonymized: 0'
+    count = loan_anonymizer(dbcommit=True, reindex=True)
+    assert count == 0
 
     # We will now update the loan `transaction_date` to 1 year ago and close
     # all open transactions about it.
@@ -251,7 +251,7 @@ def test_anonymizer_job(
     # should be anonymize.
     count = len(list(Loan.get_anonymized_candidates()))
     msg = loan_anonymizer(dbcommit=True, reindex=True)
-    assert msg == f'number_of_loans_anonymized: {count}'
+    assert msg == count
 
 
 @mock.patch.object(Loan, 'can_anonymize', mock.MagicMock(return_value=False))