Allow tracker pause and resume

codecarbon/emissions_tracker.py

@@ -214,6 +214,7 @@ def __init__(
         self._measure_occurrence: int = 0
         self._cloud = None
         self._previous_emissions = None
+        self.pause_count = 0
 
         if isinstance(self._gpu_ids, str):
             self._gpu_ids = parse_gpu_ids(self._gpu_ids)
@@ -302,7 +303,7 @@ def start(self) -> None:
         self._scheduler.start()
 
     @suppress(Exception)
-    def stop(self) -> Optional[float]:
+    def _stop(self, destroy: bool) -> Optional[float]:
         """
         Stops tracking the experiment
         :return: CO2 emissions in kgs
@@ -311,7 +312,20 @@ def stop(self) -> Optional[float]:
             logger.error("Need to first start the tracker")
             return None
 
-        self._scheduler.shutdown()
+        if destroy:
+            self._scheduler.shutdown()
+        else:
+            self._scheduler.pause()
+
+        if self._save_to_file and os.path.exists(
+            os.path.join(self._output_dir, self._output_file)
+        ):
+            self.pause_count += 1
+            filename, extension = os.path.splitext(self._output_file)
+            new_filename = f"{filename}_{self.pause_count}"
+            self.persistence_objs[0] = FileOutput(
+                os.path.join(self._output_dir, f"{new_filename}{extension}")
+            )
 
         # Run to calculate the power used from last
         # scheduled measurement to shutdown
@@ -328,6 +342,33 @@ def stop(self) -> Optional[float]:
         self.final_emissions = emissions_data.emissions
         return emissions_data.emissions
 
+    @suppress(Exception)
+    def pause(self):
+        """
+        Temporarily halts tracking the experiment.
+        Logs intermediate experiment tracking results.
+        :return: CO2 emissions in kgs
+        """
+        # check if csv already exists
+        return self._stop(destroy=False)
+
+    @suppress(Exception)
+    def resume(self):
+        """
+        Resumes previously paused tracking
+        """
+        logger.info("Restarting tracker")
+        self._last_measured_time = self._start_time = time.time()
+        self._scheduler.resume()
+
+    @suppress(Exception)
+    def stop(self) -> Optional[float]:
+        """
+        Stops tracking the experiment
+        :return: CO2 emissions in kgs
+        """
+        return self._stop(destroy=True)
+
     def _prepare_emissions_data(self, delta=False) -> EmissionsData:
         """
         :delta: True to return only the delta comsumption since last call