implement progress bar (#362)

* Add flow for streamlit

* Fix github action

* Remove Dockerfiles and update setup.py and start.py

* Add streamlit to Dockerfile

* Update dependencies in setup.py

* Update Dockerfile to install streamlit and altair<5

* Fix pip install command in Dockerfile

* Update Dockerfile to install only streamlit package

* Update mlflow version to 2.9.2

* Update streamlit command in README

* Add streamlit app with file upload and model training

* Update file uploader to only accept a single CSV file

* Refactor file uploader and training process

* Add time delay in training loop

* Refactor file uploader and add data generation functionality

* not complited changes for streamlit integration

* refactor the code in order to represent in the output and save logs in appropriate format

* implement progress bar in 'syngen/streamlit_app.py'

* refactor 'streamlit_app.py'

* minor changes

* add checkbox 'Create the accuracy report'

* refactor 'streamlit_app.py'

* Update UI styles

* update Dockerfile, 'setup.py'

* update Dockerfile, 'requirements.txt', 'setup.cfg'

* update 'README.md', revert changes in 'requirements.txt', 'setup.cfg'

* Update fonts

* refactor 'streamlit_app.py'

* delete the folder with a test data

* update 'streamlit_app.py'

* refactor the code in 'streamlit_app.py'

* fix the method 'show_data' of the class StreamlitHandler

* temporary changes in 'streamlit_app.py'

* refactor 'streamlit_app.py'

* add the handling of exceptions in 'streamlit_app.py', add the new page 'Download artifacts'

* update the list of icons used for the option menu in streamlit sidebar

* remove the page 'Download artifacts'

* put the buttons for downloading artifacts in the separate container

* refactor the class StreamlitHandler

* fix the process of collections log messages to log file

* refactor the method 'run_separate' of the class VaeInferHandler, add the expander widget in 'streamlit_app.py', update unit tests

* update the method 'file_sink' of the class Streamlit, minor refactor of the progress bar

* add the logic related to disabling the button 'Generate data' during the running the process of training and inference

* refactor 'streamlit_app.py', update 'VERSION'

* update 'README.md'

* update 'Dockerfile', 'README.md', 'setup.py', refactor the code related to fixing the beuilding process of docker images and the library

* refactor the process of building docker image, and syngen library

* fix the proccess of building the syngen library

* update 'VERSION'

* update 'README.md'

* add the progress bar to the Basic page

* refactor the code related to the previous changes

* refactor the code related to the implementation of the progress bar

* refactor the code related to the previous changes

* refactor the code related to the previous changes

* refactor the code related to the previous changes

* refactor the process of logging

* fix issues raised by 'flake8'

* add more log messages for the progress bar, refactor the code related to these changes

* refactor the method 'update_progress_bar' of the class BaseTest

* refactor 'start.py'

* update 'VERSION'

* update the log message in the method 'generate_report' of the class Report

* refactor the method 'handle' of the class VaeInferHandler, remove the spinner in the Basic UI page, remove the duplication of the success log message in the Basic UI page

* update 'Dockerfile', 'setup.cfg'

* update 'VERSION'

---------

Co-authored-by: Sergio <[email protected]>
Co-authored-by: Sergei Kudrenko <[email protected]>
Co-authored-by: denischernow <[email protected]>

Author: 3 people

Dockerfile

@@ -17,5 +17,6 @@ RUN apt-get update && \
 
 COPY src/ .
 COPY src/syngen/streamlit_app/.streamlit syngen/.streamlit
+COPY src/syngen/streamlit_app/.streamlit/config.toml /root/.streamlit/config.toml
 ENV PYTHONPATH "${PYTHONPATH}:/src/syngen"
-ENTRYPOINT ["python3", "-m", "start"]
+ENTRYPOINT ["python3", "-m", "start"]

setup.cfg

@@ -66,7 +66,7 @@ install_requires =
 ui =
     streamlit
     streamlit_option_menu
-    altair<5
+    altair>5
 
 
 [options.packages.find]

src/start.py

@@ -5,7 +5,8 @@
 
 def parse_args():
     parser = argparse.ArgumentParser(
-        description="Run training, inference tasks, or a Streamlit web UI.", add_help=False
+        description="Run training, inference tasks, or a Streamlit web UI.",
+        add_help=False,
     )
     parser.add_argument(
         "--task", choices=["train", "infer"], help="Task to run: 'train' or 'infer'."
@@ -28,15 +29,18 @@ def main():
     # Check if the Streamlit web UI should be launched
     if known_args.webui:
         # Adjust the path to your Streamlit application script if necessary
-        command = ["streamlit", "run", "syngen/streamlit_app.py"] + remaining_argv
+        command = ["streamlit", "run", "syngen/streamlit_app/run.py"] + remaining_argv
     elif known_args.task == "train":
         # Construct the command to run the training script
         command = ["python", "syngen/train.py"] + remaining_argv
     elif known_args.task == "infer":
         # Construct the command to run the inference script
         command = ["python", "syngen/infer.py"] + remaining_argv
     else:
-        print("Unknown command. Use --task=train, --task=infer, or --webui.", file=sys.stderr)
+        print(
+            "Unknown command. Use --task=train, --task=infer, or --webui.",
+            file=sys.stderr,
+        )
         sys.exit(1)
 
     # Run the command with any additional arguments

src/syngen/VERSION

@@ -1 +1 @@
-0.7.2
+0.7.3

src/syngen/ml/handlers/handlers.py

@@ -19,9 +19,13 @@
 
 from syngen.ml.vae import *  # noqa: F403
 from syngen.ml.data_loaders import DataLoader
-from syngen.ml.utils import fetch_dataset, check_if_features_assigned, generate_uuid
+from syngen.ml.utils import (
+    fetch_dataset,
+    check_if_features_assigned,
+    generate_uuid,
+    ProgressBarHandler
+)
 from syngen.ml.context import get_context
-from syngen.ml.config import TrainConfig
 
 
 class AbstractHandler(ABC):
@@ -168,8 +172,9 @@ def __fit_model(self, data: pd.DataFrame):
         self.model.batch_size = min(self.batch_size, len(data))
 
         logger.debug(
-            f"Train model with parameters: epochs={self.epochs}, row_subset={self.row_subset}, "
-            f"print_report={self.print_report}, drop_null={self.drop_null}, batch_size={self.batch_size}"
+            f"Train model with parameters: epochs={self.epochs}, "
+            f"row_subset={self.row_subset}, print_report={self.print_report}, "
+            f"drop_null={self.drop_null}, batch_size={self.batch_size}"
         )
 
         self.model.fit_on_df(
@@ -181,7 +186,9 @@ def __fit_model(self, data: pd.DataFrame):
             return
 
         self.model.save_state(self.paths["state_path"])
-        logger.info("Finished VAE training")
+        log_message = "Finished VAE training"
+        logger.info(log_message)
+        ProgressBarHandler().set_progress(message=log_message)
 
     def __prepare_dir(self):
         os.makedirs(self.paths["fk_kde_path"], exist_ok=True)
@@ -434,7 +441,19 @@ def handle(self, **kwargs):
         )
         logger.info(f"Total of {batch_num} batch(es)")
         batches = self.split_by_batches(self.size, batch_num)
-        prepared_batches = [self.run(batch, self.run_parallel) for batch in batches]
+        delta = ProgressBarHandler().delta / batch_num
+        prepared_batches = []
+        for i, batch in enumerate(batches):
+            log_message = (f"Data synthesis for the table - '{self.table_name}'. "
+                           f"Generating the batch {i + 1} of {batch_num}")
+            ProgressBarHandler().set_progress(
+                progress=ProgressBarHandler().progress + delta,
+                delta=delta,
+                message=log_message,
+            )
+            logger.info(log_message)
+            prepared_batch = self.run(batch, self.run_parallel)
+            prepared_batches.append(prepared_batch)
         prepared_data = (
             self._concat_slices_with_unique_pk(prepared_batches)
             if len(prepared_batches) > 0

src/syngen/ml/metrics/accuracy_test/accuracy_test.py

@@ -17,7 +17,7 @@
     Utility,
 )
 from syngen.ml.metrics.utils import transform_to_base64
-from syngen.ml.utils import fetch_training_config
+from syngen.ml.utils import fetch_training_config, ProgressBarHandler
 from syngen.ml.mlflow_tracker import MlflowTracker
 
 
@@ -75,13 +75,21 @@ def _log_report_to_mlflow(self, path):
             )
             pass
 
+    @staticmethod
+    def update_progress_bar(message, delta=0):
+        ProgressBarHandler().set_progress(
+            progress=ProgressBarHandler().progress + delta, delta=None, message=message
+        )
+
     def _get_cleaned_configs(self):
         """
         Get cleaned configs for the report
         """
         train_config = {
             k: v
-            for k, v in fetch_training_config(self.paths["train_config_pickle_path"]).to_dict().items()
+            for k, v in fetch_training_config(self.paths["train_config_pickle_path"])
+            .to_dict()
+            .items()
             if k != "print_report"
         }
         infer_config = {
@@ -103,90 +111,160 @@ def __init__(
     ):
         super().__init__(original, synthetic, paths, table_name, infer_config)
         self.draws_path = f"{self.paths['draws_path']}/accuracy"
-        self.univariate = UnivariateMetric(self.original, self.synthetic, self.plot_exists, self.draws_path)
-        self.bivariate = BivariateMetric(self.original, self.synthetic, self.plot_exists, self.draws_path)
-        self.correlations = Correlations(self.original, self.synthetic, self.plot_exists, self.draws_path)
-        self.clustering = Clustering(self.original, self.synthetic, self.plot_exists, self.draws_path)
-        self.utility = Utility(self.original, self.synthetic, self.plot_exists, self.draws_path)
-        self.acc = JensenShannonDistance(self.original, self.synthetic, self.plot_exists, self.draws_path)
+        self.univariate = UnivariateMetric(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
+        self.bivariate = BivariateMetric(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
+        self.correlations = Correlations(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
+        self.clustering = Clustering(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
+        self.utility = Utility(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
+        self.acc = JensenShannonDistance(
+            self.original, self.synthetic, self.plot_exists, self.draws_path
+        )
         self._prepare_dir()
 
     def _fetch_metrics(self, **kwargs):
         """
         Fetch the main metrics
         """
+        delta = ProgressBarHandler().delta / 6
+
+        self.update_progress_bar("Generation of the accuracy heatmap...")
         self.acc.calculate_all(kwargs["categ_columns"])
         acc_median = "%.4f" % self.acc.calculate_heatmap_median(self.acc.heatmap)
+        logger.info(f"Median accuracy is {acc_median}")
+        self.update_progress_bar("The accuracy heatmap has been generated", delta)
+
         uni_images = dict()
         bi_images = dict()
+
         if self.plot_exists:
+            self.update_progress_bar("Generation of the univariate distributions...")
             uni_images = self.univariate.calculate_all(
                 kwargs["cont_columns"], kwargs["categ_columns"], kwargs["date_columns"]
             )
+            self.update_progress_bar(
+                "The univariate distributions have been generated", delta
+            )
+
+            self.update_progress_bar("Generation of the bivariate distributions...")
             bi_images = self.bivariate.calculate_all(
                 kwargs["cont_columns"], kwargs["categ_columns"], kwargs["date_columns"]
             )
-        corr_result = self.correlations.calculate_all(kwargs["categ_columns"], kwargs["cont_columns"])
+            self.update_progress_bar(
+                "The bivariate distributions have been generated", delta
+            )
+
+        self.update_progress_bar("Generation of the correlations heatmap...")
+        corr_result = self.correlations.calculate_all(
+            kwargs["categ_columns"], kwargs["cont_columns"]
+        )
         corr_result = int(corr_result) if corr_result == 0 else abs(corr_result)
+        logger.info(f"Median of differences of correlations is {round(corr_result, 4)}")
+        self.update_progress_bar("The correlations heatmap has been generated", delta)
+
+        self.update_progress_bar("Generation of the clustering metric...")
         clustering_result = "%.4f" % self.clustering.calculate_all(
             kwargs["categ_columns"], kwargs["cont_columns"]
         )
-        utility_result = self.utility.calculate_all(kwargs["categ_columns"], kwargs["cont_columns"])
-
-        logger.info(f"Median accuracy is {acc_median}")
-        logger.info(f"Median of differences of correlations is {round(corr_result, 4)}")
         logger.info(f"Median clusters homogeneity is {clustering_result}")
+        self.update_progress_bar("The clustering metric has been calculated", delta)
 
-        return acc_median, corr_result, clustering_result, utility_result, uni_images, bi_images
+        self.update_progress_bar("Generation of the utility metric...")
+        utility_result = self.utility.calculate_all(
+            kwargs["categ_columns"], kwargs["cont_columns"]
+        )
+        logger.info(f"Median clusters homogeneity is {clustering_result}")
+        self.update_progress_bar("The utility metric has been calculated", delta)
 
-    def _generate_report(
-            self,
+        return (
             acc_median,
             corr_result,
             clustering_result,
             utility_result,
             uni_images,
-            bi_images
+            bi_images,
+        )
+
+    def _generate_report(
+        self,
+        acc_median,
+        corr_result,
+        clustering_result,
+        utility_result,
+        uni_images,
+        bi_images,
     ):
         """
         Generate the report
         """
-        with open(f"{os.path.dirname(os.path.realpath(__file__))}/accuracy_report.html") as file_:
+        with open(
+            f"{os.path.dirname(os.path.realpath(__file__))}/accuracy_report.html"
+        ) as file_:
             template = jinja2.Template(file_.read())
 
         draws_acc_path = f"{self.paths['draws_path']}/accuracy"
-        uni_images = {title: transform_to_base64(path) for title, path in uni_images.items()}
-        bi_images = {title: transform_to_base64(path) for title, path in bi_images.items()}
+        uni_images = {
+            title: transform_to_base64(path) for title, path in uni_images.items()
+        }
+        bi_images = {
+            title: transform_to_base64(path) for title, path in bi_images.items()
+        }
 
         train_config, infer_config = self._get_cleaned_configs()
 
         html = template.render(
             accuracy_value=acc_median,
-            accuracy_heatmap=transform_to_base64(f"{draws_acc_path}/accuracy_heatmap.svg"),
+            accuracy_heatmap=transform_to_base64(
+                f"{draws_acc_path}/accuracy_heatmap.svg"
+            ),
             uni_imgs=uni_images,
-            correlations_heatmap=transform_to_base64(f"{draws_acc_path}/correlations_heatmap.svg"),
+            correlations_heatmap=transform_to_base64(
+                f"{draws_acc_path}/correlations_heatmap.svg"
+            ),
             correlation_median=corr_result,
-            clusters_barplot=transform_to_base64(f"{draws_acc_path}/clusters_barplot.svg"),
+            clusters_barplot=transform_to_base64(
+                f"{draws_acc_path}/clusters_barplot.svg"
+            ),
             clustering_value=clustering_result,
             bi_imgs=bi_images,
-            utility_barplot=transform_to_base64(f"{draws_acc_path}/utility_barplot.svg"),
+            utility_barplot=transform_to_base64(
+                f"{draws_acc_path}/utility_barplot.svg"
+            ),
             utility_table=utility_result.to_html(),
             is_data_available=False if utility_result.empty else True,
             table_name=self.table_name,
             training_config=train_config,
             inference_config=infer_config,
             time=datetime.now().strftime("%H:%M:%S %d/%m/%Y"),
-            round=round
+            round=round,
         )
 
-        with open(f"{self.paths['draws_path']}/accuracy_report.html", "w", encoding="utf-8") as f:
+        with open(
+            f"{self.paths['draws_path']}/accuracy_report.html", "w", encoding="utf-8"
+        ) as f:
             f.write(html)
         self._log_report_to_mlflow(f"{self.paths['draws_path']}/accuracy_report.html")
         self._remove_artifacts()
 
     def report(self, *args, **kwargs):
         metrics = self._fetch_metrics(**kwargs)
-        acc_median, corr_result, clustering_result, utility_result, uni_images, bi_images = metrics
+        (
+            acc_median,
+            corr_result,
+            clustering_result,
+            utility_result,
+            uni_images,
+            bi_images,
+        ) = metrics
         MlflowTracker().log_metrics(
             {
                 "Utility_avg": utility_result["Synth to orig ratio"].mean(),
@@ -203,5 +281,5 @@ def report(self, *args, **kwargs):
                 clustering_result,
                 utility_result,
                 uni_images,
-                bi_images
+                bi_images,
             )