logicalclocks · moritzmeister · Aug 30, 2021 · Aug 27, 2021
diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb
@@ -656,18 +656,21 @@
         expect(parsed_json.key?("statisticsConfig")).to be true
         expect(parsed_json["statisticsConfig"].key?("histograms")).to be true
         expect(parsed_json["statisticsConfig"].key?("correlations")).to be true
+        expect(parsed_json["statisticsConfig"].key?("exactUniqueness")).to be true
         expect(parsed_json["statisticsConfig"].key?("enabled")).to be true
         expect(parsed_json["statisticsConfig"].key?("columns")).to be true
         expect(parsed_json["statisticsConfig"]["columns"].length).to eql(0)
         expect(parsed_json["statisticsConfig"]["enabled"]).to be true
         expect(parsed_json["statisticsConfig"]["correlations"]).to be false
         expect(parsed_json["statisticsConfig"]["histograms"]).to be false
+        expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
       end
 
       it "should be able to add a cached featuregroup with non default statistics settings to the featurestore" do
         project = get_project
         featurestore_id = get_featurestore_id(project.id)
-        stats_config = {enabled: false, histograms: false, correlations: false, columns: ["testfeature"]}
+        stats_config = {enabled: false, histograms: false, correlations: false, exactUniqueness: false, columns:
+        ["testfeature"]}
         json_result, featuregroup_name = create_cached_featuregroup(project.id, featurestore_id, statistics_config:
             stats_config)
         expect_status_details(201)
@@ -677,6 +680,7 @@
         expect(parsed_json.key?("name")).to be true
         expect(parsed_json["statisticsConfig"].key?("histograms")).to be true
         expect(parsed_json["statisticsConfig"].key?("correlations")).to be true
+        expect(parsed_json["statisticsConfig"].key?("exactUniqueness")).to be true
         expect(parsed_json["statisticsConfig"].key?("enabled")).to be true
         expect(parsed_json["statisticsConfig"].key?("columns")).to be true
         expect(parsed_json["featurestoreName"] == project.projectname.downcase + "_featurestore").to be true
@@ -687,12 +691,14 @@
         expect(parsed_json["statisticsConfig"]["enabled"]).to be false
         expect(parsed_json["statisticsConfig"]["correlations"]).to be false
         expect(parsed_json["statisticsConfig"]["histograms"]).to be false
+        expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
       end
 
       it "should not be able to add a cached feature group with non-existing statistic column to the feature store" do
         project = get_project
         featurestore_id = get_featurestore_id(project.id)
-        stats_config = {enabled: false, histograms: false, correlations: false, columns: ["wrongname"]}
+        stats_config = {enabled: false, histograms: false, correlations: false, exactUniqueness: false, columns:
+        ["wrongname"]}
         json_result, _ = create_cached_featuregroup(project.id, featurestore_id, statistics_config: stats_config)
         expect_status_details(400)
         parsed_json = JSON.parse(json_result)
@@ -718,6 +724,7 @@
         expect(parsed_json["statisticsConfig"]["columns"][0]).to eql("testfeature")
         expect(parsed_json["statisticsConfig"]["enabled"]).to be false
         expect(parsed_json["statisticsConfig"]["correlations"]).to be false
+        expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
         expect(parsed_json["statisticsConfig"]["histograms"]).to be false
       end
 
@@ -1046,7 +1053,8 @@
       it "should be able to add correct statistics commit timestamps on time travel enabled feature groups" do
         featurestore_id = get_featurestore_id(@project[:id])
         featurestore_name = @project['projectname'].downcase + "_featurestore"
-        stats_config = {enabled: true, histograms: false, correlations: false, columns: ["testfeature"]}
+        stats_config = {enabled: true, histograms: false, correlations: false, exactUniqueness: false, columns:
+        ["testfeature"]}
         json_result, featuregroup_name = create_cached_featuregroup(@project[:id], featurestore_id, time_travel_format: "HUDI", statistics_config: stats_config)
         parsed_json = JSON.parse(json_result)
         featuregroup_id = parsed_json["id"]

diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/featurestore_helper.rb
@@ -375,6 +375,7 @@ def update_cached_featuregroup_stats_settings(project_id, featurestore_id, featu
         statisticsConfig: {
             histograms: false,
             correlations: false,
+            exactUniqueness: false,
             columns: statisticColumns,
             enabled: false
         }
@@ -383,6 +384,7 @@ def update_cached_featuregroup_stats_settings(project_id, featurestore_id, featu
       json_data[:statisticsConfig][:enabled] = false
       json_data[:statisticsConfig][:histograms] = true
       json_data[:statisticsConfig][:correlations] = true
+      json_data[:statisticsConfig][:exactUniqueness] = true
     end
     json_data = json_data.to_json
     json_result = put update_featuregroup_metadata_endpoint, json_data
@@ -398,6 +400,7 @@ def update_training_dataset_stats_config(project_id, featurestore_id, training_d
         statisticsConfig: {
             histograms: false,
             correlations: false,
+            exactUniqueness: false,
             columns: statisticColumns,
             enabled: false
         }
@@ -406,6 +409,7 @@ def update_training_dataset_stats_config(project_id, featurestore_id, training_d
       json_data[:statisticsConfig][:enabled] = false
       json_data[:statisticsConfig][:histograms] = true
       json_data[:statisticsConfig][:correlations] = true
+      json_data[:statisticsConfig][:exactUniqueness] = true
     end
     json_data = json_data.to_json
     json_result = put update_trainingdataset_metadata_endpoint, json_data

diff --git a/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb b/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb
@@ -1011,18 +1011,21 @@
           expect(parsed_json.key?("statisticsConfig")).to be true
           expect(parsed_json["statisticsConfig"].key?("histograms")).to be true
           expect(parsed_json["statisticsConfig"].key?("correlations")).to be true
+          expect(parsed_json["statisticsConfig"].key?("exactUniqueness")).to be true
           expect(parsed_json["statisticsConfig"].key?("enabled")).to be true
           expect(parsed_json["statisticsConfig"].key?("columns")).to be true
           expect(parsed_json["statisticsConfig"]["columns"].length).to eql(0)
           expect(parsed_json["statisticsConfig"]["enabled"]).to be true
           expect(parsed_json["statisticsConfig"]["correlations"]).to be false
+          expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
           expect(parsed_json["statisticsConfig"]["histograms"]).to be false
         end
 
         it "should be able to create a training dataset with statistics settings and retrieve them back" do
           featurestore_id = get_featurestore_id(@project[:id])
           connector = get_hopsfs_training_datasets_connector(@project[:projectname])
-          stats_config = {enabled: false, histograms: false, correlations: false, columns: ["testfeature"]}
+          stats_config = {enabled: false, histograms: false, correlations: false, exactUniqueness: false, columns:
+          ["testfeature"]}
           json_result, _ = create_hopsfs_training_dataset(@project[:id], featurestore_id, connector,
                                                           statistics_config: stats_config)
           expect_status_details(201)
@@ -1031,13 +1034,15 @@
           expect(parsed_json["statisticsConfig"]["columns"][0]).to eql("testfeature")
           expect(parsed_json["statisticsConfig"]["enabled"]).to be false
           expect(parsed_json["statisticsConfig"]["correlations"]).to be false
+          expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
           expect(parsed_json["statisticsConfig"]["histograms"]).to be false
         end
 
         it "should not be possible to add a training dataset with non-existing statistic column" do
           featurestore_id = get_featurestore_id(@project[:id])
           connector = get_hopsfs_training_datasets_connector(@project[:projectname])
-          stats_config = {enabled: false, histograms: false, correlations: false, columns: ["wrongname"]}
+          stats_config = {enabled: false, histograms: false, correlations: false, exactUniqueness: false, columns:
+          ["wrongname"]}
           json_result, _ = create_hopsfs_training_dataset(@project[:id], featurestore_id, connector,
                                                           statistics_config: stats_config)
           expect_status_details(400)
@@ -1064,6 +1069,7 @@
           expect(parsed_json["statisticsConfig"]["columns"][0]).to eql("testfeature")
           expect(parsed_json["statisticsConfig"]["enabled"]).to be false
           expect(parsed_json["statisticsConfig"]["correlations"]).to be false
+          expect(parsed_json["statisticsConfig"]["exactUniqueness"]).to be false
           expect(parsed_json["statisticsConfig"]["histograms"]).to be false
         end
       end

diff --git a/.../main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java b/.../main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeaturegroupController.java
@@ -484,6 +484,9 @@ public FeaturegroupDTO updateFeatureGroupStatsConfig(Featurestore featurestore,
     if (featureGroupDTO.getStatisticsConfig().getCorrelations() != null) {
       featuregroup.getStatisticsConfig().setCorrelations(featureGroupDTO.getStatisticsConfig().getCorrelations());
     }
+    if (featureGroupDTO.getStatisticsConfig().getExactUniqueness() != null) {
+      featuregroup.getStatisticsConfig().setExactUniqueness(featureGroupDTO.getStatisticsConfig().getExactUniqueness());
+    }
     // compare against schema from database, as client doesn't need to send schema in update request
     statisticColumnController.verifyStatisticColumnsExist(featureGroupDTO, featuregroup, getFeatures(featuregroup,
       project, user));
@@ -702,7 +705,8 @@ private Featuregroup persistFeaturegroupMetadata(Featurestore featurestore, User
     featuregroup.setOnDemandFeaturegroup(onDemandFeaturegroup);
 
     StatisticsConfig statisticsConfig = new StatisticsConfig(featuregroupDTO.getStatisticsConfig().getEnabled(),
-      featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms());
+      featuregroupDTO.getStatisticsConfig().getCorrelations(), featuregroupDTO.getStatisticsConfig().getHistograms(),
+      featuregroupDTO.getStatisticsConfig().getExactUniqueness());
     statisticsConfig.setFeaturegroup(featuregroup);
     statisticsConfig.setStatisticColumns(featuregroupDTO.getStatisticsConfig().getColumns().stream()
       .map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));

diff --git a/...n/src/main/java/io/hops/hopsworks/common/featurestore/statistics/StatisticsConfigDTO.java b/...n/src/main/java/io/hops/hopsworks/common/featurestore/statistics/StatisticsConfigDTO.java
@@ -30,6 +30,7 @@ public class StatisticsConfigDTO {
   private Boolean enabled = true;
   private Boolean histograms = false;
   private Boolean correlations = false;
+  private Boolean exactUniqueness = false;
   private List<String> columns = new ArrayList<>();
 
   public StatisticsConfigDTO() {}
@@ -38,6 +39,7 @@ public StatisticsConfigDTO(StatisticsConfig statisticsConfig) {
     this.enabled = statisticsConfig.isDescriptive();
     this.correlations = statisticsConfig.isCorrelations();
     this.histograms = statisticsConfig.isHistograms();
+    this.exactUniqueness = statisticsConfig.isExactUniqueness();
     this.columns = statisticsConfig.getStatisticColumns().stream()
       .map(StatisticColumn::getName).collect(Collectors.toList());
   }
@@ -66,6 +68,10 @@ public void setCorrelations(Boolean correlations) {
     this.correlations = correlations;
   }
 
+  public Boolean getExactUniqueness() { return exactUniqueness; }
+
+  public void setExactUniqueness(Boolean exactUniqueness) { this.exactUniqueness = exactUniqueness; }
+
   public List<String> getColumns() {
     return columns;
   }
@@ -80,6 +86,7 @@ public String toString() {
       "enabled=" + enabled +
       ", histograms=" + histograms +
       ", correlations=" + correlations +
+      ", exactUniqueness=" + exactUniqueness +
       ", columns=" + columns +
       '}';
   }

diff --git a/...ava/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java b/...ava/io/hops/hopsworks/common/featurestore/trainingdatasets/TrainingDatasetController.java
@@ -344,7 +344,8 @@ private TrainingDatasetDTO createTrainingDatasetMetadata(Users user, Project pro
 
     StatisticsConfig statisticsConfig = new StatisticsConfig(trainingDatasetDTO.getStatisticsConfig().getEnabled(),
       trainingDatasetDTO.getStatisticsConfig().getCorrelations(),
-      trainingDatasetDTO.getStatisticsConfig().getHistograms());
+      trainingDatasetDTO.getStatisticsConfig().getHistograms(),
+      trainingDatasetDTO.getStatisticsConfig().getExactUniqueness());
     statisticsConfig.setTrainingDataset(trainingDataset);
     statisticsConfig.setStatisticColumns(trainingDatasetDTO.getStatisticsConfig().getColumns().stream()
       .map(sc -> new StatisticColumn(statisticsConfig, sc)).collect(Collectors.toList()));
@@ -603,6 +604,10 @@ public TrainingDatasetDTO updateTrainingDatasetStatsConfig(Users user, Project p
     if (trainingDatasetDTO.getStatisticsConfig().getCorrelations() != null) {
       trainingDataset.getStatisticsConfig().setCorrelations(trainingDatasetDTO.getStatisticsConfig().getCorrelations());
     }
+    if (trainingDatasetDTO.getStatisticsConfig().getExactUniqueness() != null) {
+      trainingDataset.getStatisticsConfig().setExactUniqueness(trainingDatasetDTO.getStatisticsConfig()
+        .getExactUniqueness());
+    }
     // compare against schema from database, as client doesn't need to send schema in update request
     statisticColumnController.verifyStatisticColumnsExist(trainingDatasetDTO, trainingDataset);
     trainingDataset = trainingDatasetFacade.update(trainingDataset);

diff --git a/...n/java/io/hops/hopsworks/persistence/entity/featurestore/statistics/StatisticsConfig.java b/...n/java/io/hops/hopsworks/persistence/entity/featurestore/statistics/StatisticsConfig.java
@@ -76,13 +76,17 @@ public class StatisticsConfig implements Serializable {
   @Basic(optional = false)
   @Column(name = "histograms")
   private boolean histograms;
+  @Basic(optional = false)
+  @Column(name = "exact_uniqueness")
+  private boolean exactUniqueness;
 
   public StatisticsConfig() {}
-
-  public StatisticsConfig(boolean descriptive, boolean correlations, boolean histograms) {
+  
+  public StatisticsConfig(boolean descriptive, boolean correlations, boolean histograms, boolean exactUniqueness) {
     this.descriptive = descriptive;
     this.histograms = histograms;
     this.correlations = correlations;
+    this.exactUniqueness = exactUniqueness;
   }
 
   public Integer getId() {
@@ -143,6 +147,10 @@ public void setHistograms(boolean featHistEnabled) {
     this.histograms = featHistEnabled;
   }
 
+  public boolean isExactUniqueness() { return exactUniqueness; }
+
+  public void setExactUniqueness(boolean exactUniqueness) { this.exactUniqueness = exactUniqueness; }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -163,6 +171,9 @@ public boolean equals(Object o) {
     if (histograms != that.histograms) {
       return false;
     }
+    if (exactUniqueness != that.exactUniqueness) {
+      return false;
+    }
     if (!id.equals(that.id)) {
       return false;
     }
@@ -184,6 +195,7 @@ public int hashCode() {
     result = 31 * result + (descriptive ? 1 : 0);
     result = 31 * result + (correlations ? 1 : 0);
     result = 31 * result + (histograms ? 1 : 0);
+    result = 31 * result + (exactUniqueness ? 1 : 0);
     return result;
   }
 }
diff --git a/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js b/hopsworks-web/yo/app/scripts/controllers/newTrainingDatasetCtrl.js
@@ -309,6 +309,7 @@ angular.module('hopsWorksApp')
                         "enabled": true,
                         "histograms": true,
                         "correlations": true,
+                        "exactUniqueness": true
                     },
                     "storageConnector": self.getStorageConnector(),
                     "type": "trainingDatasetDTO"
@@ -351,6 +352,7 @@ angular.module('hopsWorksApp')
                         "enabled": true,
                         "histograms": true,
                         "correlations": true,
+                        "exactUniqueness": true
                     },
                     "storageConnector": self.getStorageConnector(),
                     "type": "trainingDatasetDTO"

diff --git a/hopsworks-web/yo/app/views/featuregroupViewInfo.html b/hopsworks-web/yo/app/views/featuregroupViewInfo.html
@@ -514,6 +514,15 @@ <h3 style="margin-left: 10px" ng-if="featuregroupViewInfoCtrl.selectedFeaturegro
                         {{featuregroupViewInfoCtrl.selectedFeaturegroup.statisticsConfig.histograms ? "yes" : "no"}}
                     </td>
                 </tr>
+                <tr class="fs-table">
+                    <td style="font-weight:700;" class="fs-table">
+                        Exact Uniqueness
+                    </td>
+                    <td class="fs-table">
+                        {{featuregroupViewInfoCtrl.selectedFeaturegroup.statisticsConfig.exactUniqueness ? "yes" :
+                        "no"}}
+                    </td>
+                </tr>
                 </tbody>
             </table>
 

diff --git a/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html b/hopsworks-web/yo/app/views/trainingDatasetViewInfo.html
@@ -413,6 +413,15 @@ <h3 style="margin-left: 10px" ng-if="!trainingDatasetViewInfoCtrl.selectedTraini
                         {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.statisticsConfig.histograms ? "yes" : "no"}}
                     </td>
                 </tr>
+                <tr class="fs-table">
+                    <td style="font-weight:700;" class="fs-table">
+                        Exact Uniqueness
+                    </td>
+                    <td class="fs-table">
+                        {{trainingDatasetViewInfoCtrl.selectedTrainingDataset.statisticsConfig.exactUniqueness ?
+                        "yes" : "no"}}
+                    </td>
+                </tr>
                 </tbody>
             </table>