incorporate suggestions

Author: nikhilsinhaparseable

src/cli.rs

@@ -35,6 +35,7 @@ use crate::{
 pub const DEFAULT_USERNAME: &str = "admin";
 pub const DEFAULT_PASSWORD: &str = "admin";
 
+pub const DATASET_FIELD_COUNT_LIMIT: usize = 250;
 #[derive(Parser)]
 #[command(
     name = "parseable",
@@ -371,10 +372,10 @@ pub struct Options {
 
     #[arg(
         long,
-        env = "P_DATASET_FIELDS_ALLOWED_LIMIT",
-        default_value = "250",
+        env = "P_DATASET_FIELD_COUNT_LIMIT",
+        default_value_t = DATASET_FIELD_COUNT_LIMIT,
         value_parser = validation::validate_dataset_fields_allowed_limit,
-        help = "allowed limit for fields count in a dataset"
+        help = "total number of fields recommended in a dataset"
     )]
     pub dataset_fields_allowed_limit: usize,
 }

src/handlers/http/ingest.rs

@@ -468,7 +468,7 @@ pub enum PostError {
     #[error("Ingestion is not allowed to stream {0} as it is already associated with a different OTEL format")]
     IncorrectLogFormat(String),
     #[error("Failed to ingest events in dataset {0}. Total number of fields {1} exceeds the permissible limit of {2}. We recommend creating a new dataset beyond {2} for better query performance.")]
-    FieldsLimitExceeded(String, usize, usize),
+    FieldsCountLimitExceeded(String, usize, usize),
 }
 
 impl actix_web::ResponseError for PostError {
@@ -497,7 +497,7 @@ impl actix_web::ResponseError for PostError {
             PostError::MissingTimePartition(_) => StatusCode::BAD_REQUEST,
             PostError::KnownFormat(_) => StatusCode::BAD_REQUEST,
             PostError::IncorrectLogFormat(_) => StatusCode::BAD_REQUEST,
-            PostError::FieldsLimitExceeded(_, _, _) => StatusCode::BAD_REQUEST,
+            PostError::FieldsCountLimitExceeded(_, _, _) => StatusCode::BAD_REQUEST,
         }
     }
 

src/handlers/http/modal/utils/ingest_utils.rs

@@ -47,8 +47,6 @@ use crate::{
 const IGNORE_HEADERS: [&str; 3] = [STREAM_NAME_HEADER_KEY, LOG_SOURCE_KEY, EXTRACT_LOG_KEY];
 const MAX_CUSTOM_FIELDS: usize = 10;
 const MAX_FIELD_VALUE_LENGTH: usize = 100;
-// Maximum allowed count for fields in a dataset
-pub const DATASET_FIELDS_ALLOWED_LIMIT: usize = 250;
 
 pub async fn flatten_and_push_logs(
     json: Value,
@@ -216,29 +214,27 @@ fn verify_dataset_fields_count(stream_name: &str) -> Result<(), PostError> {
         .get_schema()
         .fields()
         .len();
-    let dataset_fields_warn_threshold = 0.8 * DATASET_FIELDS_ALLOWED_LIMIT as f64;
+    let dataset_fields_warn_threshold = 0.8 * PARSEABLE.options.dataset_fields_allowed_limit as f64;
     // Check if the fields count exceeds the warn threshold
     if fields_count > dataset_fields_warn_threshold as usize {
         tracing::warn!(
-            "Fields count {0} for dataset {1} has exceeded the warning threshold of {2} fields, Parseable recommends creating a new dataset.",
-            fields_count,
+            "Total fields in dataset {0} has reached the warning threshold of {1}. Ingestion will not be possible after reaching {2} fields. We recommend creating a new dataset.",
             stream_name,
-            dataset_fields_warn_threshold);
+            dataset_fields_warn_threshold,
+            PARSEABLE.options.dataset_fields_allowed_limit
+            );
     }
     // Check if the fields count exceeds the limit
     // Return an error if the fields count exceeds the limit
     if fields_count > PARSEABLE.options.dataset_fields_allowed_limit {
-        tracing::error!(
-            "Ingestion has been stopped for dataset {0} as fields count {1} exceeds the allowed limit of {2}, Please create a new dataset.",
-            stream_name,
-            fields_count,
-            PARSEABLE.options.dataset_fields_allowed_limit);
-        // Return an error if the fields count exceeds the limit
-        return Err(PostError::FieldsLimitExceeded(
+        let error = PostError::FieldsCountLimitExceeded(
             stream_name.to_string(),
             fields_count,
             PARSEABLE.options.dataset_fields_allowed_limit,
-        ));
+        );
+        tracing::error!("{}", error);
+        // Return an error if the fields count exceeds the limit
+        return Err(error);
     }
     Ok(())
 }

src/option.rs

@@ -91,7 +91,7 @@ pub mod validation {
         path::{Path, PathBuf},
     };
 
-    use crate::handlers::http::modal::utils::ingest_utils::DATASET_FIELDS_ALLOWED_LIMIT;
+    use crate::cli::DATASET_FIELD_COUNT_LIMIT;
     use path_clean::PathClean;
 
     use super::{Compression, Mode};
@@ -177,16 +177,16 @@ pub mod validation {
 
     pub fn validate_dataset_fields_allowed_limit(s: &str) -> Result<usize, String> {
         if let Ok(size) = s.parse::<usize>() {
-            if (1..=DATASET_FIELDS_ALLOWED_LIMIT).contains(&size) {
+            if (1..=DATASET_FIELD_COUNT_LIMIT).contains(&size) {
                 Ok(size)
             } else {
                 Err(format!(
-                    "Invalid value for P_DATASET_FIELDS_ALLOWED_LIMIT. It should be between 1 and {}",
-                    DATASET_FIELDS_ALLOWED_LIMIT
+                    "Invalid value for P_DATASET_FIELD_COUNT_LIMIT. It should be between 1 and {}",
+                    DATASET_FIELD_COUNT_LIMIT
                 ))
             }
         } else {
-            Err("Invalid value for P_DATASET_FIELDS_ALLOWED_LIMIT. It should be given as integer value".to_string())
+            Err("Invalid value for P_DATASET_FIELD_COUNT_LIMIT. It should be given as integer value".to_string())
         }
     }
 }