add preliminary unset field semantics (disc #536)

annotate Struct fields as optional and Structs as strict

add validation logic

add preliminary testing

Signed-off-by: Simon Beyzerov <[email protected]>

Author: Simon Beyzerov

cpp/csp/adapters/kafka/KafkaInputAdapter.cpp

@@ -113,6 +113,8 @@ void KafkaInputAdapter::processMessage( RdKafka::Message* message, bool live, cs
         if( m_tickTimestampField )
             msgTime = m_tickTimestampField->value<DateTime>(tick.get());
 
+        tick.get() -> validate();
+
         bool pushLive = shouldPushLive(live, msgTime);
         if( shouldProcessMessage( pushLive, msgTime ) )
             pushTick(pushLive, msgTime, std::move(tick), batch);

cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp

@@ -520,6 +520,7 @@ void ParquetStructAdapter::dispatchValue( const utils::Symbol *symbol, bool isNu
         {
             fieldSetter( s );
         }
+        s -> validate();
         dispatchedValue = &s;
     }
 

cpp/csp/adapters/utils/JSONMessageStructConverter.cpp

@@ -145,6 +145,8 @@ StructPtr JSONMessageStructConverter::convertJSON( const char * fieldname, const
             } );
     }
 
+    struct_ -> validate();
+
     return struct_;
 }
 
@@ -251,6 +253,7 @@ csp::StructPtr JSONMessageStructConverter::asStruct( void * bytes, size_t size )
             }
         );
     }
+    // root struct validation (validate()) deferred to adapter level
 
     return data;
 }

cpp/csp/adapters/websocket/ClientInputAdapter.cpp

@@ -31,6 +31,7 @@ void ClientInputAdapter::processMessage( void* c, size_t t, PushBatch* batch )
     if( dataType() -> type() == CspType::Type::STRUCT )
     {
         auto tick = m_converter -> asStruct( c, t );
+        tick.get() -> validate();
         pushTick( std::move(tick), batch );
     } else if ( dataType() -> type() == CspType::Type::STRING )
     {

cpp/csp/cppnodes/baselibimpl.cpp

@@ -705,6 +705,7 @@ DECLARE_CPPNODE( struct_fromts )
                 );
         }
 
+        out.get() -> validate( );
         CSP_OUTPUT( std::move( out ) );
     }
 
@@ -758,7 +759,7 @@ DECLARE_CPPNODE( struct_collectts )
                            }
                 );
         }
-
+        out.get() -> validate( );
         CSP_OUTPUT( std::move( out ) );
     }
 

cpp/csp/engine/Struct.cpp

@@ -1,20 +1,23 @@
 #include <csp/core/System.h>
 #include <csp/engine/Struct.h>
 #include <algorithm>
+#include <ranges>
+#include <string>
 
 namespace csp
 {
 
 StructField::StructField( CspTypePtr type, const std::string & fieldname, 
-                          size_t size, size_t alignment ) :
+                          size_t size, size_t alignment, bool isOptional ) :
     m_fieldname( fieldname ),
     m_offset( 0 ),
     m_size( size ),
     m_alignment( alignment ),
     m_maskOffset( 0 ),
     m_maskBit( 0 ),
     m_maskBitMask( 0 ),
-    m_type( type )
+    m_type( type ),
+    m_isOptional( isOptional )
 {
 }
 
@@ -33,8 +36,8 @@ and adjustments required for the hidden fields
 
 */
 
-StructMeta::StructMeta( const std::string & name, const Fields & fields,
-                        std::shared_ptr<StructMeta> base ) : m_name( name ), m_base( base ), m_fields( fields ),
+StructMeta::StructMeta( const std::string & name, const Fields & fields, bool isStrict,
+                        std::shared_ptr<StructMeta> base ) : m_name( name ), m_base( base ), m_isStrict( isStrict ), m_fields( fields ), 
                                                              m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ),
                                                              m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ),
                                                              m_isPartialNative( true ), m_isFullyNative( true )
@@ -494,6 +497,43 @@ void StructMeta::destroy( Struct * s ) const
         m_base -> destroy( s );
 }
 
+void StructMeta::validate( const Struct * s ) const
+{
+    bool encountered_non_strict = false;
+    
+    for( const StructMeta * cur = this; cur; cur = cur -> m_base.get() )
+    {
+        encountered_non_strict |= !cur -> isStrict();
+        if( !cur -> isStrict() ) {
+            continue;
+        }
+        
+        // rule 1: a non-strict struct may not inherit (directly or indirectly) from a strict base
+        if (encountered_non_strict)
+                CSP_THROW( ValueError, "Struct '" << s -> meta() -> name() << "' has non-strict inheritance of strict base '" << cur -> name() << "'" );
+
+        // rule 2: all local fields are set
+        std::string missing_fields;
+        for(const auto& field : cur->m_fields) {
+            if(!field->isSet(s)) {
+                if(missing_fields.empty()) {
+                    missing_fields = field->fieldname();
+                } else {
+                    missing_fields += ", " + field->fieldname();
+                }
+            }
+        }
+        
+        // raise error if any fields are missing
+        if (!missing_fields.empty()) {
+            CSP_THROW(ValueError,
+                     "Strict struct '" << cur->name() << "' missing required fields: " << missing_fields);
+        }
+    }
+}
+
+
+
 Struct::Struct( const std::shared_ptr<const StructMeta> & meta )
 {
     //Initialize meta shared_ptr

cpp/csp/engine/Struct.h

@@ -35,6 +35,8 @@ class StructField
 
     bool isNative() const                 { return m_type -> type() <= CspType::Type::MAX_NATIVE_TYPE; }
 
+    bool isOptional() const               { return m_isOptional; }
+
     void setOffset( size_t off )          { m_offset = off; }
     void setMaskOffset( size_t off, uint8_t bit  )
     {
@@ -75,7 +77,7 @@ class StructField
 protected:
 
     StructField( CspTypePtr type, const std::string & fieldname,
-                 size_t size, size_t alignment );
+                 size_t size, size_t alignment, bool isOptional );
 
     void setIsSet( Struct * s ) const
     {
@@ -108,6 +110,7 @@ class StructField
     uint8_t      m_maskBit;
     uint8_t      m_maskBitMask;
     CspTypePtr   m_type;
+    const bool   m_isOptional;
 };
 
 using StructFieldPtr = std::shared_ptr<StructField>;
@@ -120,7 +123,7 @@ class NativeStructField : public StructField
 
 public:
     NativeStructField() {}
-    NativeStructField( const std::string & fieldname ) : NativeStructField( CspType::fromCType<T>::type(), fieldname )
+    NativeStructField( const std::string & fieldname, bool isOptional ) : NativeStructField( CspType::fromCType<T>::type(), fieldname, isOptional )
     {
     }
 
@@ -157,7 +160,7 @@ class NativeStructField : public StructField
     }
 
 protected:
-    NativeStructField( CspTypePtr type, const std::string & fieldname ) : StructField( type, fieldname, sizeof( T ), alignof( T ) )
+    NativeStructField( CspTypePtr type, const std::string & fieldname, bool isOptional ) : StructField( type, fieldname, sizeof( T ), alignof( T ), isOptional )
     {}
 };
 
@@ -179,7 +182,8 @@ using TimeStructField      = NativeStructField<Time>;
 class CspEnumStructField final : public NativeStructField<CspEnum>
 {
 public:
-    CspEnumStructField( CspTypePtr type, const std::string & fieldname ) : NativeStructField( type, fieldname )
+    CspEnumStructField( CspTypePtr type, const std::string & fieldname, bool isOptional ) : NativeStructField( type, 
+    fieldname, isOptional )
     {}
 };
 
@@ -218,8 +222,8 @@ class NotImplementedStructField : public StructField
 class NonNativeStructField : public StructField
 {
 public:
-    NonNativeStructField( CspTypePtr type, const std::string &fieldname, size_t size, size_t alignment ) :
-        StructField( type, fieldname, size, alignment )
+    NonNativeStructField( CspTypePtr type, const std::string &fieldname, size_t size, size_t alignment, bool isOptional ) :
+        StructField( type, fieldname, size, alignment, isOptional )
     {}
 
     virtual void initialize( Struct * s ) const = 0;
@@ -244,8 +248,8 @@ class StringStructField final : public NonNativeStructField
 public:
     using CType = csp::CspType::StringCType;
 
-    StringStructField( CspTypePtr type, const std::string & fieldname ) :
-        NonNativeStructField( type, fieldname, sizeof( CType ), alignof( CType ) )
+    StringStructField( CspTypePtr type, const std::string & fieldname, bool isOptional ) :
+        NonNativeStructField( type, fieldname, sizeof( CType ), alignof( CType ), isOptional )
     {}
 
     void initialize( Struct * s ) const override
@@ -343,8 +347,8 @@ class ArrayStructField : public NonNativeStructField
     }
 
 public:
-    ArrayStructField( CspTypePtr arrayType, const std::string & fieldname ) :
-        NonNativeStructField( arrayType, fieldname, sizeof( CType ), alignof( CType ) )
+    ArrayStructField( CspTypePtr arrayType, const std::string & fieldname, bool isOptional ) :
+        NonNativeStructField( arrayType, fieldname, sizeof( CType ), alignof( CType ), isOptional )
     {}
 
     const CType & value( const Struct * s ) const
@@ -421,8 +425,8 @@ class ArrayStructField : public NonNativeStructField
 class DialectGenericStructField : public NonNativeStructField
 {
 public:
-    DialectGenericStructField( const std::string & fieldname, size_t size, size_t alignment ) :
-        NonNativeStructField( CspType::DIALECT_GENERIC(), fieldname, size, alignment )
+    DialectGenericStructField( const std::string & fieldname, size_t size, size_t alignment, bool isOptional ) :
+        NonNativeStructField( CspType::DIALECT_GENERIC(), fieldname, size, alignment, isOptional )
     {}
 
     const DialectGenericType & value( const Struct * s ) const
@@ -587,21 +591,24 @@ class StructMeta : public std::enable_shared_from_this<StructMeta>
     using FieldNames = std::vector<std::string>;
 
     //Fields will be re-arranged and assigned their offsets in StructMeta for optimal performance
-    StructMeta( const std::string & name, const Fields & fields, std::shared_ptr<StructMeta> base = nullptr );
+    StructMeta( const std::string & name, const Fields & fields, bool isStrict, std::shared_ptr<StructMeta> base = nullptr );
     virtual ~StructMeta();
 
     const std::string & name() const          { return m_name; }
     size_t size() const                       { return m_size; }
     size_t partialSize() const                { return m_partialSize; }
 
     bool isNative() const                     { return m_isFullyNative; }
+    bool isStrict() const                     { return m_isStrict; }
 
     const Fields & fields() const             { return m_fields; }
     const FieldNames & fieldNames() const     { return m_fieldnames; }
 
     size_t maskLoc() const                    { return m_maskLoc; }
     size_t maskSize() const                   { return m_maskSize; }
 
+    void validate( const Struct * s ) const;
+
     const StructFieldPtr & field( const char * name ) const
     {
         static StructFieldPtr s_empty;
@@ -652,7 +659,8 @@ class StructMeta : public std::enable_shared_from_this<StructMeta>
     std::shared_ptr<StructMeta> m_base;
     StructPtr                   m_default;
     FieldMap                    m_fieldMap;
-
+    bool                        m_isStrict;          
+    
     //fields in order, memory owners of field objects which in turn own the key memory
     //m_fields includes all base fields as well. m_fieldnames maintains the proper iteration order of fields
     Fields                      m_fields;
@@ -738,6 +746,11 @@ class Struct
         return meta() -> allFieldsSet( this );
     }
 
+    void validate() const
+    {
+        meta() -> validate( this );
+    }
+
 
     //used to cache dialect representations of this struct, if needed
     void * dialectPtr() const      { return hidden() -> dialectPtr; }
@@ -822,8 +835,8 @@ bool TypedStructPtr<T>::operator==( const TypedStructPtr<T> & rhs ) const
 class StructStructField final : public NonNativeStructField
 {
 public:
-    StructStructField( CspTypePtr cspType, const std::string &fieldname ) :
-        NonNativeStructField( cspType, fieldname, sizeof( StructPtr ), alignof( StructPtr ) )
+    StructStructField( CspTypePtr cspType, const std::string &fieldname, bool isOptional ) :
+        NonNativeStructField( cspType, fieldname, sizeof( StructPtr ), alignof( StructPtr ), isOptional )
     {
         CSP_ASSERT( cspType -> type() == CspType::Type::STRUCT );
         m_meta = std::static_pointer_cast<const CspStructType>( cspType ) -> meta();