diff --git a/cmake/sample_defs/cpu1_platform_cfg.h b/cmake/sample_defs/cpu1_platform_cfg.h index 7ee7b5cb5..63f7537b4 100644 --- a/cmake/sample_defs/cpu1_platform_cfg.h +++ b/cmake/sample_defs/cpu1_platform_cfg.h @@ -1337,6 +1337,30 @@ */ #define CFE_PLATFORM_EVS_MAX_EVENT_FILTERS 8 +/** +** \cfeevscfg Maximum number of event before squelching +** +** \par Description: +** Maximum number of events that may be emitted per app per second. +** Setting this to 0 will cause events to be unrestricted. +** +** \par Limits +** This number must be less than or equal to INT_MAX/1000 +*/ +#define CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST 16 + +/** +** \cfeevscfg Sustained number of event messages per second per app before squelching +** +** \par Description: +** Sustained number of events that may be emitted per app per second. +** +** \par Limits +** This number must be less than or equal to #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST. +** Values lower than 8 may cause functional and unit test failures. +*/ +#define CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC 8 + /** ** \cfeevscfg Default Event Log Filename ** diff --git a/docs/cFE_FunctionalRequirements.csv b/docs/cFE_FunctionalRequirements.csv index 741a38c5d..f7e7dcf0d 100644 --- a/docs/cFE_FunctionalRequirements.csv +++ b/docs/cFE_FunctionalRequirements.csv @@ -299,6 +299,12 @@ EVS: Initialize Logging Mode On Power On Reset,cEVS3203,"Upon a Power-on Reset, EVS: Preserve Event Log Reset Mode On Processor Reset,cEVS3207,"Upon a Processor Reset, the cFE shall preserve the Event Logging Mode configuration parameter.",Try to retain mode across a processor reset. The contents of the Local Event Log will be preserved if the Event Logging Mode is configured to Discard. The contents of the Local Event Log may be overwritten (depending on the size and contents of the log prior to the reset) if the Event Logging Mode is configured to Overwrite. EVS: Preserve Log Full State On Processor Reset,cEVS3208,"Upon a Processor Reset, the cFE shall preserve the Local Event Log Full state.",Retain the cFE state across Processor Resets. EVS: Preserve Log Overflow Counter On Processor Reset,cEVS3210,"Upon a Processor Reset, the cFE shall preserve the Local Event Log Overflow Counter.",Retain the cFE state across Processor Resets. +EVS: Suppress Flooding Event Messages,cEVS3220,The cFE shall suppress event messages from an application that exceed a pre-defined rate.,Event message flooding may exhaust critical resources required on the spacecraft +EVS: Suppress Flooding Event Messages - Allow short bursts,cEVS3220.1,The cFE shall allow limited bursts of event messages from an application exceeding the suppress threshold rate. ,"Startup or other events may cause a brief burst of events that exceed the nominal rate, but should still be captured." +EVS: Suppress Flooding Event Messages - Resume events when not flooding,cEVS3220.2,The cFE shall automatically resume processing event messages from an application which is no longer flooding,"Flooding may be a transient condition, and resuming nominal operations when this condition goes away would allow continued delivery of useful diagnostic data. This also eases implementation. " +EVS: Suppress Flooding Event Messages - Increment counter,cEVS3220.3,The cFE shall increment the squelched counter for a flooding app when events for that app have been suppressed.,A counter provides a means for keeping track of how many events have been suppressed +EVS: Suppress Flooding Event Messages - Retain Maximum Value,cEVS3220.4,If the squelched event counters have reached their maximum value of (2^8)-1 the cFE shall retain the maximum value (i.e. do not rollover to zero).,Preventing a counter rollover to zero eliminates the case when a user may think no events have occurred when in fact many events have occurred. +EVS: Suppress Flooding Event Messages - Retain Maximum Value,cEVS3220.5,"The cFE shall issue an event message when an application transitions to the squelched state, containing the name of the offending application.","An event message provides an obvious useful diagnostic when event suppression occurs, which is an anomaly which should be dealt with." SB: NOOP Event,cSB4000,"Upon receipt of a Command, the cFE shall generate a NO-OP event message.",This command is useful as a general sub-system aliveness test. SB: Zero Counters,cSB4001,"Upon receipt of Command the cFE shall set to zero the following counters in housekeeping telemetry: diff --git a/docs/src/cfe_evs.dox b/docs/src/cfe_evs.dox index b1c9648da..7fe668310 100644 --- a/docs/src/cfe_evs.dox +++ b/docs/src/cfe_evs.dox @@ -23,6 +23,7 @@
  • \subpage cfeevsugcounters
  • \subpage cfeevsugresetctrs
  • \subpage cfeevsugprocreset
    +
  • \subpage cfeevsugsquelch
  • \subpage cfeevsugfaq
    @@ -428,11 +429,38 @@ transfer it to the ground in order to help debug a reset. - Next: \ref cfeevsugfaq
    + Next: \ref cfeevsugsquelch
    Prev: \ref cfeevsugresetctrs
    Up To: \ref cfeevsovr **/ +/** + \page cfeevsugsquelch EVS squelching of misbehaving apps + + Event squelching is an optional feature for suppressing excessive events from misbehaving apps. It + is enabled by setting #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST to a nonzero positive value, and + #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC equal to or less than that value. + + #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST controls the maximum events that can be sent at a given moment, + and #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC is the sustained event throughput per second. + + The suppression mechanism initializes with #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000 credits. + Each event costs 1000 credits. Credits are restored at a rate of + #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC * 1000 up to a maximum balance of #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST*1000, and the + maximum "debt" is -#CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000. When the credit count crosses from positive to negative, + a squelched event message is emitted and events are supppressed, until the credit count becomes positive again. + + Figure EVS-1 is a notional state diagram of the event squelching mechanism. + + \image html evs_squelch_states.png "Figure EVS-1: EVS Squelching State Diagram" + \image latex evs_squelch_states.png "Figure EVS-1: EVS Squelching State Diagram" + + + Next: \ref cfeevsugfaq
    + Prev: \ref cfeevsugprocreset
    + Up To: \ref cfeevsovr +**/ + /** \page cfeevsugfaq Frequently Asked Questions about Event Services diff --git a/docs/src/evs_squelch_states.png b/docs/src/evs_squelch_states.png new file mode 100644 index 000000000..ccc8ceb4d Binary files /dev/null and b/docs/src/evs_squelch_states.png differ diff --git a/docs/src/mnem_maps/cfe_evs_tlm_mnem_map b/docs/src/mnem_maps/cfe_evs_tlm_mnem_map index 32f1928ca..1752b9182 100644 --- a/docs/src/mnem_maps/cfe_evs_tlm_mnem_map +++ b/docs/src/mnem_maps/cfe_evs_tlm_mnem_map @@ -33,7 +33,7 @@ EVS_APP=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS] \ EVS_APPID=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPID \ EVS_APPMSGSENTC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPMSGSENTC \ EVS_APPENASTAT=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPENASTAT \ -EVS_SPARE2ALIGN3=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SPARE2ALIGN3 \ +EVS_SQUELCHEDC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SQUELCHEDC \ EVS_APPNAME=$sc_$cpu_EVS_APPNAME[OS_MAX_API_NAME] \ EVS_EVENTID=$sc_$cpu_EVS_EVENTID \ EVS_EVENTTYPE=$sc_$cpu_EVS_EVENTTYPE \ diff --git a/modules/cfe_testcase/src/evs_send_test.c b/modules/cfe_testcase/src/evs_send_test.c index d7b340e8e..6ecc47ad4 100644 --- a/modules/cfe_testcase/src/evs_send_test.c +++ b/modules/cfe_testcase/src/evs_send_test.c @@ -31,6 +31,15 @@ void TestSendEvent(void) { + int status; + int i; + + if (CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST) + { + /* Allow squelch credits to accumulate */ + OS_TaskDelay((CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC) * 1000); + } + UtPrintf("Testing: CFE_EVS_SendEvent"); UtAssert_INT32_EQ(CFE_EVS_SendEvent(0, CFE_EVS_EventType_INFORMATION, "OK Send"), CFE_SUCCESS); @@ -44,6 +53,22 @@ void TestSendEvent(void) UtAssert_INT32_EQ(CFE_EVS_SendEvent(0, CFE_EVS_EventType_CRITICAL, "OK (Critical) Send"), CFE_SUCCESS); UtAssert_INT32_EQ(CFE_EVS_SendEvent(0, CFE_EVS_EventType_CRITICAL, NULL), CFE_EVS_INVALID_PARAMETER); + + if (CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST) + { + /* Allow squelch credits to accumulate */ + OS_TaskDelay((CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC) * 1000); + + for (i = 0; i < CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST; i++) + UtAssert_INT32_EQ(CFE_EVS_SendEvent(0, CFE_EVS_EventType_INFORMATION, "OK Send"), CFE_SUCCESS); + + status = CFE_EVS_SendEvent(0, CFE_EVS_EventType_INFORMATION, "OK Squelch"); + + /* Allow squelch credits to accumulate */ + OS_TaskDelay((CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC) * 1000); + + UtAssert_INT32_EQ(status, CFE_EVS_APP_SQUELCHED); + } } void TestSendEventAppID(void) diff --git a/modules/core_api/fsw/inc/cfe_error.h b/modules/core_api/fsw/inc/cfe_error.h index 5815887ca..ecdc09016 100644 --- a/modules/core_api/fsw/inc/cfe_error.h +++ b/modules/core_api/fsw/inc/cfe_error.h @@ -256,6 +256,14 @@ typedef int32 CFE_Status_t; */ #define CFE_EVS_INVALID_PARAMETER ((CFE_Status_t)0xc2000008) +/** + * @brief Event squelched + * + * Event squelched due to being sent at too high a rate + * + */ +#define CFE_EVS_APP_SQUELCHED ((CFE_Status_t)0xc2000009) + /** * @brief Not Implemented * diff --git a/modules/evs/eds/cfe_evs.xml b/modules/evs/eds/cfe_evs.xml index 4ca47111c..4e5adc9bb 100644 --- a/modules/evs/eds/cfe_evs.xml +++ b/modules/evs/eds/cfe_evs.xml @@ -179,6 +179,11 @@ \cfetlmmnemonic \EVS_APPENASTAT + + + \cfetlmmnemonic \EVS_SQUELCHEDC + + diff --git a/modules/evs/fsw/inc/cfe_evs_events.h b/modules/evs/fsw/inc/cfe_evs_events.h index fdbd6a46d..72003ed3e 100644 --- a/modules/evs/fsw/inc/cfe_evs_events.h +++ b/modules/evs/fsw/inc/cfe_evs_events.h @@ -466,6 +466,19 @@ * Invalid length for the command code in message ID #CFE_EVS_CMD_MID received on the EVS message pipe. */ #define CFE_EVS_LEN_ERR_EID 43 + +/** + * \brief EVS Events Squelched Error Event ID + * + * \par Type: ERROR + * + * \par Cause: + * + * Events generated in app at a rate in excess of + * #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST in one moment or + * #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC sustained + */ +#define CFE_EVS_SQUELCHED_ERR_EID 44 /**\}*/ #endif /* CFE_EVS_EVENTS_H */ diff --git a/modules/evs/fsw/inc/cfe_evs_msg.h b/modules/evs/fsw/inc/cfe_evs_msg.h index 242e1daa2..389adc515 100644 --- a/modules/evs/fsw/inc/cfe_evs_msg.h +++ b/modules/evs/fsw/inc/cfe_evs_msg.h @@ -1157,14 +1157,14 @@ typedef CFE_EVS_AppNameEventIDMaskCmd_t CFE_EVS_SetFilterCmd_t; /**********************************/ typedef struct CFE_EVS_AppTlmData { - CFE_ES_AppId_t AppID; /**< \cfetlmmnemonic \EVS_APPID - \brief Numerical application identifier */ - uint16 AppMessageSentCounter; /**< \cfetlmmnemonic \EVS_APPMSGSENTC - \brief Application message sent counter */ - uint8 AppEnableStatus; /**< \cfetlmmnemonic \EVS_APPENASTAT - \brief Application event service enable status */ - uint8 Padding; /**< \cfetlmmnemonic \EVS_SPARE2ALIGN3 - \brief Padding for 32 bit boundary */ + CFE_ES_AppId_t AppID; /**< \cfetlmmnemonic \EVS_APPID + \brief Numerical application identifier */ + uint16 AppMessageSentCounter; /**< \cfetlmmnemonic \EVS_APPMSGSENTC + \brief Application message sent counter */ + uint8 AppEnableStatus; /**< \cfetlmmnemonic \EVS_APPENASTAT + \brief Application event service enable status */ + uint8 AppMessageSquelchedCounter; /**< \cfetlmmnemonic \EVS_SQUELCHEDC + \brief Number of events squelched */ } CFE_EVS_AppTlmData_t; diff --git a/modules/evs/fsw/src/cfe_evs.c b/modules/evs/fsw/src/cfe_evs.c index c4b68e71c..6a79ab1a1 100644 --- a/modules/evs/fsw/src/cfe_evs.c +++ b/modules/evs/fsw/src/cfe_evs.c @@ -71,6 +71,7 @@ CFE_Status_t CFE_EVS_Register(const void *Filters, uint16 NumEventFilters, uint1 /* Initialize application event data */ AppDataPtr->ActiveFlag = true; AppDataPtr->EventTypesActiveFlag = CFE_PLATFORM_EVS_DEFAULT_TYPE_FLAG; + AppDataPtr->SquelchTokens = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000; /* Set limit for number of provided filters */ if (NumEventFilters < CFE_PLATFORM_EVS_MAX_EVENT_FILTERS) @@ -143,13 +144,20 @@ CFE_Status_t CFE_EVS_SendEvent(uint16 EventID, uint16 EventType, const char *Spe } else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false) { - /* Get current spacecraft time */ - Time = CFE_TIME_GetTime(); + if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true) + { + /* Get current spacecraft time */ + Time = CFE_TIME_GetTime(); - /* Send the event packets */ - va_start(Ptr, Spec); - EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); - va_end(Ptr); + /* Send the event packets */ + va_start(Ptr, Spec); + EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); + va_end(Ptr); + } + else + { + Status = CFE_EVS_APP_SQUELCHED; + } } } @@ -188,13 +196,20 @@ CFE_Status_t CFE_EVS_SendEventWithAppID(uint16 EventID, uint16 EventType, CFE_ES } else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false) { - /* Get current spacecraft time */ - Time = CFE_TIME_GetTime(); + if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true) + { + /* Get current spacecraft time */ + Time = CFE_TIME_GetTime(); - /* Send the event packets */ - va_start(Ptr, Spec); - EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); - va_end(Ptr); + /* Send the event packets */ + va_start(Ptr, Spec); + EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); + va_end(Ptr); + } + else + { + Status = CFE_EVS_APP_SQUELCHED; + } } return Status; @@ -231,10 +246,17 @@ CFE_Status_t CFE_EVS_SendTimedEvent(CFE_TIME_SysTime_t Time, uint16 EventID, uin } else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false) { - /* Send the event packets */ - va_start(Ptr, Spec); - EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); - va_end(Ptr); + if (EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true) + { + /* Send the event packets */ + va_start(Ptr, Spec); + EVS_GenerateEventTelemetry(AppDataPtr, EventID, EventType, &Time, Spec, Ptr); + va_end(Ptr); + } + else + { + Status = CFE_EVS_APP_SQUELCHED; + } } } diff --git a/modules/evs/fsw/src/cfe_evs_task.c b/modules/evs/fsw/src/cfe_evs_task.c index 5b6cd7c0b..6c85ef695 100644 --- a/modules/evs/fsw/src/cfe_evs_task.c +++ b/modules/evs/fsw/src/cfe_evs_task.c @@ -78,6 +78,8 @@ int32 CFE_EVS_EarlyInit(void) CFE_EVS_Global.EVS_TlmPkt.Payload.OutputPort = CFE_PLATFORM_EVS_PORT_DEFAULT; CFE_EVS_Global.EVS_TlmPkt.Payload.LogMode = CFE_PLATFORM_EVS_DEFAULT_LOG_MODE; + CFE_EVS_Global.EVS_EventBurstMax = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST; + /* Get a pointer to the CFE reset area from the BSP */ PspStatus = CFE_PSP_GetResetArea(&resetAreaAddr, &resetAreaSize); @@ -653,9 +655,11 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data) { if (EVS_AppDataIsUsed(AppDataPtr)) { - AppTlmDataPtr->AppID = EVS_AppDataGetID(AppDataPtr); - AppTlmDataPtr->AppEnableStatus = AppDataPtr->ActiveFlag; - AppTlmDataPtr->AppMessageSentCounter = AppDataPtr->EventCount; + AppTlmDataPtr->AppID = EVS_AppDataGetID(AppDataPtr); + AppTlmDataPtr->AppEnableStatus = AppDataPtr->ActiveFlag; + AppTlmDataPtr->AppMessageSentCounter = AppDataPtr->EventCount; + AppTlmDataPtr->AppMessageSquelchedCounter = AppDataPtr->SquelchedCount; + ++j; ++AppTlmDataPtr; } @@ -665,9 +669,10 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data) /* Clear unused portion of event state data in telemetry packet */ for (i = j; i < CFE_MISSION_ES_MAX_APPLICATIONS; i++) { - AppTlmDataPtr->AppID = CFE_ES_APPID_UNDEFINED; - AppTlmDataPtr->AppEnableStatus = false; - AppTlmDataPtr->AppMessageSentCounter = 0; + AppTlmDataPtr->AppID = CFE_ES_APPID_UNDEFINED; + AppTlmDataPtr->AppEnableStatus = false; + AppTlmDataPtr->AppMessageSentCounter = 0; + AppTlmDataPtr->AppMessageSquelchedCounter = 0; } CFE_SB_TimeStampMsg(CFE_MSG_PTR(CFE_EVS_Global.EVS_TlmPkt.TelemetryHeader)); @@ -1245,7 +1250,8 @@ int32 CFE_EVS_ResetAppCounterCmd(const CFE_EVS_ResetAppCounterCmd_t *data) if (Status == CFE_SUCCESS) { - AppDataPtr->EventCount = 0; + AppDataPtr->EventCount = 0; + AppDataPtr->SquelchedCount = 0; EVS_SendEvent(CFE_EVS_RSTEVTCNT_EID, CFE_EVS_EventType_DEBUG, "Reset Event Counter Command Received with AppName = %s", LocalName); @@ -1614,6 +1620,7 @@ int32 CFE_EVS_WriteAppDataFileCmd(const CFE_EVS_WriteAppDataFileCmd_t *data) AppDataFile.ActiveFlag = AppDataPtr->ActiveFlag; AppDataFile.EventCount = AppDataPtr->EventCount; AppDataFile.EventTypesActiveFlag = AppDataPtr->EventTypesActiveFlag; + AppDataFile.SquelchedCount = AppDataPtr->SquelchedCount; /* Copy application filter data to application file data record */ memcpy(AppDataFile.Filters, AppDataPtr->BinFilters, diff --git a/modules/evs/fsw/src/cfe_evs_task.h b/modules/evs/fsw/src/cfe_evs_task.h index 81c779028..f17ee6252 100644 --- a/modules/evs/fsw/src/cfe_evs_task.h +++ b/modules/evs/fsw/src/cfe_evs_task.h @@ -44,6 +44,7 @@ #include "cfe_platform_cfg.h" #include "cfe_mission_cfg.h" #include "osconfig.h" +#include "cfe_time.h" #include "cfe_evs_api_typedefs.h" #include "cfe_evs_log_typedef.h" #include "cfe_sb_api_typedefs.h" @@ -57,6 +58,7 @@ #define CFE_EVS_PIPE_DEPTH 32 #define CFE_EVS_MAX_EVENT_SEND_COUNT 65535 #define CFE_EVS_MAX_FILTER_COUNT 65535 +#define CFE_EVS_MAX_SQUELCH_COUNT 255 #define CFE_EVS_PIPE_NAME "EVS_CMD_PIPE" #define CFE_EVS_MAX_PORT_MSG_LENGTH (CFE_MISSION_EVS_MAX_MESSAGE_LENGTH + OS_MAX_API_NAME + 30) @@ -86,18 +88,23 @@ typedef struct EVS_BinFilter_t BinFilters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Array of binary filters */ - uint8 ActiveFlag; /* Application event service active flag */ - uint8 EventTypesActiveFlag; /* Application event types active flag */ - uint16 EventCount; /* Application event counter */ + uint8 ActiveFlag; /* Application event service active flag */ + uint8 EventTypesActiveFlag; /* Application event types active flag */ + uint16 EventCount; /* Application event counter */ + OS_time_t LastSquelchCreditableTime; /* Time of last squelch token return */ + int32 SquelchTokens; /* Application event squelch token counter */ + uint8 SquelchedCount; /* Application events squelched counter */ } EVS_AppData_t; typedef struct { - char AppName[OS_MAX_API_NAME]; /* Application name */ - uint8 ActiveFlag; /* Application event service active flag */ - uint8 EventTypesActiveFlag; /* Application event types active flag */ - uint16 EventCount; /* Application event counter */ + char AppName[OS_MAX_API_NAME]; /* Application name */ + uint8 ActiveFlag; /* Application event service active flag */ + uint8 EventTypesActiveFlag; /* Application event types active flag */ + uint16 EventCount; /* Application event counter */ + uint8 SquelchedCount; /* Application events squelched counter */ + uint8 Spare[3]; EVS_BinFilter_t Filters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Application event filters */ } CFE_EVS_AppDataFile_t; @@ -117,7 +124,7 @@ typedef struct CFE_SB_PipeId_t EVS_CommandPipe; osal_id_t EVS_SharedDataMutexID; CFE_ES_AppId_t EVS_AppID; - + uint32 EVS_EventBurstMax; } CFE_EVS_Global_t; /* diff --git a/modules/evs/fsw/src/cfe_evs_utils.c b/modules/evs/fsw/src/cfe_evs_utils.c index fb8e9de43..669aefda8 100644 --- a/modules/evs/fsw/src/cfe_evs_utils.c +++ b/modules/evs/fsw/src/cfe_evs_utils.c @@ -28,6 +28,7 @@ /* Include Files */ #include "cfe_evs_module_all.h" /* All EVS internal definitions and API */ +#include "cfe_evs_utils.h" #include #include @@ -288,6 +289,126 @@ bool EVS_IsFiltered(EVS_AppData_t *AppDataPtr, uint16 EventID, uint16 EventType) return (Filtered); } +/*---------------------------------------------------------------- + * + * Function: EVS_CheckAndIncrementSquelchTokens + * + * Application-scope internal function + * See description in header file for argument/return detail + * + *-----------------------------------------------------------------*/ +bool EVS_CheckAndIncrementSquelchTokens(EVS_AppData_t *AppDataPtr) +{ + bool NotSquelched = true; + bool SendSquelchEvent = false; + OS_time_t CurrentTime; + int64 DeltaTimeMs; + int64 CreditCount; + char AppName[OS_MAX_API_NAME]; + + /* Set maximum token credits to burst size */ + const int32 UPPER_THRESHOLD = CFE_EVS_Global.EVS_EventBurstMax * 1000; + /* + * Set lower threshold to stop decrementing + * Make this -CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST to add some hysteresis + * Events will resume (CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / + * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC + 1 / + * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC) seconds after flooding stops if + * saturated + */ + const int32 LOWER_THRESHOLD = -CFE_EVS_Global.EVS_EventBurstMax * 1000; + + /* + * Set this to 1000 to avoid integer division while computing CreditCount + */ + const int32 EVENT_COST = 1000; + + if (CFE_EVS_Global.EVS_EventBurstMax != 0) + { + /* + * We use a timer here since configurations are not guaranteed to send EVS HK wakeups at 1Hz + * Use a non-settable timer to prevent this from breaking w/ time changes + */ + OS_MutSemTake(CFE_EVS_Global.EVS_SharedDataMutexID); + CFE_PSP_GetTime(&CurrentTime); + DeltaTimeMs = OS_TimeGetTotalMilliseconds(OS_TimeSubtract(CurrentTime, AppDataPtr->LastSquelchCreditableTime)); + + /* Calculate how many tokens to credit in elapsed time since last creditable event */ + CreditCount = DeltaTimeMs * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC; + + /* + * Don't immediately credit < 1 event worth of credits; defer until + * enough time that CreditCount > EVENT_COST + * + * This prevents condition where credits would creep down slowly + * through the range which squelch event messages are emitted causing + * those events to be spammed instead, defeating the suppression. + */ + if (CreditCount >= EVENT_COST) + { + /* Update last squelch returned time if we credited any tokens */ + AppDataPtr->LastSquelchCreditableTime = CurrentTime; + + /* + * Add Credits, to a maximum of UPPER_THRESHOLD + * Shouldn't rollover, as calculations are done in int64 space due to + * promotion rules then bounded before demotion + */ + if (AppDataPtr->SquelchTokens + CreditCount > UPPER_THRESHOLD) + { + AppDataPtr->SquelchTokens = UPPER_THRESHOLD; + } + else + { + AppDataPtr->SquelchTokens += (int32)CreditCount; + } + } + + if (AppDataPtr->SquelchTokens <= 0) + { + if (AppDataPtr->SquelchedCount < CFE_EVS_MAX_SQUELCH_COUNT) + { + AppDataPtr->SquelchedCount++; + } + NotSquelched = false; + + /* + * Send squelch event message if cross threshold. This has to be a + * range between -EVENT_COST and 0 due to non-whole event-cost credits being + * returned allowing 0 to be skipped over. This is solved by + * checking a range and ensuring EVENT_COST credits are returned at minimum. + */ + if (AppDataPtr->SquelchTokens > -EVENT_COST && CreditCount < EVENT_COST) + { + // Set flag and send event later, since we still own mutex + SendSquelchEvent = true; + } + } + + /* + * Subtract event cost + */ + if (AppDataPtr->SquelchTokens - EVENT_COST < LOWER_THRESHOLD) + { + AppDataPtr->SquelchTokens = LOWER_THRESHOLD; + } + else + { + AppDataPtr->SquelchTokens -= EVENT_COST; + } + + OS_MutSemGive(CFE_EVS_Global.EVS_SharedDataMutexID); + + if (SendSquelchEvent) + { + CFE_ES_GetAppName(AppName, EVS_AppDataGetID(AppDataPtr), sizeof(AppName)); + EVS_SendEvent(CFE_EVS_SQUELCHED_ERR_EID, CFE_EVS_EventType_ERROR, "Events squelched, AppName = %s", + AppName); + } + } + return NotSquelched; +} + /*---------------------------------------------------------------- * * Function: EVS_FindEventID @@ -558,6 +679,8 @@ int32 EVS_SendEvent(uint16 EventID, uint16 EventType, const char *Spec, ...) AppDataPtr = EVS_GetAppDataByID(CFE_EVS_Global.EVS_AppID); /* Unlikely, but possible that an EVS event filter was added by command */ + /* Note that we do not squelch events coming from EVS to prevent event recursion, + * and EVS is assumed to be "well-behaved" */ if (EVS_AppDataIsMatch(AppDataPtr, CFE_EVS_Global.EVS_AppID) && EVS_IsFiltered(AppDataPtr, EventID, EventType) == false) { diff --git a/modules/evs/fsw/src/cfe_evs_utils.h b/modules/evs/fsw/src/cfe_evs_utils.h index 4a18ec3ea..be4457dff 100644 --- a/modules/evs/fsw/src/cfe_evs_utils.h +++ b/modules/evs/fsw/src/cfe_evs_utils.h @@ -190,6 +190,18 @@ int32 EVS_NotRegistered(EVS_AppData_t *AppDataPtr, CFE_ES_AppId_t CallerID); */ bool EVS_IsFiltered(EVS_AppData_t *AppDataPtr, uint16 EventID, uint16 EventType); +/*---------------------------------------------------------------------------------------*/ +/** + * @brief Check if event is squelched + * + * This routine returns false if the squelch token counter has become negative. + * Otherwise a value of true is returned. In addition, it updates the squelch + * token counter based on time, and emits an event message if squelched. + * + * If #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST == 0, this function returns true and is otherwise a no-op + */ +bool EVS_CheckAndIncrementSquelchTokens(EVS_AppData_t *AppDataPtr); + /*---------------------------------------------------------------------------------------*/ /** * @brief Find the filter record corresponding to the given event ID diff --git a/modules/evs/fsw/src/cfe_evs_verify.h b/modules/evs/fsw/src/cfe_evs_verify.h index 3a8391a47..7c8c8df6e 100644 --- a/modules/evs/fsw/src/cfe_evs_verify.h +++ b/modules/evs/fsw/src/cfe_evs_verify.h @@ -54,6 +54,14 @@ #error CFE_PLATFORM_EVS_PORT_DEFAULT cannot be greater than 0x0F! #endif +#if CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST > INT32_MAX / 1000 +#error CFE_PLATFORM_EVS_MAX_APP_EVENTS_PER_SEC cannot be greater than INT32_MAX/1000 +#endif + +#if CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC > CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST +#error CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC must be <= CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST +#endif + /* ** Validate task stack size... */ diff --git a/modules/evs/ut-coverage/evs_UT.c b/modules/evs/ut-coverage/evs_UT.c index e4cd2c4ba..0e6499449 100644 --- a/modules/evs/ut-coverage/evs_UT.c +++ b/modules/evs/ut-coverage/evs_UT.c @@ -37,6 +37,8 @@ ** Includes */ #include "evs_UT.h" +#include "cfe_evs.h" +#include "utstubs.h" static const char *EVS_SYSLOG_MSGS[] = { NULL, @@ -128,6 +130,8 @@ typedef struct CFE_MSG_Size_t Size; } UT_EVS_MSGInitData_t; +typedef CFE_Status_t (*UT_EVS_SendEventFunc_t)(uint32); + /* Message init hook to stora last MsgId passed in */ static int32 UT_EVS_MSGInitHook(void *UserObj, int32 StubRetcode, uint32 CallCount, const UT_StubContext_t *Context) { @@ -184,6 +188,47 @@ static void UT_EVS_DoGenericCheckEvents(void (*Func)(void), UT_EVS_EventCapture_ UT_SetHookFunction(UT_KEY(CFE_SB_TransmitMsg), NULL, NULL); } +static CFE_Status_t UT_EVS_SendSquelchedEvent(uint32 EventId) +{ + return CFE_EVS_SendEvent(EventId, CFE_EVS_EventType_INFORMATION, "Suppressed Message"); +} + +static CFE_Status_t UT_EVS_SendSquelchedEventWithAppId(uint32 EventId) +{ + CFE_ES_AppId_t AppID; + CFE_ES_GetAppID(&AppID); + return CFE_EVS_SendEventWithAppID(EventId, CFE_EVS_EventType_INFORMATION, AppID, "Suppressed Message"); +} + +static CFE_Status_t UT_EVS_SendSquelchedTimedEvent(uint32 EventId) +{ + CFE_TIME_SysTime_t Time = {0, 0}; + return CFE_EVS_SendTimedEvent(Time, EventId, CFE_EVS_EventType_INFORMATION, "Suppressed Message"); +} + +static void UT_EVS_ResetSquelchCurrentContext(void) +{ + EVS_AppData_t *AppDataPtr; + + EVS_GetCurrentContext(&AppDataPtr, NULL); + if (AppDataPtr) + { + AppDataPtr->SquelchedCount = 0; + AppDataPtr->SquelchTokens = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 1000; + AppDataPtr->LastSquelchCreditableTime = OS_TimeAssembleFromMilliseconds(0, 0); + } +} + +static void UT_EVS_DisableSquelch(void) +{ + CFE_EVS_Global.EVS_EventBurstMax = 0; +} + +static void UT_EVS_ResetSquelch(void) +{ + CFE_EVS_Global.EVS_EventBurstMax = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST; +} + /* ** Functions */ @@ -207,6 +252,7 @@ void UtTest_Setup(void) UT_ADD_TEST(Test_EventCmd); UT_ADD_TEST(Test_FilterCmd); UT_ADD_TEST(Test_InvalidCmd); + UT_ADD_TEST(Test_Squelching); UT_ADD_TEST(Test_Misc); } @@ -360,6 +406,8 @@ void Test_Init(void) UT_EVS_DoDispatchCheckEvents(&bitmaskcmd, sizeof(bitmaskcmd), UT_TPID_CFE_EVS_CMD_DISABLE_PORTS_CC, &UT_EVS_EventBuf); UtAssert_UINT32_EQ(UT_EVS_EventBuf.EventID, CFE_EVS_DISPORT_EID); + + UT_EVS_DisableSquelch(); } /* @@ -1814,6 +1862,126 @@ void Test_InvalidCmd(void) UtAssert_UINT32_EQ(UT_EVS_EventBuf.EventID, CFE_EVS_LEN_ERR_EID); } +/* +** Test squelching +*/ +void Test_Squelching(void) +{ + int i, j; + OS_time_t InjectedTime; + CFE_EVS_LongEventTlm_t CapturedTlm; + UT_SoftwareBusSnapshot_Entry_t SnapshotData = {.MsgId = CFE_SB_MSGID_WRAP_VALUE(CFE_EVS_LONG_EVENT_MSG_MID), + .SnapshotBuffer = &CapturedTlm, + .SnapshotOffset = 0, + .SnapshotSize = sizeof(CapturedTlm)}; + const uint32 EVENT_ID = 142; + CFE_Status_t EVS_Retval; + UT_EVS_SendEventFunc_t SendEventFuncs[] = {UT_EVS_SendSquelchedEvent, UT_EVS_SendSquelchedEventWithAppId, + UT_EVS_SendSquelchedTimedEvent}; + + EVS_AppData_t *AppDataPtr; + + UtPrintf("Begin Test Squelching"); + + UT_InitData(); + UT_EVS_ResetSquelch(); + CFE_EVS_Global.EVS_TlmPkt.Payload.MessageFormatMode = CFE_EVS_MsgFormat_LONG; + + UT_SetHookFunction(UT_KEY(CFE_SB_TransmitMsg), UT_SoftwareBusSnapshotHook, &SnapshotData); + + for (j = 0; j < sizeof(SendEventFuncs) / sizeof(SendEventFuncs[0]); j++) + { + UT_EVS_ResetSquelchCurrentContext(); + EVS_GetCurrentContext(&AppDataPtr, NULL); + AppDataPtr->SquelchedCount = CFE_EVS_MAX_SQUELCH_COUNT - CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST; + + SnapshotData.Count = 0; + /* + * Test CFE_EVS_SendEvent squelching + */ + for (i = 0; i < CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST * 3; i++) + { + InjectedTime = OS_TimeAssembleFromMilliseconds(0, 0); + + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + if (i < CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST) + { + UtAssert_UINT32_EQ(SnapshotData.Count, i + 1); + UtAssert_UINT32_EQ(CapturedTlm.Payload.PacketID.EventID, EVENT_ID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_SUCCESS); + } + else if (i == CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST) + { + UtAssert_UINT32_EQ(SnapshotData.Count, i + 1); + UtAssert_UINT32_EQ(CapturedTlm.Payload.PacketID.EventID, CFE_EVS_SQUELCHED_ERR_EID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_EVS_APP_SQUELCHED); + } + else + { + UtAssert_UINT32_EQ(SnapshotData.Count, CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST + 1); + UtAssert_UINT32_EQ(EVS_Retval, CFE_EVS_APP_SQUELCHED); + } + } + /* + * Test unsquelching w/ passage of time + */ + /* Allow time for tokens to return */ + InjectedTime = OS_TimeAssembleFromMilliseconds(0, 1000 * CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / + CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC + + 1000 / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC); + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + UtAssert_UINT32_EQ(SnapshotData.Count, CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST + 2); + UtAssert_UINT32_EQ(CapturedTlm.Payload.PacketID.EventID, EVENT_ID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_SUCCESS); + + /* + * Test re-squelching event message + */ + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + UtAssert_UINT32_EQ(SnapshotData.Count, CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST + 3); + UtAssert_UINT32_EQ(CapturedTlm.Payload.PacketID.EventID, CFE_EVS_SQUELCHED_ERR_EID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_EVS_APP_SQUELCHED); + + /* + * Test re-squelching + */ + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + UtAssert_UINT32_EQ(SnapshotData.Count, CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST + 3); + UtAssert_UINT32_EQ(EVS_Retval, CFE_EVS_APP_SQUELCHED); + + /* + * Test positive saturation of tokens + */ + InjectedTime = OS_TimeAssembleFromMilliseconds(0, 3 * 1000 * CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST / + CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC + + 1000 / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC); + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + UtAssert_UINT32_EQ(SnapshotData.Count, CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST + 4); + UtAssert_UINT32_EQ(CapturedTlm.Payload.PacketID.EventID, EVENT_ID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_SUCCESS); + UtAssert_UINT32_EQ(AppDataPtr->SquelchedCount, CFE_EVS_MAX_SQUELCH_COUNT); + + /* + * Test boundary condition where SquelchTokens > -EVENT_COST && CreditCount >= EVENT_COST + */ + InjectedTime = OS_TimeAssembleFromMilliseconds(1, 1200 / CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC); + UT_SetDataBuffer(UT_KEY(CFE_PSP_GetTime), &InjectedTime, sizeof(InjectedTime), false); + AppDataPtr->SquelchedCount = 0; + AppDataPtr->SquelchTokens = -1500; + AppDataPtr->LastSquelchCreditableTime = OS_TimeAssembleFromMilliseconds(1, 0); + EVS_Retval = SendEventFuncs[j](EVENT_ID); + UtAssert_UINT32_EQ(EVS_Retval, CFE_EVS_APP_SQUELCHED); + } + + UT_EVS_DisableSquelch(); + UT_SetHookFunction(UT_KEY(CFE_SB_TransmitMsg), NULL, NULL); +} + /* ** Test miscellaneous functionality */ diff --git a/modules/evs/ut-coverage/evs_UT.h b/modules/evs/ut-coverage/evs_UT.h index c2d16fbb9..71b0de9ed 100644 --- a/modules/evs/ut-coverage/evs_UT.h +++ b/modules/evs/ut-coverage/evs_UT.h @@ -248,6 +248,22 @@ void Test_FilterCmd(void); ******************************************************************************/ void Test_InvalidCmd(void); +/*****************************************************************************/ +/** +** \brief Test squelching of app events +** +** \par Description +** This function tests squelching of app events that exceeds threshold +** of events sent per second. +** +** \par Assumptions, External Events, and Notes: +** None +** +** \returns +** This function does not return a value. +******************************************************************************/ +void Test_Squelching(void); + /*****************************************************************************/ /** ** \brief Test miscellaneous functionality