Merge pull request nasa#595 from jphickey/fix-456-perflog-race

Fix nasa#456, Perf log threading and concurrency issues

fsw/cfe-core/src/es/cfe_es_backgroundtask.c

@@ -0,0 +1,250 @@
+/*
+**  GSC-18128-1, "Core Flight Executive Version 6.7"
+**
+**  Copyright (c) 2006-2019 United States Government as represented by
+**  the Administrator of the National Aeronautics and Space Administration.
+**  All Rights Reserved.
+**
+**  Licensed under the Apache License, Version 2.0 (the "License");
+**  you may not use this file except in compliance with the License.
+**  You may obtain a copy of the License at
+**
+**    http://www.apache.org/licenses/LICENSE-2.0
+**
+**  Unless required by applicable law or agreed to in writing, software
+**  distributed under the License is distributed on an "AS IS" BASIS,
+**  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+**  See the License for the specific language governing permissions and
+**  limitations under the License.
+*/
+
+/*
+** File: cfe_es_backgroundtask.c
+**
+** Purpose: This file contains the implementation of the ES "background task"
+**
+** This task sits idle most of the time, but is woken by the ES application
+** for various maintenance duties that may take time to execute, such as
+** writing status/log files.
+**
+*/
+
+/*
+** Include Section
+*/
+
+#include <string.h>
+
+#include "osapi.h"
+#include "private/cfe_private.h"
+#include "cfe_es_perf.h"
+#include "cfe_es_global.h"
+#include "cfe_es_task.h"
+
+#define CFE_ES_BACKGROUND_SEM_NAME             "ES_BackgroundSem"
+#define CFE_ES_BACKGROUND_CHILD_NAME           "ES_BackgroundTask"
+#define CFE_ES_BACKGROUND_CHILD_STACK_PTR      NULL
+#define CFE_ES_BACKGROUND_CHILD_STACK_SIZE     CFE_PLATFORM_ES_PERF_CHILD_STACK_SIZE
+#define CFE_ES_BACKGROUND_CHILD_PRIORITY       CFE_PLATFORM_ES_PERF_CHILD_PRIORITY
+#define CFE_ES_BACKGROUND_CHILD_FLAGS          0
+#define CFE_ES_BACKGROUND_MAX_IDLE_DELAY       30000        /* 30 seconds */
+
+
+typedef struct
+{
+    bool (*RunFunc)(uint32 ElapsedTime, void *Arg);
+    void *JobArg;
+    uint32 ActivePeriod;            /**< max wait/delay time between calls when job is active */
+    uint32 IdlePeriod;              /**< max wait/delay time between calls when job is idle */
+} CFE_ES_BackgroundJobEntry_t;
+
+/*
+ * List of "background jobs"
+ *
+ * This is just a list of functions to periodically call from the context of the background task,
+ * and can be added/extended as needed.
+ *
+ * Each Job function returns a boolean, and should return "true" if it is active, or "false" if it is idle.
+ *
+ * This uses "cooperative multitasking" -- the function should do some limited work, then return to the
+ * background task.  It will be called again after a delay period to do more work.
+ */
+const CFE_ES_BackgroundJobEntry_t CFE_ES_BACKGROUND_JOB_TABLE[] =
+{
+        {   /* Performance Log Data Dump to file */
+                .RunFunc = CFE_ES_RunPerfLogDump,
+                .JobArg = &CFE_ES_TaskData.BackgroundPerfDumpState,
+                .ActivePeriod = CFE_PLATFORM_ES_PERF_CHILD_MS_DELAY,
+                .IdlePeriod = CFE_PLATFORM_ES_PERF_CHILD_MS_DELAY * 1000
+        }
+};
+
+#define CFE_ES_BACKGROUND_NUM_JOBS          (sizeof(CFE_ES_BACKGROUND_JOB_TABLE) / sizeof(CFE_ES_BACKGROUND_JOB_TABLE[0]))
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* Name: CFE_ES_BackgroundTask                                                   */
+/*                                                                               */
+/* Purpose: A helper task for low priority routines that may take time to        */
+/* execute, such as writing log files.                                           */
+/*                                                                               */
+/* Assumptions and Notes: This is started from the ES initialization, and        */
+/* pends on a semaphore until a work request comes in.  This is intended to      */
+/* avoid the need to create a child task "on demand" when work items arrive,     */
+/* which is a form of dynamic allocation.                                        */
+/*                                                                               */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+void CFE_ES_BackgroundTask(void)
+{
+    int32 status;
+    uint32 JobTotal;
+    uint32 NumJobsRunning;
+    uint32 NextDelay;
+    uint32 ElapsedTime;
+    OS_time_t CurrTime;
+    OS_time_t LastTime;
+    const CFE_ES_BackgroundJobEntry_t *JobPtr;
+
+    status = CFE_ES_RegisterChildTask();
+    if (status != CFE_SUCCESS)
+    {
+        /* should never occur */
+        CFE_ES_WriteToSysLog("CFE_ES: Background Task Failed to register: %08lx\n", (unsigned long)status);
+        return;
+    }
+
+    CFE_PSP_GetTime(&LastTime);
+
+    while (true)
+    {
+        /*
+         * compute the elapsed time (difference) between last
+         * execution and now, in microseconds.
+         *
+         * Note this calculation is done as a uint32 which will overflow
+         * after about 35 minutes, but the max delays ensure that this
+         * executes at least every few seconds, so that should never happen.
+         */
+        CFE_PSP_GetTime(&CurrTime);
+        ElapsedTime = 1000000 * (CurrTime.seconds - LastTime.seconds);
+        ElapsedTime += CurrTime.microsecs;
+        ElapsedTime -= LastTime.microsecs;
+        LastTime = CurrTime;
+
+        /*
+         * convert to milliseconds.
+         * we do not really need high precision
+         * for background task timings
+         */
+        ElapsedTime /= 1000;
+
+        NextDelay = CFE_ES_BACKGROUND_MAX_IDLE_DELAY;   /* default; will be adjusted based on active jobs */
+        JobPtr = CFE_ES_BACKGROUND_JOB_TABLE;
+        JobTotal = CFE_ES_BACKGROUND_NUM_JOBS;
+        NumJobsRunning = 0;
+
+        while (JobTotal > 0)
+        {
+            /*
+             * call the background job -
+             * if it returns "true" that means it is active,
+             * if it returns "false" that means it is idle
+             */
+            if (JobPtr->RunFunc != NULL && JobPtr->RunFunc(ElapsedTime, JobPtr->JobArg))
+            {
+                ++NumJobsRunning;
+
+                if (JobPtr->ActivePeriod != 0 && NextDelay > JobPtr->ActivePeriod)
+                {
+                    /* next delay is based on this active job wait time */
+                    NextDelay = JobPtr->ActivePeriod;
+                }
+            }
+            else if (JobPtr->IdlePeriod != 0 && NextDelay > JobPtr->IdlePeriod)
+            {
+                /* next delay is based on this idle job wait time */
+                NextDelay = JobPtr->IdlePeriod;
+            }
+            --JobTotal;
+            ++JobPtr;
+        }
+
+        CFE_ES_Global.BackgroundTask.NumJobsRunning = NumJobsRunning;
+
+        status = OS_BinSemTimedWait(CFE_ES_Global.BackgroundTask.WorkSem, NextDelay);
+        if (status != OS_SUCCESS && status != OS_SEM_TIMEOUT)
+        {
+            /* should never occur */
+            CFE_ES_WriteToSysLog("CFE_ES: Failed to take background sem: %08lx\n", (unsigned long)status);
+            break;
+        }
+
+    }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* Name: CFE_ES_BackgroundInit                                                   */
+/*                                                                               */
+/* Purpose: Initialize the background task                                       */
+/*                                                                               */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+int32 CFE_ES_BackgroundInit(void)
+{
+    int32 status;
+
+    status = OS_BinSemCreate(&CFE_ES_Global.BackgroundTask.WorkSem, CFE_ES_BACKGROUND_SEM_NAME, 0, 0);
+    if (status != OS_SUCCESS)
+    {
+        CFE_ES_WriteToSysLog("CFE_ES: Failed to create background sem: %08lx\n", (unsigned long)status);
+        return status;
+    }
+
+    /* Spawn a task to write the performance data to a file */
+    status = CFE_ES_CreateChildTask(&CFE_ES_Global.BackgroundTask.TaskID,
+            CFE_ES_BACKGROUND_CHILD_NAME,
+            CFE_ES_BackgroundTask,
+            CFE_ES_BACKGROUND_CHILD_STACK_PTR,
+            CFE_ES_BACKGROUND_CHILD_STACK_SIZE,
+            CFE_ES_BACKGROUND_CHILD_PRIORITY,
+            CFE_ES_BACKGROUND_CHILD_FLAGS);
+
+    if (status != OS_SUCCESS)
+    {
+        CFE_ES_WriteToSysLog("CFE_ES: Failed to create background task: %08lx\n", (unsigned long)status);
+        return status;
+    }
+
+    return CFE_SUCCESS;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* Name: CFE_ES_BackgroundCleanup                                                */
+/*                                                                               */
+/* Purpose: Exit/Stop the background task                                        */
+/*                                                                               */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+void CFE_ES_BackgroundCleanup(void)
+{
+    CFE_ES_DeleteChildTask(CFE_ES_Global.BackgroundTask.TaskID);
+    OS_BinSemDelete(CFE_ES_Global.BackgroundTask.WorkSem);
+
+    CFE_ES_Global.BackgroundTask.TaskID = 0;
+    CFE_ES_Global.BackgroundTask.WorkSem = 0;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* Name: CFE_ES_BackgroundWakeup                                                 */
+/*                                                                               */
+/* Purpose: Wake up the background task                                          */
+/* Notifies the background task to perform an extra poll for new work            */
+/*                                                                               */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+void CFE_ES_BackgroundWakeup(void)
+{
+    /* wake up the background task by giving the sem.
+     * This is "informational" and not strictly required,
+     * but it will make the task immediately wake up and check for new
+     * work if it was idle. */
+    OS_BinSemGive(CFE_ES_Global.BackgroundTask.WorkSem);
+}
+
+

fsw/cfe-core/src/es/cfe_es_global.h

@@ -19,8 +19,8 @@
 */
 
 /*
-**  File: 
-**  cfe_es_global.h 
+**  File:
+**  cfe_es_global.h
 **
 **  Purpose:
 **  This file contains the ES global data definitions.
@@ -66,13 +66,23 @@ typedef struct
 {
    bool           RecordUsed;                     /* Is the record used(1) or available(0) ? */
    uint32         Counter;
-   char           CounterName[OS_MAX_API_NAME];   /* Counter Name */      
+   char           CounterName[OS_MAX_API_NAME];   /* Counter Name */
 } CFE_ES_GenCounterRecord_t;
 
+/*
+ * Encapsulates the state of the ES background task
+ */
+typedef struct
+{
+    uint32 TaskID;          /**< OSAL ID of the background task */
+    uint32 WorkSem;         /**< Semaphore that is given whenever background work is pending */
+    uint32 NumJobsRunning;  /**< Current Number of active jobs (updated by background task) */
+} CFE_ES_BackgroundTaskState_t;
+
 
 /*
 ** Executive Services Global Memory Data
-** This is the regular global data that is not preserved on a 
+** This is the regular global data that is not preserved on a
 **  processor reset.
 */
 typedef struct
@@ -81,20 +91,25 @@ typedef struct
    ** Debug Variables
    */
    CFE_ES_DebugVariables_t DebugVars;
-   
+
    /*
    ** Shared Data Semaphore
    */
    uint32 SharedDataMutex;
-
+
+   /*
+   ** Performance Data Mutex
+   */
+   uint32 PerfDataMutex;
+
    /*
    ** Startup Sync
    */
    uint32  SystemState;
 
    /*
    ** ES Task Table
-   */ 
+   */
    uint32              RegisteredTasks;
    CFE_ES_TaskRecord_t TaskTable[OS_MAX_TASKS];
 
@@ -104,7 +119,7 @@ typedef struct
    uint32             RegisteredCoreApps;
    uint32             RegisteredExternalApps;
    CFE_ES_AppRecord_t AppTable[CFE_PLATFORM_ES_MAX_APPLICATIONS];
-   
+
    /*
    ** ES Shared Library Table
    */
@@ -121,12 +136,19 @@ typedef struct
    */
    CFE_ES_CDSVariables_t CDSVars;
 
+   /*
+    * Background task for handling long-running, non real time tasks
+    * such as maintenance, file writes, and other items.
+    */
+   CFE_ES_BackgroundTaskState_t BackgroundTask;
+
+
 } CFE_ES_Global_t;
 
 /*
 ** The Executive Services Global Data declaration
 */
-extern CFE_ES_Global_t CFE_ES_Global; 
+extern CFE_ES_Global_t CFE_ES_Global;
 
 /*
 ** The Executive Services Nonvolatile Data declaration