[perf] Update redux store with only the values that changed

app/client/src/reducers/evaluationReducers/treeReducer.ts

@@ -1,6 +1,7 @@
-import { createImmerReducer } from "utils/AppsmithUtils";
-import { DataTree } from "entities/DataTree/dataTreeFactory";
 import { ReduxAction, ReduxActionTypes } from "constants/ReduxActionConstants";
+import { applyChange, Diff } from "deep-diff";
+import { DataTree } from "entities/DataTree/dataTreeFactory";
+import { createImmerReducer } from "utils/AppsmithUtils";
 
 export type EvaluatedTreeState = DataTree;
 
@@ -9,8 +10,24 @@ const initialState: EvaluatedTreeState = {};
 const evaluatedTreeReducer = createImmerReducer(initialState, {
   [ReduxActionTypes.SET_EVALUATED_TREE]: (
     state: EvaluatedTreeState,
-    action: ReduxAction<DataTree>,
-  ) => action.payload,
+    action: ReduxAction<{
+      dataTree: DataTree;
+      updates: Diff<DataTree, DataTree>[];
+      removedPaths: [string];
+    }>,
+  ) => {
+    const { dataTree, updates } = action.payload;
+    if (Object.keys(dataTree).length) {
+      return dataTree;
+    }
+    for (const update of updates) {
+      // Null check for typescript
+      if (!Array.isArray(update.path) || update.path.length === 0) {
+        continue;
+      }
+      applyChange(state, undefined, update);
+    }
+  },
   [ReduxActionTypes.FETCH_PAGE_INIT]: () => initialState,
 });
 

app/client/src/sagas/EvaluationsSaga.ts

@@ -14,7 +14,10 @@ import {
   ReduxActionTypes,
   ReduxActionWithoutPayload,
 } from "constants/ReduxActionConstants";
-import { getUnevaluatedDataTree } from "selectors/dataTreeSelectors";
+import {
+  getDataTree,
+  getUnevaluatedDataTree,
+} from "selectors/dataTreeSelectors";
 import WidgetFactory, { WidgetTypeConfigMap } from "../utils/WidgetFactory";
 import { GracefulWorkerService } from "utils/WorkerUtil";
 import Worker from "worker-loader!../workers/evaluation.worker";
@@ -312,7 +315,6 @@ function* logSuccessfulBindings(
 function* updateTernDefinitions(
   dataTree: DataTree,
   evaluationOrder: string[],
-  removedPaths: string[],
   isFirstEvaluation: boolean,
 ) {
   const updatedEntities: Set<string> = new Set();
@@ -333,12 +335,12 @@ function* updateTernDefinitions(
       TernServer.updateDef(name, def);
     }
   });
-  removedPaths.forEach((path) => {
-    // No '.' means that the path is an entity name
-    if (path.split(".").length === 1) {
-      TernServer.removeDef(path);
-    }
-  });
+  // removedPaths.forEach((path) => {
+  //   // No '.' means that the path is an entity name
+  //   if (path.split(".").length === 1) {
+  //     TernServer.removeDef(path);
+  //   }
+  // });
 }
 
 function* postEvalActionDispatcher(
@@ -355,6 +357,7 @@ function* evaluateTreeSaga(
 ) {
   const unevalTree = yield select(getUnevaluatedDataTree);
   log.debug({ unevalTree });
+  console.log(unevalTree);
   PerformanceTracker.startAsyncTracking(
     PerformanceTransactionName.DATA_TREE_EVALUATION,
   );
@@ -372,33 +375,40 @@ function* evaluateTreeSaga(
     errors,
     evaluationOrder,
     logs,
-    removedPaths,
+    updates,
   } = workerResponse;
   PerformanceTracker.stopAsyncTracking(
     PerformanceTransactionName.DATA_TREE_EVALUATION,
   );
-  log.debug({ dataTree: dataTree });
-  logs.forEach((evalLog: any) => log.debug(evalLog));
-  yield call(evalErrorHandler, errors, dataTree, evaluationOrder);
-  yield fork(logSuccessfulBindings, unevalTree, dataTree, evaluationOrder);
-  yield fork(
-    updateTernDefinitions,
-    dataTree,
-    evaluationOrder,
-    removedPaths,
-    isFirstEvaluation,
-  );
-
   PerformanceTracker.startAsyncTracking(
     PerformanceTransactionName.SET_EVALUATED_TREE,
   );
   yield put({
     type: ReduxActionTypes.SET_EVALUATED_TREE,
-    payload: dataTree,
+    payload: { dataTree, updates },
   });
   PerformanceTracker.stopAsyncTracking(
     PerformanceTransactionName.SET_EVALUATED_TREE,
   );
+
+  const updatedDataTree = yield select(getDataTree);
+
+  log.debug({ dataTree: updatedDataTree });
+  logs.forEach((evalLog: any) => log.debug(evalLog));
+  yield call(evalErrorHandler, errors, updatedDataTree, evaluationOrder);
+  yield fork(
+    logSuccessfulBindings,
+    unevalTree,
+    updatedDataTree,
+    evaluationOrder,
+  );
+  yield fork(
+    updateTernDefinitions,
+    updatedDataTree,
+    evaluationOrder,
+    isFirstEvaluation,
+  );
+
   yield put({
     type: ReduxActionTypes.SET_EVALUATION_INVERSE_DEPENDENCY_MAP,
     payload: { inverseDependencyMap: dependencies },

app/client/src/workers/DataTreeEvaluator.ts

@@ -36,6 +36,7 @@ import {
   translateDiffEventToDataTreeDiffEvent,
   trimDependantChangePaths,
   validateWidgetProperty,
+  isDynamicLeaf,
 } from "workers/evaluationUtils";
 import _ from "lodash";
 import { applyChange, Diff, diff } from "deep-diff";
@@ -50,12 +51,6 @@ import evaluate, { EvalResult } from "workers/evaluate";
 import { substituteDynamicBindingWithValues } from "workers/evaluationSubstitution";
 import { Severity } from "entities/AppsmithConsole";
 
-type EvaluateDataTreeResponse = {
-  dataTree: DataTree;
-  evaluationOrder: string[];
-  removedPaths: string[];
-};
-
 export default class DataTreeEvaluator {
   dependencyMap: DependencyMap = {};
   sortedDependencies: Array<string> = [];
@@ -78,7 +73,7 @@ export default class DataTreeEvaluator {
     this.widgetConfigMap = widgetConfigMap;
   }
 
-  createFirstTree(unEvalTree: DataTree): EvaluateDataTreeResponse {
+  createFirstTree(unEvalTree: DataTree): DataTree {
     const totalStart = performance.now();
     // Create dependency map
     const createDependencyStart = performance.now();
@@ -121,27 +116,13 @@ export default class DataTreeEvaluator {
       },
     };
     this.logs.push({ timeTakenForFirstTree });
-    return {
-      dataTree: this.evalTree,
-      evaluationOrder: this.sortedDependencies,
-      removedPaths: [],
-    };
-  }
-
-  isDynamicLeaf(unEvalTree: DataTree, propertyPath: string) {
-    const [entityName, ...propPathEls] = _.toPath(propertyPath);
-    // Framework feature: Top level items are never leaves
-    if (entityName === propertyPath) return false;
-    // Ignore if this was a delete op
-    if (!(entityName in unEvalTree)) return false;
-
-    const entity = unEvalTree[entityName];
-    if (!isAction(entity) && !isWidget(entity)) return false;
-    const relativePropertyPath = convertPathToString(propPathEls);
-    return relativePropertyPath in entity.bindingPaths;
+    return this.evalTree;
   }
 
-  updateDataTree(unEvalTree: DataTree): EvaluateDataTreeResponse {
+  updateDataTree(
+    unEvalTree: DataTree,
+  ): { updates: Diff<DataTree, DataTree>[]; evaluationOrder: string[] } {
+    const oldEvalTree = _.cloneDeep(this.evalTree);
     const totalStart = performance.now();
     // Calculate diff
     const diffCheckTimeStart = performance.now();
@@ -150,9 +131,8 @@ export default class DataTreeEvaluator {
     // We want to check if no diffs are present and bail out early
     if (differences.length === 0) {
       return {
-        dataTree: this.evalTree,
+        updates: [],
         evaluationOrder: [],
-        removedPaths: [],
       };
     }
     const diffCheckTimeStop = performance.now();
@@ -185,15 +165,14 @@ export default class DataTreeEvaluator {
     const evaluationOrder = subTreeSortOrder.filter((propertyPath) => {
       // We are setting all values from our uneval tree to the old eval tree we have
       // So that the actual uneval value can be evaluated
-      if (this.isDynamicLeaf(unEvalTree, propertyPath)) {
+      if (isDynamicLeaf(unEvalTree, propertyPath)) {
         const unEvalPropValue = _.get(unEvalTree, propertyPath);
         _.set(this.evalTree, propertyPath, unEvalPropValue);
         return true;
       }
       return false;
     });
     this.logs.push({
-      evaluationOrder,
       sortedDependencies: this.sortedDependencies,
       inverse: this.inverseDependencyMap,
       updatedDependencyMap: this.dependencyMap,
@@ -203,16 +182,24 @@ export default class DataTreeEvaluator {
     removedPaths.forEach((removedPath) => {
       _.unset(this.evalTree, removedPath);
     });
-    this.evalTree = this.evaluateTree(this.evalTree, evaluationOrder);
+    const newEvalTree = this.evaluateTree(this.evalTree, evaluationOrder);
     const evalStop = performance.now();
 
+    const evalTreeDiffsStart = performance.now();
+
+    const evaluationChanges = diff(oldEvalTree, newEvalTree);
+
+    const evalTreeDiffsStop = performance.now();
+
     const totalEnd = performance.now();
     // TODO: For some reason we are passing some reference which are getting mutated.
     // Need to check why big api responses are getting split between two eval runs
     this.oldUnEvalTree = unEvalTree;
+    this.evalTree = newEvalTree;
     const timeTakenForSubTreeEval = {
       total: (totalEnd - totalStart).toFixed(2),
       findDifferences: (diffCheckTimeStop - diffCheckTimeStart).toFixed(2),
+      findEvalDifferences: (evalTreeDiffsStop - evalTreeDiffsStart).toFixed(2),
       updateDependencies: (
         updateDependenciesStop - updateDependenciesStart
       ).toFixed(2),
@@ -223,9 +210,8 @@ export default class DataTreeEvaluator {
     };
     this.logs.push({ timeTakenForSubTreeEval });
     return {
-      dataTree: this.evalTree,
+      updates: evaluationChanges || [],
       evaluationOrder,
-      removedPaths,
     };
   }
 
@@ -396,7 +382,6 @@ export default class DataTreeEvaluator {
     try {
       return sortedDependencies.reduce(
         (currentTree: DataTree, fullPropertyPath: string) => {
-          this.logs.push(`evaluating ${fullPropertyPath}`);
           const { entityName, propertyPath } = getEntityNameAndPropertyPath(
             fullPropertyPath,
           );
@@ -775,10 +760,7 @@ export default class DataTreeEvaluator {
             // If a new entity/property was added, add all the internal bindings for this entity to the global dependency map
             if (
               (isWidget(entity) || isAction(entity)) &&
-              !this.isDynamicLeaf(
-                unEvalDataTree,
-                dataTreeDiff.payload.propertyPath,
-              )
+              !isDynamicLeaf(unEvalDataTree, dataTreeDiff.payload.propertyPath)
             ) {
               const entityDependencyMap: DependencyMap = this.listEntityDependencies(
                 entity,
@@ -997,7 +979,7 @@ export default class DataTreeEvaluator {
 
       changePaths.add(convertPathToString(d.path));
       // If this is a property path change, simply add for evaluation and move on
-      if (!this.isDynamicLeaf(unEvalTree, convertPathToString(d.path))) {
+      if (!isDynamicLeaf(unEvalTree, convertPathToString(d.path))) {
         // A parent level property has been added or deleted
         /**
          * We want to add all pre-existing dynamic and static bindings in dynamic paths of this entity to get evaluated and validated.

app/client/src/workers/evaluation.test.ts

@@ -207,6 +207,7 @@ const WIDGET_CONFIG_MAP: WidgetTypeConfigMap = {
 };
 
 const BASE_WIDGET: DataTreeWidget = {
+  logBlackList: {},
   widgetId: "randomID",
   widgetName: "randomWidgetName",
   bottomRow: 0,
@@ -227,6 +228,7 @@ const BASE_WIDGET: DataTreeWidget = {
 };
 
 const BASE_ACTION: DataTreeAction = {
+  logBlackList: {},
   actionId: "randomId",
   name: "randomActionName",
   config: {
@@ -384,7 +386,8 @@ describe("DataTreeEvaluator", () => {
         text: "Hey there",
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     expect(dataTree).toHaveProperty("Text2.text", "Hey there");
     expect(dataTree).toHaveProperty("Text3.text", "Hey there");
   });
@@ -397,7 +400,8 @@ describe("DataTreeEvaluator", () => {
         text: "Label 3",
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     const updatedDependencyMap = evaluator.dependencyMap;
     expect(dataTree).toHaveProperty("Text2.text", "Label");
     expect(dataTree).toHaveProperty("Text3.text", "Label 3");
@@ -445,7 +449,8 @@ describe("DataTreeEvaluator", () => {
       },
     };
 
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     expect(dataTree).toHaveProperty("Input1.text", "Default value");
   });
 
@@ -481,7 +486,8 @@ describe("DataTreeEvaluator", () => {
         },
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     expect(dataTree).toHaveProperty("Dropdown2.options.0.label", "newValue");
   });
 
@@ -501,7 +507,8 @@ describe("DataTreeEvaluator", () => {
         ],
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     const updatedDependencyMap = evaluator.dependencyMap;
     expect(dataTree).toHaveProperty("Table1.tableData", [
       {
@@ -565,7 +572,8 @@ describe("DataTreeEvaluator", () => {
         ],
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedUnEvalTree);
+    evaluator.updateDataTree(updatedUnEvalTree);
+    const dataTree = evaluator.evalTree;
     const updatedDependencyMap = evaluator.dependencyMap;
     expect(dataTree).toHaveProperty("Table1.tableData", [
       {
@@ -654,7 +662,8 @@ describe("DataTreeEvaluator", () => {
         },
       },
     };
-    const { dataTree } = evaluator.updateDataTree(updatedTree2);
+    evaluator.updateDataTree(updatedTree2);
+    const dataTree = evaluator.evalTree;
     expect(evaluator.dependencyMap["Api2.config.body"]).toStrictEqual([
       "Text1.text",
       "Api2.config.pluginSpecifiedTemplates[0].value",
@@ -680,8 +689,8 @@ describe("DataTreeEvaluator", () => {
         },
       },
     };
-    const evaluatedDataTreeObject = evaluator.updateDataTree(updatedTree3);
-    const dataTree3 = evaluatedDataTreeObject.dataTree;
+    evaluator.updateDataTree(updatedTree3);
+    const dataTree3 = evaluator.evalTree;
     expect(evaluator.dependencyMap["Api2.config.body"]).toStrictEqual([
       "Text1.text",
       "Api2.config.pluginSpecifiedTemplates[0].value",