Add BDN tests: Operations.RawStringOperations and Operations.CustomOp…

…erations (#876) * Added two more BDN perf tests: Operations.CustomOperations and Operations.RawStringOperations * Added BDN Charts to the badges * Added link to BDN charts to website * Setting results array to 70 so we have plenty of space for different results (in a single BDN result set). * Added ScriptOperations and conrresponding expected values in Config file --------- Co-authored-by: Badrish Chandramouli <[email protected]>
microsoft · Dec 13, 2024 · 3d3e8d6 · 3d3e8d6
1 parent f1ce0da
commit 3d3e8d6
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 8 deletions.
diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml
@@ -42,7 +42,7 @@ jobs:
         os: [ ubuntu-latest, windows-latest ]
         framework: [ 'net8.0' ]
         configuration: [ 'Release' ]
-        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ]
+        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Operations.HashObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts','Operations.RawStringOperations','Operations.ScriptOperations' ]
     steps:
       - name: Check out code
         uses: actions/checkout@v4

diff --git a/README.md b/README.md
@@ -3,6 +3,7 @@
 [![](https://img.shields.io/github/release/microsoft/garnet.svg?label=latest%20release&color=007edf)](https://github.com/microsoft/garnet/releases/latest)
 [![](https://img.shields.io/nuget/dt/microsoft.garnet.svg?label=nuget%20library&color=007edf&logo=nuget)](https://www.nuget.org/packages/microsoft.garnet)
 [![](https://img.shields.io/nuget/dt/garnet-server.svg?label=dotnet%20tool&color=007edf&logo=nuget)](https://www.nuget.org/packages/garnet-server)
+[![BDN Charts](https://img.shields.io/badge/BDN%20Charts-8A2BE2)](https://microsoft.github.io/garnet/charts/)
 [![Discord Shield](https://discordapp.com/api/guilds/1213937452272582676/widget.png?style=shield)](https://aka.ms/garnet-discord)
 
 Garnet is a new remote cache-store from Microsoft Research, that offers several unique benefits:

diff --git a/test/BDNPerfTests/BDN_Benchmark_Config.json b/test/BDNPerfTests/BDN_Benchmark_Config.json
@@ -1,6 +1,7 @@
 {
   "_comment": "NOTE: If adding a new BDN perf test to this file, you will need to add the name to the test: line in ci-bdnbenchmark.yml (~line 45). This ensures all the tests are not ran sequentially when ran as github action.",
   "_comment2": "NOTE: The BDN Perf name in the ci-bdnbenchmark.yml file test: line should NOT have the BDN.benchmark prefix and the .* ending. It needs to be this way for the reporting part of the BDN Charts.",
+  "_comment3": "NOTE: The expected values are the Allocated results from BDN tests.",
 
   "BDN.benchmark.Operations.BasicOperations.*": {
     "expected_InlinePing_ACL": 0,
@@ -19,21 +20,63 @@
     "expected_SAddRem_None": 12.5
   },
   "BDN.benchmark.Operations.HashObjectOperations.*": {
-    "expected_HSetDel_ACL": 42.19,
-    "expected_HSetDel_AOF": 42.19,
-    "expected_HSetDel_None": 42.19
+    "expected_HSetDel_ACL": 16000,
+    "expected_HExists_ACL": 0,
+    "expected_HGet_ACL": 0,
+    "expected_HGetAll_ACL": 0,
+    "expected_HIncrby_ACL": 0,
+    "expected_HIncrbyFloat_ACL": 0,
+    "expected_HKeys_ACL": 0,
+    "expected_HLen_ACL": 0,
+    "expected_HMGet_ACL": 0,
+    "expected_HMSet_ACL": 0,
+    "expected_HRandField_ACL": 0,
+    "expected_HScan_ACL": 776,
+    "expected_HSetNx_ACL": 0,
+    "expected_HStrLen_ACL": 0,
+    "expected_HVals_ACL": 0,
+    "expected_HSetDel_AOF": 16000,
+    "expected_HExists_AOF": 6400,
+    "expected_HGet_AOF": 6400,
+    "expected_HGetAll_AOF": 3200,
+    "expected_HIncrby_AOF": 14400,
+    "expected_HIncrbyFloat_AOF": 14800,
+    "expected_HKeys_AOF": 3200,
+    "expected_HLen_AOF": 3200,
+    "expected_HMGet_AOF": 9600,
+    "expected_HMSet_AOF": 16000,
+    "expected_HRandField_AOF": 8800,
+    "expected_HScan_AOF": 776,
+    "expected_HSetNx_AOF": 9600,
+    "expected_HStrLen_AOF": 6400,
+    "expected_HVals_AOF": 3200,
+    "expected_HSetDel_None": 16000,
+    "expected_HExists_None": 6400,
+    "expected_HGet_None": 6400,
+    "expected_HGetAll_None": 3200,
+    "expected_HIncrby_None": 14400,
+    "expected_HIncrbyFloat_None": 14800,
+    "expected_HKeys_None": 3200,
+    "expected_HLen_None": 3200,
+    "expected_HMGet_None": 9600,
+    "expected_HMSet_None": 16000,
+    "expected_HRandField_None": 8800,
+    "expected_HScan_None": 776,
+    "expected_HSetNx_None": 9600,
+    "expected_HStrLen_None": 6400,
+    "expected_HVals_None": 3200
   },
   "BDN.benchmark.Cluster.ClusterOperations.*": {
     "expected_Get_DSV": 0,
     "expected_Set_DSV": 0,
     "expected_MGet_DSV": 0,
     "expected_MSet_DSV": 0,
-    "expected_CPBSet_DSV": 0,
+    "expected_CTXNSET_DSV": 0,
     "expected_Get_None": 0,
     "expected_Set_None": 0,
     "expected_MGet_None": 0,
     "expected_MSet_None": 0,
-    "expected_CPBSet_None": 0
+    "expected_CTXNSET_None": 0
   },
   "BDN.benchmark.Cluster.ClusterMigrate.*": {
     "expected_Get_None": 0,
@@ -46,5 +89,68 @@
     "expected_Script2_None": 144,
     "expected_Script3_None": 240,
     "expected_Script4_None": 776
+  },
+  "BDN.benchmark.Operations.CustomOperations.*": {
+    "expected_CustomRawStringCommand_ACL": 0,
+    "expected_CustomObjectCommand_ACL": 24000,
+    "expected_CustomTransaction_ACL": 0,
+    "expected_CustomProcedure_ACL": 0,
+    "expected_CustomRawStringCommand_AOF": 0,
+    "expected_CustomObjectCommand_AOF": 24000,
+    "expected_CustomTransaction_AOF": 0,
+    "expected_CustomProcedure_AOF": 0,
+    "expected_CustomRawStringCommand_None": 0,
+    "expected_CustomObjectCommand_None": 24000,
+    "expected_CustomTransaction_None": 0,
+    "expected_CustomProcedure_None": 0
+  },
+  "BDN.benchmark.Operations.RawStringOperations.*": {
+    "expected_Set_ACL": 0,
+    "expected_SetEx_ACL": 0,
+    "expected_SetNx_ACL": 0,
+    "expected_SetXx_ACL": 0,
+    "expected_GetFound_ACL": 0,
+    "expected_GetNotFound_ACL": 0,
+    "expected_Increment_ACL": 0,
+    "expected_Decrement_ACL": 0,
+    "expected_IncrementBy_ACL": 0,
+    "expected_DecrementBy_ACL": 0,
+    "expected_Set_AOF": 0,
+    "expected_SetEx_AOF": 0,
+    "expected_SetNx_AOF": 0,
+    "expected_SetXx_AOF": 0,
+    "expected_GetFound_AOF": 0,
+    "expected_GetNotFound_AOF": 0,
+    "expected_Increment_AOF": 0,
+    "expected_Decrement_AOF": 0,
+    "expected_IncrementBy_AOF": 0,
+    "expected_DecrementBy_AOF": 0,
+    "expected_Set_None": 0,
+    "expected_SetEx_None": 0,
+    "expected_SetNx_None": 0,
+    "expected_SetXx_None": 0,
+    "expected_GetFound_None": 0,
+    "expected_GetNotFound_None": 0,
+    "expected_Increment_None": 0,
+    "expected_Decrement_None": 0,
+    "expected_IncrementBy_None": 0,
+    "expected_DecrementBy_None": 0
+  },
+  "BDN.benchmark.Operations.ScriptOperations.*": {
+    "expected_ScriptLoad_ACL": 0,
+    "expected_ScriptExistsTrue_ACL": 0,
+    "expected_ScriptExistsFalse_ACL": 0,
+    "expected_Eval_ACL": 0,
+    "expected_EvalSha_ACL": 0,
+    "expected_ScriptLoad_AOF": 16000,
+    "expected_ScriptExistsTrue_AOF": 0,
+    "expected_ScriptExistsFalse_AOF": 0,
+    "expected_Eval_AOF": 16000,
+    "expected_EvalSha_AOF": 12000,
+    "expected_ScriptLoad_None": 16000,
+    "expected_ScriptExistsTrue_None": 0,
+    "expected_ScriptExistsFalse_None": 0,
+    "expected_Eval_None": 16000,
+    "expected_EvalSha_None": 12000
   }
 }
diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1
@@ -135,7 +135,7 @@ $testProperties = $json.$currentTest
 
 # create a matrix of expected results for specific test
 $splitTextArray = New-Object 'string[]' 3
-$expectedResultsArray = New-Object 'string[,]' 20, 3
+$expectedResultsArray = New-Object 'string[,]' 70, 3
 
 [int]$currentRow = 0
 

diff --git a/website/sidebars.js b/website/sidebars.js
@@ -19,7 +19,7 @@ const sidebars = {
 	garnetDocSidebar: [
 		{type: 'category', label: 'Welcome', collapsed: false, items: ["welcome/intro", "welcome/news", "welcome/features", "welcome/releases", "welcome/compatibility", "welcome/roadmap", "welcome/faq", "welcome/about-us"]},
 		{type: 'category', label: 'Getting Started', items: ["getting-started/build", "getting-started/configuration", "getting-started/memory", "getting-started/security", "getting-started/compaction"]},
-		{type: 'category', label: 'Benchmarking', items: ["benchmarking/overview", "benchmarking/results-resp-bench", "benchmarking/resp-bench"]},
+		{type: 'category', label: 'Benchmarking', items: ["benchmarking/overview", "benchmarking/results-resp-bench", "benchmarking/resp-bench", {type: 'link', label: 'BDN Charts', href: 'https://microsoft.github.io/garnet/charts/'}]},
 		{type: 'category', label: 'Commands', items: ["commands/overview", "commands/api-compatibility", "commands/raw-string", "commands/generic-commands", "commands/analytics-commands", "commands/data-structures", "commands/server-commands", "commands/client-commands", "commands/checkpoint-commands", "commands/transactions-commands", "commands/cluster", "commands/acl-commands", "commands/scripting-commands", "commands/garnet-specific-commands"]},
 		{type: 'category', label: 'Server Extensions', items: ["extensions/overview", "extensions/raw-strings", "extensions/objects", "extensions/transactions", "extensions/procedure", "extensions/module"]},
 		{type: 'category', label: 'Cluster Mode', items: ["cluster/overview", "cluster/replication", "cluster/key-migration"]},