feat: make PML compatible with OE by staging in-memory CLOB side effe…

…cts (#2447)

(cherry picked from commit 161c34d)

# Conflicts:
#	protocol/x/listing/keeper/listing.go

indexer/packages/v4-protos/src/codegen/dydxprotocol/clob/finalize_block.ts

@@ -0,0 +1,66 @@
+import { ClobPair, ClobPairSDKType } from "./clob_pair";
+import * as _m0 from "protobufjs/minimal";
+import { DeepPartial } from "../../helpers";
+/**
+ * ClobStagedFinalizeBlockEvent defines a CLOB event staged during
+ * FinalizeBlock.
+ */
+
+export interface ClobStagedFinalizeBlockEvent {
+  /** create_clob_pair indicates a new CLOB pair creation. */
+  createClobPair?: ClobPair;
+}
+/**
+ * ClobStagedFinalizeBlockEvent defines a CLOB event staged during
+ * FinalizeBlock.
+ */
+
+export interface ClobStagedFinalizeBlockEventSDKType {
+  /** create_clob_pair indicates a new CLOB pair creation. */
+  create_clob_pair?: ClobPairSDKType;
+}
+
+function createBaseClobStagedFinalizeBlockEvent(): ClobStagedFinalizeBlockEvent {
+  return {
+    createClobPair: undefined
+  };
+}
+
+export const ClobStagedFinalizeBlockEvent = {
+  encode(message: ClobStagedFinalizeBlockEvent, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.createClobPair !== undefined) {
+      ClobPair.encode(message.createClobPair, writer.uint32(10).fork()).ldelim();
+    }
+
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): ClobStagedFinalizeBlockEvent {
+    const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseClobStagedFinalizeBlockEvent();
+
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+
+      switch (tag >>> 3) {
+        case 1:
+          message.createClobPair = ClobPair.decode(reader, reader.uint32());
+          break;
+
+        default:
+          reader.skipType(tag & 7);
+          break;
+      }
+    }
+
+    return message;
+  },
+
+  fromPartial(object: DeepPartial<ClobStagedFinalizeBlockEvent>): ClobStagedFinalizeBlockEvent {
+    const message = createBaseClobStagedFinalizeBlockEvent();
+    message.createClobPair = object.createClobPair !== undefined && object.createClobPair !== null ? ClobPair.fromPartial(object.createClobPair) : undefined;
+    return message;
+  }
+
+};

indexer/packages/v4-protos/src/codegen/gogoproto/bundle.ts

@@ -1,3 +1,3 @@
-import * as _128 from "./gogo";
-export const gogoproto = { ..._128
+import * as _129 from "./gogo";
+export const gogoproto = { ..._129
 };

indexer/packages/v4-protos/src/codegen/google/bundle.ts

@@ -1,16 +1,16 @@
-import * as _129 from "./api/annotations";
-import * as _130 from "./api/http";
-import * as _131 from "./protobuf/descriptor";
-import * as _132 from "./protobuf/duration";
-import * as _133 from "./protobuf/timestamp";
-import * as _134 from "./protobuf/any";
+import * as _130 from "./api/annotations";
+import * as _131 from "./api/http";
+import * as _132 from "./protobuf/descriptor";
+import * as _133 from "./protobuf/duration";
+import * as _134 from "./protobuf/timestamp";
+import * as _135 from "./protobuf/any";
 export namespace google {
-  export const api = { ..._129,
-    ..._130
+  export const api = { ..._130,
+    ..._131
   };
-  export const protobuf = { ..._131,
-    ..._132,
+  export const protobuf = { ..._132,
     ..._133,
-    ..._134
+    ..._134,
+    ..._135
   };
 }

proto/dydxprotocol/clob/finalize_block.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+package dydxprotocol.clob;
+
+import "dydxprotocol/clob/clob_pair.proto";
+
+option go_package = "github.com/dydxprotocol/v4-chain/protocol/x/clob/types";
+
+// ClobStagedFinalizeBlockEvent defines a CLOB event staged during
+// FinalizeBlock.
+message ClobStagedFinalizeBlockEvent {
+  // event is the staged event.
+  oneof event {
+    // create_clob_pair indicates a new CLOB pair creation.
+    ClobPair create_clob_pair = 1;
+  }
+}

protocol/testutil/keeper/clob.go

@@ -282,7 +282,7 @@ func CreateTestClobPairs(
 	clobPairs []types.ClobPair,
 ) {
 	for _, clobPair := range clobPairs {
-		_, err := clobKeeper.CreatePerpetualClobPair(
+		_, err := clobKeeper.CreatePerpetualClobPairAndMemStructs(
 			ctx,
 			clobPair.Id,
 			clobPair.MustGetPerpetualId(),
@@ -341,7 +341,7 @@ func CreateNClobPair(
 			),
 		).Return()
 
-		_, err := keeper.CreatePerpetualClobPair(
+		_, err := keeper.CreatePerpetualClobPairAndMemStructs(
 			ctx,
 			items[i].Id,
 			clobtest.MustPerpetualId(items[i]),

protocol/x/clob/abci.go

@@ -51,6 +51,12 @@ func Precommit(
 	ctx sdk.Context,
 	keeper keeper.Keeper,
 ) {
+	// Process all staged finalize block events, and apply necessary side effects
+	// (e.g. MemClob orderbook creation) that could not be done during FinalizeBlock.
+	// Note: this must be done in `Precommit` which is prior to `PrepareCheckState`, when
+	// MemClob could access the new orderbooks.
+	keeper.ProcessStagedFinalizeBlockEvents(ctx)
+
 	if streamingManager := keeper.GetFullNodeStreamingManager(); !streamingManager.Enabled() {
 		return
 	}

protocol/x/clob/abci_test.go

@@ -530,7 +530,7 @@ func TestEndBlocker_Success(t *testing.T) {
 					),
 				),
 			).Once().Return()
-			_, err = ks.ClobKeeper.CreatePerpetualClobPair(
+			_, err = ks.ClobKeeper.CreatePerpetualClobPairAndMemStructs(
 				ctx,
 				constants.ClobPair_Btc.Id,
 				clobtest.MustPerpetualId(constants.ClobPair_Btc),
@@ -563,7 +563,7 @@ func TestEndBlocker_Success(t *testing.T) {
 					),
 				),
 			).Once().Return()
-			_, err = ks.ClobKeeper.CreatePerpetualClobPair(
+			_, err = ks.ClobKeeper.CreatePerpetualClobPairAndMemStructs(
 				ctx,
 				constants.ClobPair_Eth.Id,
 				clobtest.MustPerpetualId(constants.ClobPair_Eth),
@@ -1170,7 +1170,7 @@ func TestPrepareCheckState(t *testing.T) {
 
 			// Create all CLOBs.
 			for _, clobPair := range tc.clobs {
-				_, err = ks.ClobKeeper.CreatePerpetualClobPair(
+				_, err = ks.ClobKeeper.CreatePerpetualClobPairAndMemStructs(
 					ctx,
 					clobPair.Id,
 					clobtest.MustPerpetualId(clobPair),

protocol/x/clob/genesis.go

@@ -19,7 +19,7 @@ func InitGenesis(ctx sdk.Context, k *keeper.Keeper, genState types.GenesisState)
 		if err != nil {
 			panic(errorsmod.Wrap(types.ErrInvalidClobPairParameter, err.Error()))
 		}
-		_, err = k.CreatePerpetualClobPair(
+		_, err = k.CreatePerpetualClobPairAndMemStructs(
 			ctx,
 			elem.Id,
 			perpetualId,