Fix: replace sidecar with node (#7470)

* fix: replace sidecar with node

* fix: use erasStakersOverview to fetch tvl for each validator

* fix: remove identities code for polkadot

* get rid of sidecar api

* fix unit tests

* fix: polkadot integration test

* refactoring

.changeset/lemon-jeans-tie.md

@@ -0,0 +1,5 @@
+---
+"@ledgerhq/coin-polkadot": patch
+---
+
+get rid of sidecar fork api and fix validator list for polkadot

libs/coin-modules/coin-polkadot/src/api/index.integ.test.ts

@@ -52,7 +52,7 @@ describe("Polkadot Api", () => {
           operation.senders.includes(address) || operation.recipients.includes(address);
         expect(isSenderOrReceipt).toBeTruthy();
       });
-    });
+    }, 20000);
   });
 
   describe("lastBlock", () => {

libs/coin-modules/coin-polkadot/src/network/node/node.integ.test.ts

@@ -0,0 +1,22 @@
+import { ApiPromise, HttpProvider } from "@polkadot/api";
+import { fetchValidators } from "./validators";
+import getApiPromise from "./apiPromise";
+
+jest.mock("./apiPromise");
+
+describe("fetchValidators", () => {
+  let provider: HttpProvider;
+  beforeAll(async () => {
+    provider = new HttpProvider("https://polkadot-rpc.publicnode.com");
+    const api = await ApiPromise.create({ provider, noInitWarn: true });
+    (getApiPromise as jest.Mock).mockResolvedValue(api);
+  });
+
+  it("should not exceed 40 RPC API calls to fetch all validators", async () => {
+    const result = await fetchValidators();
+    expect(result.length).toBeGreaterThan(300);
+    const requestCount = provider.stats.total.requests;
+    expect(requestCount).toBeGreaterThan(0); // should have made at least one request
+    expect(requestCount).toBeLessThanOrEqual(40); // should not exceed 50 requests
+  }, 10000);
+});

libs/coin-modules/coin-polkadot/src/network/node/node.unit.test.ts

@@ -22,7 +22,9 @@ describe("fetchNominations", () => {
           nominators: {
             at: jest.fn(),
           },
-          erasStakersOverview: jest.fn(),
+          erasStakersOverview: {
+            multi: jest.fn(),
+          },
           erasStakersPaged: jest.fn(),
         },
       },
@@ -58,7 +60,7 @@ describe("fetchNominations", () => {
       unwrap: () => ({ targets: mockTargets, submittedIn: "submittedIn" }),
     };
     const mockStashes = [{ toString: () => "stash1" }, { toString: () => "stash2" }];
-    const mockExposure = { isNone: false, toJSON: () => ({ pageCount: 1 }) };
+    const mockExposure = [{ isNone: false, toJSON: () => ({ pageCount: 1 }) }];
     const mockNominators = {
       unwrap: () => ({
         others: [{ who: { toString: () => mockAddress }, value: { toString: () => "100" } }],
@@ -72,7 +74,7 @@ describe("fetchNominations", () => {
     });
     api.query.staking.nominators.at.mockResolvedValue(mockNominationsOpt);
     api.derive.staking.stashes.mockResolvedValue(mockStashes);
-    api.query.staking.erasStakersOverview.mockResolvedValue(mockExposure);
+    api.query.staking.erasStakersOverview.multi.mockResolvedValue(mockExposure);
     api.query.staking.erasStakersPaged.mockResolvedValue(mockNominators);
 
     const result = await fetchNominations(mockAddress);
@@ -98,7 +100,7 @@ describe("fetchNominations", () => {
     });
     api.query.staking.nominators.at.mockResolvedValue(mockNominationsOpt);
     api.derive.staking.stashes.mockResolvedValue(mockStashes);
-    api.query.staking.erasStakersOverview.mockResolvedValue({ isNone: true });
+    api.query.staking.erasStakersOverview.multi.mockResolvedValue([{ isNone: true }]);
 
     const result = await fetchNominations(mockAddress);
 

libs/coin-modules/coin-polkadot/src/network/node/nominations.ts

@@ -29,20 +29,25 @@ export const fetchNominations = async (address: string): Promise<SidecarNominati
 
   const allStashes = stashes.map(stash => stash.toString());
   const returnTargets = [];
-  for (const target of targets) {
+  const targetIds = targets.map(t => t.toString());
+
+  const queries = targetIds.map(targetId => [activeEra, targetId]);
+  const exposures = await api.query.staking.erasStakersOverview.multi(queries);
+  let targetIndex = 0;
+  for (const id of targetIds) {
     let status: "active" | "inactive" | "waiting" | null = null;
     let value = "0";
-    const exposure = await api.query.staking.erasStakersOverview(activeEra, target.toString());
+    const exposure = exposures[targetIndex];
+    targetIndex++;
     if (exposure.isNone) {
-      if (allStashes.includes(target.toString())) {
+      if (allStashes.includes(id)) {
         status = "waiting";
       }
     } else {
       const pageCount: number = (exposure.toJSON() as any).pageCount ?? 0;
       for (let i = 0; i < pageCount; i++) {
-        const nominators = (
-          await api.query.staking.erasStakersPaged(activeEra, target.toString(), i)
-        ).unwrap().others;
+        const nominators = (await api.query.staking.erasStakersPaged(activeEra, id, i)).unwrap()
+          .others;
         if (!status && nominators.length > 0) {
           status = "inactive";
         }
@@ -55,7 +60,7 @@ export const fetchNominations = async (address: string): Promise<SidecarNominati
       }
     }
     returnTargets.push({
-      address: target.toString(),
+      address: id,
       value,
       status,
     });

libs/coin-modules/coin-polkadot/src/network/node/validators.ts

@@ -1,21 +1,13 @@
-import { DeriveStakingQuery } from "@polkadot/api-derive/types";
-import { EraIndex, RewardPoint } from "@polkadot/types/interfaces";
+import { EraIndex } from "@polkadot/types/interfaces";
 import {
   SidecarValidators,
   SidecarValidatorsParamAddresses,
   SidecarValidatorsParamStatus,
   IValidator,
-  IIdentity,
 } from "../types";
 import getApiPromise from "./apiPromise";
-import { multiIdentities } from "./identities";
 import { ApiPromise } from "@polkadot/api";
-
-const QUERY_OPTS = {
-  withExposure: true,
-  withLedger: true,
-  withPrefs: true,
-};
+import type { SpStakingPagedExposureMetadata } from "@polkadot/types/lookup";
 
 /**
  * Fetch a list of validators with some info and indentity.
@@ -34,17 +26,14 @@ export const fetchValidators = async (
 ): Promise<SidecarValidators> => {
   const api = await getApiPromise();
 
-  const [activeOpt, allStashes, elected, nextElected] = await Promise.all([
+  const [activeOpt, allStashes, elected] = await Promise.all([
     api.query.staking.activeEra(),
     api.derive.staking.stashes(),
     api.query.session.validators(),
-    api.derive.staking.nextElected(),
   ]);
 
   const { index: activeEra } = activeOpt.unwrapOrDefault();
 
-  const rewards = await fetchRewardsPoints(api)(activeEra);
-
   let selected: string[] = [];
   const allIds = allStashes.map(s => s.toString());
   const electedIds = elected.map(s => s.toString());
@@ -58,9 +47,6 @@ export const fetchValidators = async (
       selected = waitingIds;
       break;
     }
-    case "nextElected":
-      selected = nextElected.map(s => s.toString());
-      break;
     case "all": {
       const waitingIds = allIds.filter(v => !electedIds.includes(v));
       // Keep order of elected validators
@@ -77,54 +63,69 @@ export const fetchValidators = async (
       .filter(address => selected.includes(address.toString()));
   }
 
-  const [validators, identities] = await Promise.all([
-    api.derive.staking.queryMulti(selected, QUERY_OPTS),
-    multiIdentities(selected),
-  ]);
-
-  return validators.map((validator, index) =>
-    formatValidator(api)(validator, identities[index], rewards, electedIds),
-  );
-};
-
-const fetchRewardsPoints =
-  (api: ApiPromise) =>
-  async (activeEra: EraIndex): Promise<Map<string, RewardPoint>> => {
-    const { individual } = await api.query.staking.erasRewardPoints(activeEra);
-
-    // recast BTreeMap<AccountId,RewardPoint> to Map<String, RewardPoint> because strict equality does not work
-    const rewards = new Map<string, RewardPoint>(
-      [...individual.entries()].map(([k, v]) => [k.toString(), v]),
+  const validatorsCommissions = await getValidatorCommissions(api, selected);
+  const validatorsExposure = await getValidatorsExposure(api, activeEra, selected);
+  const maxNominatorRewardedPerValidator = api.consts?.staking?.maxExposurePageSize.toNumber();
+  return selected.map(validator => {
+    const commission = validatorsCommissions[validator] || "";
+    const exposure = validatorsExposure[validator] || null;
+    return formatValidator(
+      validator,
+      electedIds,
+      commission,
+      exposure,
+      maxNominatorRewardedPerValidator,
     );
+  });
+};
 
-    return rewards;
+const formatValidator = (
+  validator: string,
+  electedIds: string[],
+  commission: string,
+  exposure: SpStakingPagedExposureMetadata | null,
+  maxNominatorRewardedPerValidator: number,
+): IValidator => {
+  const nominatorsCount = exposure?.nominatorCount?.toNumber() ?? 0;
+  return {
+    accountId: validator,
+    identity: null,
+    own: exposure?.own?.toString() ?? "0",
+    total: exposure?.total?.toString() ?? "0",
+    nominatorsCount,
+    commission,
+    rewardsPoints: null,
+    isElected: electedIds.includes(validator),
+    isOversubscribed: nominatorsCount > maxNominatorRewardedPerValidator,
   };
+};
 
-const formatValidator =
-  (api: ApiPromise) =>
-  (
-    validator: DeriveStakingQuery,
-    identity: IIdentity,
-    rewards: Map<string, RewardPoint>,
-    electedIds: string[],
-  ): IValidator => {
-    const validatorId = validator.accountId.toString();
-    const maxNominatorRewardedPerValidator = api.consts?.staking?.maxNominatorRewardedPerValidator;
+async function getValidatorCommissions(
+  api: ApiPromise,
+  electedIds: string[],
+): Promise<{ [key: string]: string }> {
+  const validatorInfos = await api.query.staking.validators.multi(electedIds);
+  const commissions: { [key: string]: string } = {};
+  electedIds.forEach((address, index) => {
+    const validatorInfo = validatorInfos[index];
+    const commission = validatorInfo.commission.toString();
+    commissions[address] = commission;
+  });
+  return commissions;
+}
 
-    return {
-      accountId: validator.accountId.toString(),
-      identity,
-      // own: validator.exposure.own.toString(),
-      own: validator.exposureEraStakers.own.toString(),
-      // total: validator.exposure.total.toString(),
-      total: validator.exposureEraStakers.total.toString(),
-      // nominatorsCount: validator.exposure.others.length,
-      nominatorsCount: validator.exposureEraStakers.others.length,
-      commission: validator.validatorPrefs.commission.toString(),
-      rewardsPoints: rewards.get(validatorId)?.toString() || null,
-      isElected: electedIds.includes(validatorId),
-      isOversubscribed: maxNominatorRewardedPerValidator
-        ? validator.exposureEraStakers.others.length > Number(maxNominatorRewardedPerValidator)
-        : false,
-    };
-  };
+async function getValidatorsExposure(
+  api: ApiPromise,
+  activeEra: EraIndex,
+  electedIds: string[],
+): Promise<{ [key: string]: SpStakingPagedExposureMetadata }> {
+  const queries = electedIds.map(id => [activeEra, id]);
+  const exposures = await api.query.staking.erasStakersOverview.multi(queries);
+  const exposureMap: { [key: string]: SpStakingPagedExposureMetadata } = {};
+  exposures.forEach((exposure, index) => {
+    if (exposure.isSome) {
+      exposureMap[electedIds[index]] = exposure.unwrap();
+    }
+  });
+  return exposureMap;
+}