diff --git a/block-explorer-indexer/jest.config.js b/block-explorer-indexer/jest.config.js index 464d232..b4472b5 100644 --- a/block-explorer-indexer/jest.config.js +++ b/block-explorer-indexer/jest.config.js @@ -1,5 +1,12 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - // You can add other configurations here as needed + moduleNameMapper: { + '^@/(.*)$': '/src/libs/$1', + }, }; + +process.env = Object.assign(process.env, { + DEBUG: process.env.DEBUG || 'rootscan:*', + __JEST__: process.env.__JEST__ || true, +}); diff --git a/block-explorer-indexer/package.json b/block-explorer-indexer/package.json index 3c47b58..9082a8a 100644 --- a/block-explorer-indexer/package.json +++ b/block-explorer-indexer/package.json @@ -49,6 +49,7 @@ "express-session": "^1.18.0", "helmet": "^7.1.0", "ioredis": "^5.3.2", + "is-url": "^1.2.4", "lodash": "^4.17.21", "lru-cache": "^11.0.0", "moment": "^2.30.1", diff --git a/block-explorer-indexer/pnpm-lock.yaml b/block-explorer-indexer/pnpm-lock.yaml index 0dd003d..2afab9d 100644 --- a/block-explorer-indexer/pnpm-lock.yaml +++ b/block-explorer-indexer/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: ioredis: specifier: ^5.3.2 version: 5.3.2 + is-url: + specifier: ^1.2.4 + version: 1.2.4 lodash: specifier: ^4.17.21 version: 4.17.21 @@ -1792,6 +1795,9 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} + is-url@1.2.4: + resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -4911,6 +4917,8 @@ snapshots: is-stream@2.0.1: {} + is-url@1.2.4: {} + isexe@2.0.0: {} isows@1.0.4(ws@8.17.1): diff --git a/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.test.ts b/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.test.ts deleted file mode 100644 index 10c4f26..0000000 --- a/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -function sum(a: number, b: number): number { - return a + b; -} -test('adds 1 + 2 to equal 3', () => { - expect(sum(1, 2)).toBe(3); -}); diff --git a/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.ts b/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.ts index 339c708..988bdef 100644 --- a/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.ts +++ b/block-explorer-indexer/src/libs/nft-indexer/nft-token-data.ts @@ -1,16 +1,12 @@ import ABIs from '@/constants/abi'; import { ethereumClient } from '@/rpc'; import { INftOwner, TNftTokenType } from '@/types'; +import { prepareTokenMetadataUrl } from '@/utils/url-utils'; import { noop } from 'lodash'; import pLimit from 'p-limit'; import { Address, PublicClient, getAddress } from 'viem'; const limiter = pLimit(100); -const skipDomains = ['example.com', 'localhost']; -const skipDomainsRegex = new RegExp(skipDomains.map((domain) => `(${domain})`).join('|'), 'i'); -const containsIpWithPortRegex = /(https?:\/\/)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(:\d+)?\/?[\w\\/.-]*\b/; -const isUrlCorrectRegex = - /(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})/; interface TokenMetadata { image: string; @@ -97,35 +93,17 @@ export class NftTokenData { }) .catch(noop); } - return prepareUrl(data); + console.log(data); + return prepareTokenMetadataUrl(data); } async fillNftsMetadata(nfts: INftOwner[]) { const metadatas = await this.getBulkTokenMetadata(nfts); nfts.forEach((nft, index) => { nft.attributes = metadatas[index]?.attributes; - nft.image = prepareUrl(metadatas[index]?.image); - nft.animation_url = prepareUrl(metadatas[index]?.animation_url); + nft.image = prepareTokenMetadataUrl(metadatas[index]?.image); + nft.animation_url = prepareTokenMetadataUrl(metadatas[index]?.animation_url); nft._metadataProcessed = nft.image ? true : undefined; }); } } - -function prepareUrl(link: string | undefined): string | undefined { - if (!link) { - return; - } - link = link.trim(); - - if (link.toLowerCase().startsWith('ipfs://')) { - // See https://docs.ipfs.tech/quickstart/retrieve/#fetching-the-cid-with-an-ipfs-gateway - return link.replace(/^ipfs:\/\//i, 'https://ipfs.io/ipfs/'); - } - if (!link.includes('://')) { - return `https://${link}`; - } - if (!isUrlCorrectRegex.test(link) || containsIpWithPortRegex.test(link) || skipDomainsRegex.test(link)) { - return; - } - return link; -} diff --git a/block-explorer-indexer/src/libs/utils/url-utils.test.ts b/block-explorer-indexer/src/libs/utils/url-utils.test.ts new file mode 100644 index 0000000..c1434bd --- /dev/null +++ b/block-explorer-indexer/src/libs/utils/url-utils.test.ts @@ -0,0 +1,49 @@ +import { prepareTokenMetadataUrl } from './url-utils'; + +describe('UrlUtils', () => { + test('prepareTokenMetadataUrl: incorrect urls should skip', () => { + const badUrls = [ + 'https://1', + 'https://42', + 'https://1024', + undefined, + null, + '', + 'null', + 'null0', + 'test123', + 'https://test123', + '1', + 'test1', + 'true', + 'example.com', + 'localhost', + 'https://localhost/x', + '0.0.0.0', + '0.0.0', + '0.0', + '0', + 'https://192.168.1.1:30333', + ]; + for (const url of badUrls) { + expect(prepareTokenMetadataUrl(url as string)).toBe(undefined); + } + }); + + test('prepareTokenMetadataUrl: correct urls', () => { + const correctUrls = [ + 'https://rns-metadata.fly.dev/mainnet/0x44640D662A423d738D5ebF8B51E57AfC0f2cf4Df/34568736216980230572277344339278661393462213082217059465217671489155940463826', + 'https://www.projecttempus.xyz/api/51300/token/77', + ]; + for (const url of correctUrls) { + expect(prepareTokenMetadataUrl(url as string)).toBe(url); + } + expect(prepareTokenMetadataUrl('projecttempus.xyz/api/51300/token/77')).toBe( + 'https://projecttempus.xyz/api/51300/token/77', + ); + }); + + test('prepareTokenMetadataUrl: ipfs urls', () => { + expect(prepareTokenMetadataUrl('ipfs://test')).toBe('https://ipfs.io/ipfs/test'); + }); +}); diff --git a/block-explorer-indexer/src/libs/utils/url-utils.ts b/block-explorer-indexer/src/libs/utils/url-utils.ts new file mode 100644 index 0000000..452e3af --- /dev/null +++ b/block-explorer-indexer/src/libs/utils/url-utils.ts @@ -0,0 +1,24 @@ +import isUrl from 'is-url'; + +const skipDomains = ['example.com', 'localhost', '0.0.0']; +const skipDomainsRegex = new RegExp(skipDomains.map((domain) => `(${domain})`).join('|'), 'i'); +const containsIpWithPortRegex = /(https?:\/\/)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(:\d+)?\/?[\w\\/.-]*\b/; + +export function prepareTokenMetadataUrl(link: string | undefined): string | undefined { + if (!link) { + return; + } + link = link.trim(); + + if (link.toLowerCase().startsWith('ipfs://')) { + // See https://docs.ipfs.tech/quickstart/retrieve/#fetching-the-cid-with-an-ipfs-gateway + link = link.replace(/^ipfs:\/\//i, 'https://ipfs.io/ipfs/'); + } + if (!link.includes('://')) { + link = `https://${link}`; + } + if (!isUrl(link) || containsIpWithPortRegex.test(link) || skipDomainsRegex.test(link)) { + return; + } + return link; +}