Skip to content

Commit

Permalink
feat: IPNI UX
Browse files Browse the repository at this point in the history
  • Loading branch information
dirkmc committed Jul 11, 2023
1 parent f69945e commit 3165c26
Show file tree
Hide file tree
Showing 17 changed files with 1,100 additions and 140 deletions.
8 changes: 7 additions & 1 deletion gql/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/filecoin-project/boost/db"
"github.com/filecoin-project/boost/fundmanager"
gqltypes "github.com/filecoin-project/boost/gql/types"
"github.com/filecoin-project/boost/indexprovider"
"github.com/filecoin-project/boost/lib/legacy"
"github.com/filecoin-project/boost/lib/mpoolmonitor"
"github.com/filecoin-project/boost/markets/storageadapter"
Expand All @@ -35,6 +36,7 @@ import (
"github.com/google/uuid"
"github.com/graph-gophers/graphql-go"
"github.com/ipfs/go-cid"
provider "github.com/ipni/index-provider"
"github.com/libp2p/go-libp2p/core/event"
"github.com/libp2p/go-libp2p/core/host"
)
Expand Down Expand Up @@ -70,12 +72,14 @@ type resolver struct {
sa retrievalmarket.SectorAccessor
piecedirectory *piecedirectory.PieceDirectory
publisher *storageadapter.DealPublisher
idxProv provider.Interface
idxProvWrapper *indexprovider.Wrapper
spApi sealingpipeline.API
fullNode v1api.FullNode
mpool *mpoolmonitor.MpoolMonitor
}

func NewResolver(ctx context.Context, cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyDeals *legacy.LegacyDealsManager, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps piecestore.PieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, publisher *storageadapter.DealPublisher, fullNode v1api.FullNode, ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *resolver {
func NewResolver(ctx context.Context, cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyDeals *legacy.LegacyDealsManager, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps piecestore.PieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, publisher *storageadapter.DealPublisher, indexProv provider.Interface, idxProvWrapper *indexprovider.Wrapper, fullNode v1api.FullNode, ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *resolver {
return &resolver{
ctx: ctx,
cfg: cfg,
Expand All @@ -97,6 +101,8 @@ func NewResolver(ctx context.Context, cfg *config.Boost, r lotus_repo.LockedRepo
piecedirectory: piecedirectory,
publisher: publisher,
spApi: spApi,
idxProv: indexProv,
idxProvWrapper: idxProvWrapper,
fullNode: fullNode,
ssm: ssm,
mpool: mpool,
Expand Down
152 changes: 152 additions & 0 deletions gql/resolver_ipni.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package gql

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/ipfs/go-cid"
"github.com/ipni/go-libipni/metadata"
"github.com/multiformats/go-multicodec"
)

type ipniResolver struct {
PeerID string
Config string
}

func (r *resolver) IpniProviderInfo() (*ipniResolver, error) {
cfg, err := json.Marshal(r.cfg.IndexProvider)
if err != nil {
return nil, err
}

return &ipniResolver{PeerID: r.h.ID().String(), Config: string(cfg)}, nil
}

type adMetadata struct {
Protocol string
Metadata string
}

type adResolver struct {
ContextID string
Entries string
PreviousEntry string
Provider string
Addresses []string
IsRemove bool
Metadata []*adMetadata
}

func (r *resolver) IpniAdvertisement(ctx context.Context, args struct{ AdCid string }) (*adResolver, error) {
adCid, err := cid.Parse(args.AdCid)
if err != nil {
return nil, fmt.Errorf("parsing ad cid %s: %w", args.AdCid, err)
}

ad, err := r.idxProv.GetAdv(ctx, adCid)
if err != nil {
return nil, err
}

var contextId string
c, err := cid.Parse(ad.ContextID)
if err == nil {
contextId = c.String()
} else {
contextId = base64.StdEncoding.EncodeToString(ad.ContextID)
}

var mds []*adMetadata
if len(ad.Metadata) > 0 {
// Metadata may contain more than one protocol, sorted by ascending order of their protocol ID.
// Therefore, decode the metadata as metadata.Metadata, then read each protocol's data.
// See: https://github.com/ipni/specs/blob/main/IPNI.md#metadata
dtm := metadata.Default.New()
if err := dtm.UnmarshalBinary(ad.Metadata); err != nil {
for _, protoId := range dtm.Protocols() {
adMd := &adMetadata{
Protocol: protoId.String(),
}
mds = append(mds, adMd)
if protoId == multicodec.TransportGraphsyncFilecoinv1 {
proto := dtm.Get(protoId)
gsmd, ok := proto.(*metadata.GraphsyncFilecoinV1)
if ok {
bz, err := json.Marshal(gsmd)
if err == nil {
adMd.Metadata = string(bz)
} else {
adMd.Metadata = err.Error()
}
}
}
}
}
}

return &adResolver{
ContextID: contextId,
Metadata: mds,
PreviousEntry: ad.PreviousID.String(),
Provider: ad.Provider,
Addresses: ad.Addresses,
IsRemove: ad.IsRm,
}, nil
}

func (r *resolver) IpniAdvertisementEntries(ctx context.Context, args struct{ AdCid string }) ([]*string, error) {
adCid, err := cid.Parse(args.AdCid)
if err != nil {
return nil, fmt.Errorf("parsing ad cid %s: %w", args.AdCid, err)
}

ad, err := r.idxProv.GetAdv(ctx, adCid)
if err != nil {
return nil, err
}

if len(ad.ContextID) == 0 {
return nil, nil
}

var entries []*string
it, err := r.idxProvWrapper.MultihashLister(ctx, "", ad.ContextID)
if err != nil {
return nil, err
}
for entry, err := it.Next(); err == nil; entry, err = it.Next() {
str := entry.B58String()
entries = append(entries, &str)
}

return entries, nil
}

func (r *resolver) IpniAdvertisementEntriesCount(ctx context.Context, args struct{ AdCid string }) (int32, error) {
adCid, err := cid.Parse(args.AdCid)
if err != nil {
return 0, fmt.Errorf("parsing ad cid %s: %w", args.AdCid, err)
}

ad, err := r.idxProv.GetAdv(ctx, adCid)
if err != nil {
return 0, err
}

if len(ad.ContextID) == 0 {
return 0, nil
}

it, err := r.idxProvWrapper.MultihashLister(ctx, "", ad.ContextID)
if err != nil {
return 0, err
}
var count int32
for _, err := it.Next(); err == nil; _, err = it.Next() {
count++
}

return count, nil
}
27 changes: 18 additions & 9 deletions gql/resolver_rtvllog.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,10 @@ func (r *resolver) RetrievalLog(ctx context.Context, args retLogArgs) (*retrieva
}

type retrievalStatesArgs struct {
Cursor *gqltypes.Uint64 // database row id
Offset graphql.NullInt
Limit graphql.NullInt
Cursor *gqltypes.Uint64 // database row id
IsIndexer graphql.NullBool
Offset graphql.NullInt
Limit graphql.NullInt
}

func (r *resolver) RetrievalLogs(ctx context.Context, args retrievalStatesArgs) (*retrievalStateListResolver, error) {
Expand All @@ -158,14 +159,18 @@ func (r *resolver) RetrievalLogs(ctx context.Context, args retrievalStatesArgs)
limit = int(*args.Limit.Value)
}

// Fetch one extra row so that we can check if there are more rows
// beyond the limit
var isIndexer *bool
if args.IsIndexer.Set {
isIndexer = args.IsIndexer.Value
}
var cursor *uint64
if args.Cursor != nil {
cursorptr := uint64(*args.Cursor)
cursor = &cursorptr
}
rows, err := r.retDB.List(ctx, cursor, offset, limit+1)
// Fetch one extra row so that we can check if there are more rows
// beyond the limit
rows, err := r.retDB.List(ctx, isIndexer, cursor, offset, limit+1)
if err != nil {
return nil, err
}
Expand All @@ -176,7 +181,7 @@ func (r *resolver) RetrievalLogs(ctx context.Context, args retrievalStatesArgs)
}

// Get the total row count
count, err := r.retDB.Count(ctx)
count, err := r.retDB.Count(ctx, isIndexer)
if err != nil {
return nil, err
}
Expand All @@ -198,8 +203,12 @@ type retStateCount struct {
Period gqltypes.Uint64
}

func (r *resolver) RetrievalLogsCount(ctx context.Context) (*retStateCount, error) {
count, err := r.retDB.Count(ctx)
func (r *resolver) RetrievalLogsCount(ctx context.Context, args struct{ IsIndexer graphql.NullBool }) (*retStateCount, error) {
var isIndexer *bool
if args.IsIndexer.Set {
isIndexer = args.IsIndexer.Value
}
count, err := r.retDB.Count(ctx, isIndexer)
return &retStateCount{
Count: int32(count),
Period: gqltypes.Uint64(r.cfg.Dealmaking.RetrievalLogDuration),
Expand Down
35 changes: 33 additions & 2 deletions gql/schema.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,25 @@ type LID {
FlaggedPieces: Int!
}

type IpniProviderInfo {
PeerID: String!
Config: String!
}

type IpniAdvertisementMetadata {
Protocol: String!
Metadata: String!
}

type IpniAdvertisement {
ContextID: String!
Metadata: [IpniAdvertisementMetadata]!
PreviousEntry: String!
Provider: String!
Addresses: [String!]!
IsRemove: Boolean!
}

type FundsEscrow {
Available: BigInt!
Locked: BigInt!
Expand Down Expand Up @@ -485,10 +504,10 @@ type RootQuery {
retrievalLog(peerID: String!, transferID: Uint64!): RetrievalState

"""Get retrieval logs"""
retrievalLogs(cursor: Uint64, offset: Int, limit: Int): RetrievalStateList!
retrievalLogs(isIndexer: Boolean, cursor: Uint64, offset: Int, limit: Int): RetrievalStateList!

"""Get the number of retrieval logs"""
retrievalLogsCount: RetrievalStatesCount!
retrievalLogsCount(isIndexer: Boolean): RetrievalStatesCount!

"""Get a list of pieces that have been flagged as having problems"""
piecesFlagged(hasUnsealedCopy: Boolean, cursor: BigInt, offset: Int, limit: Int): FlaggedPiecesList!
Expand Down Expand Up @@ -517,6 +536,18 @@ type RootQuery {
"""Get sealing pipeline state"""
sealingpipeline: SealingPipeline!

"""Get IPNI Provider configuration and state"""
ipniProviderInfo: IpniProviderInfo!

"""Get IPNI advertisement"""
ipniAdvertisement(adCid: String!): IpniAdvertisement!

"""Get entries for an IPNI advertisement"""
ipniAdvertisementEntries(adCid: String!): [String]!

"""Get count of entries for an IPNI advertisement"""
ipniAdvertisementEntriesCount(adCid: String!): Int!

"""Get LID state"""
lid: LID!

Expand Down
8 changes: 5 additions & 3 deletions node/modules/storageminer.go
Original file line number Diff line number Diff line change
Expand Up @@ -650,14 +650,16 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(
}
}

func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyDeals *legacy.LegacyDealsManager, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter, ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *gql.Server {
func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyDeals *legacy.LegacyDealsManager, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, indexProv provider.Interface, idxProvWrapper *indexprovider.Wrapper, fullNode v1api.FullNode, bg gql.BlockGetter, ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *gql.Server {
return func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager,
storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API,
legacyDeals *legacy.LegacyDealsManager, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer,
ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter, ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *gql.Server {
ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory,
indexProv provider.Interface, idxProvWrapper *indexprovider.Wrapper, fullNode v1api.FullNode, bg gql.BlockGetter,
ssm *sectorstatemgr.SectorStateMgr, mpool *mpoolmonitor.MpoolMonitor) *gql.Server {

resolverCtx, cancel := context.WithCancel(context.Background())
resolver := gql.NewResolver(resolverCtx, cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyDeals, legacyProv, legacyDT, ps, sa, piecedirectory, publisher, fullNode, ssm, mpool)
resolver := gql.NewResolver(resolverCtx, cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyDeals, legacyProv, legacyDT, ps, sa, piecedirectory, publisher, indexProv, idxProvWrapper, fullNode, ssm, mpool)
server := gql.NewServer(resolver, bg)

lc.Append(fx.Hook{
Expand Down
7 changes: 6 additions & 1 deletion react/src/App.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import './App.css';
import React from 'react';
import { BrowserRouter, Routes, Route } from 'react-router-dom'
import {ApolloProvider} from "@apollo/react-hooks";
import {gqlClient,} from "./gql";
import {gqlClient} from "./gql";
import {Menu} from "./Menu";
import {StorageSpacePage} from "./StorageSpace";
import {FundsPage} from "./Funds";
Expand All @@ -22,6 +22,7 @@ import {PieceDoctorPage, InspectPiecePage, LIDPage, NoUnsealedSectorPieces, NoUn
import {RetrievalLogsPage} from "./RetrievalLogs";
import {RetrievalLogDetail} from "./RetrievalLogDetail";
import {MonitoringAlert} from "./MonitoringAlert";
import {IpniAdDetail, IpniAdEntries, IpniPage} from "./Ipni";

function App(props) {
return (
Expand All @@ -46,6 +47,10 @@ function App(props) {
<Route path="/retrieval-logs" element={<RetrievalLogsPage />} />
<Route path="/retrieval-logs/from/:cursor/page/:pageNum" element={<RetrievalLogsPage />} />
<Route path="/retrieval-logs/:peerID/:transferID" element={<RetrievalLogDetail />} />
<Route path="/ipni" element={<IpniPage />} />
<Route path="/ipni/from/:cursor/page/:pageNum" element={<IpniPage />} />
<Route path="/ipni/ad/:adCid" element={<IpniAdDetail />} />
<Route path="/ipni/ad/:adCid/entries" element={<IpniAdEntries />} />
<Route path="/storage-space" element={<StorageSpacePage />} />
<Route path="/lid" element={<LIDPage />} />
<Route path="/sealing-pipeline" element={<SealingPipelinePage />} />
Expand Down
31 changes: 31 additions & 0 deletions react/src/Components.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
.js-obj {
position: relative;
color: #777777;
line-height: 1.5em;
}

.js-obj.expandable > .js-obj-name {
cursor: pointer;
}

.js-obj .js-obj {
padding-left: 1em;
display: none;
}
.js-obj.expanded .js-obj {
display: block;
}

.js-obj .expand-collapse {
margin-left: 0.5em;
opacity: 0.5;
width: 0.8em;
height: 0.8em;
display: inline-block;
background: url("./bootstrap-icons/icons/caret-right-fill.svg");
background-size: cover;
}

.js-obj.expanded .expand-collapse {
background: url("./bootstrap-icons/icons/caret-down-fill.svg")
}
Loading

0 comments on commit 3165c26

Please sign in to comment.