Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add graphman config check, place, pools to graphql api #5751

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Next Next commit
core,node: move config to core, use config check(cli) from core & upd…
…ate imports
shiyasmohd committed Dec 30, 2024
commit 605eb530b60057966dd5f318e1ef91e3c9dde9ff
8 changes: 7 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions core/graphman/Cargo.toml
Original file line number Diff line number Diff line change
@@ -12,3 +12,7 @@ graphman-store = { workspace = true }
itertools = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
graph-chain-ethereum = { path = "../../chain/ethereum" }
serde = { workspace = true }
url = "2.5.2"
shellexpand = "3.1.0"
42 changes: 42 additions & 0 deletions core/graphman/src/commands/config/check.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
use crate::config::Config;
use anyhow::Error;
use graph::{components::subgraph::Settings, env::EnvVars};

pub struct ConfigCheckResult {
pub validated: bool,
pub validated_subgraph_settings: bool,
pub config_json: Option<String>,
}

pub fn check(config: &Config, print: bool) -> Result<ConfigCheckResult, Error> {
let mut result = ConfigCheckResult {
validated: false,
validated_subgraph_settings: false,
config_json: None,
};
if print {
match config.to_json() {
Ok(txt) => {
result.config_json = Some(txt);
}
Err(err) => return Err(anyhow::format_err!("error serializing config: {}", err)),
};
}
let env_vars = EnvVars::from_env().unwrap();
if let Some(path) = &env_vars.subgraph_settings {
match Settings::from_file(path) {
Ok(_) => {
result.validated_subgraph_settings = true;
}
Err(e) => {
return Err(anyhow::format_err!(
"configuration error in subgraph settings {}: {}",
path,
e
));
}
}
};
result.validated = true;
Ok(result)
}
1 change: 1 addition & 0 deletions core/graphman/src/commands/config/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod check;
1 change: 1 addition & 0 deletions core/graphman/src/commands/mod.rs
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pub mod config;
pub mod deployment;
2 changes: 1 addition & 1 deletion node/src/config.rs → core/graphman/src/config.rs
Original file line number Diff line number Diff line change
@@ -19,9 +19,9 @@ use graph::{
use graph_chain_ethereum as ethereum;
use graph_chain_ethereum::NodeCapabilities;
use graph_store_postgres::{DeploymentPlacer, Shard as ShardName, PRIMARY_SHARD};
use serde::Serialize;

use graph::http::{HeaderMap, Uri};
use serde::Serialize;
use std::{
collections::{BTreeMap, BTreeSet},
fmt,
1 change: 1 addition & 0 deletions core/graphman/src/lib.rs
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
mod error;

pub mod commands;
pub mod config;
pub mod deployment;
pub mod execution_tracker;

4 changes: 2 additions & 2 deletions node/src/bin/manager.rs
Original file line number Diff line number Diff line change
@@ -19,7 +19,6 @@ use graph::{
};
use graph_chain_ethereum::EthereumAdapter;
use graph_graphql::prelude::GraphQlRunner;
use graph_node::config::{self, Config as Cfg};
use graph_node::manager::color::Terminal;
use graph_node::manager::commands;
use graph_node::network_setup::Networks;
@@ -34,6 +33,7 @@ use graph_store_postgres::{
connection_pool::ConnectionPool, BlockStore, NotificationSender, Shard, Store, SubgraphStore,
SubscriptionManager, PRIMARY_SHARD,
};
use graphman::config::{self, Config as Cfg};
use itertools::Itertools;
use lazy_static::lazy_static;
use std::str::FromStr;
@@ -1184,7 +1184,7 @@ async fn main() -> anyhow::Result<()> {
Place { name, network } => {
commands::config::place(&ctx.config.deployment, &name, &network)
}
Check { print } => commands::config::check(&ctx.config, print),
Check { print } => commands::config_cmd::check::run(&ctx.config, print),
Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard),
Provider { features, network } => {
let logger = ctx.logger.clone();
6 changes: 3 additions & 3 deletions node/src/chain.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use crate::config::{Config, ProviderDetails};
use crate::network_setup::{
AdapterConfiguration, EthAdapterConfig, FirehoseAdapterConfig, Networks,
};
@@ -31,6 +30,7 @@ use graph::tokio::time::timeout;
use graph::url::Url;
use graph_chain_ethereum::{self as ethereum, Transport};
use graph_store_postgres::{BlockStore, ChainHeadUpdateListener};
use graphman::config::{Config, ProviderDetails};
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::sync::Arc;
@@ -240,7 +240,7 @@ pub async fn create_ethereum_networks_for_chain(
"capabilities" => capabilities
);

use crate::config::Transport::*;
use graphman::config::Transport::*;

let transport = match web3.transport {
Rpc => Transport::new_rpc(
@@ -553,13 +553,13 @@ pub async fn networks_as_chains(

#[cfg(test)]
mod test {
use crate::config::{Config, Opt};
use crate::network_setup::{AdapterConfiguration, Networks};
use graph::components::network_provider::ChainName;
use graph::endpoint::EndpointMetrics;
use graph::log::logger;
use graph::prelude::{tokio, MetricsRegistry};
use graph_chain_ethereum::NodeCapabilities;
use graphman::config::{Config, Opt};
use std::sync::Arc;

#[tokio::test]
1 change: 0 additions & 1 deletion node/src/lib.rs
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@ use graph::{prelude::MetricsRegistry, prometheus::Registry};
extern crate diesel;

pub mod chain;
pub mod config;
pub mod network_setup;
pub mod opt;
pub mod store_builder;
2 changes: 1 addition & 1 deletion node/src/main.rs
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@ use graph_core::{
SubgraphRegistrar as IpfsSubgraphRegistrar,
};
use graph_graphql::prelude::GraphQlRunner;
use graph_node::config::Config;
use graph_node::network_setup::Networks;
use graph_node::opt;
use graph_node::store_builder::StoreBuilder;
@@ -32,6 +31,7 @@ use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer;
use graph_store_postgres::connection_pool::ConnectionPool;
use graph_store_postgres::Store;
use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender};
use graphman::config::Config;
use graphman_server::GraphmanServer;
use graphman_server::GraphmanServerConfig;
use std::io::{BufRead, BufReader};
3 changes: 2 additions & 1 deletion node/src/manager/commands/config.rs
Original file line number Diff line number Diff line change
@@ -16,7 +16,8 @@ use graph::{
use graph_chain_ethereum::NodeCapabilities;
use graph_store_postgres::DeploymentPlacer;

use crate::{config::Config, network_setup::Networks};
use crate::network_setup::Networks;
use graphman::config::Config;

pub fn place(placer: &dyn DeploymentPlacer, name: &str, network: &str) -> Result<(), Error> {
match placer.place(name, network).map_err(|s| anyhow!(s))? {
26 changes: 26 additions & 0 deletions node/src/manager/commands/config_cmd/check.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
use anyhow::Error;
use graphman::{commands::config::check::check, config::Config};

pub fn run(config: &Config, print: bool) -> Result<(), Error> {
let check = check(config, print);

match check {
Ok(res) => {
if res.validated {
println!("Successfully validated configuration");
}
if res.validated_subgraph_settings {
println!("Successfully validated subgraph settings");
}
if let Some(txt) = res.config_json {
println!("{}", txt);
}
}
Err(e) => {
eprintln!("configuration error: {}", e);
std::process::exit(1);
}
}

Ok(())
}
1 change: 1 addition & 0 deletions node/src/manager/commands/config_cmd/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod check;
1 change: 1 addition & 0 deletions node/src/manager/commands/mod.rs
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ pub mod assign;
pub mod chain;
pub mod check_blocks;
pub mod config;
pub mod config_cmd;
pub mod copy;
pub mod create;
pub mod database;
2 changes: 1 addition & 1 deletion node/src/manager/commands/run.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

use crate::config::Config;
use crate::manager::PanicSubscriptionManager;
use crate::network_setup::Networks;
use crate::store_builder::StoreBuilder;
@@ -26,6 +25,7 @@ use graph_core::{
SubgraphAssignmentProvider as IpfsSubgraphAssignmentProvider, SubgraphInstanceManager,
SubgraphRegistrar as IpfsSubgraphRegistrar,
};
use graphman::config::Config;

fn locate(store: &dyn SubgraphStore, hash: &str) -> Result<DeploymentLocator, anyhow::Error> {
let mut locators = store.locators(hash)?;
2 changes: 1 addition & 1 deletion node/src/network_setup.rs
Original file line number Diff line number Diff line change
@@ -185,7 +185,7 @@ impl Networks {

pub async fn from_config(
logger: Logger,
config: &crate::config::Config,
config: &graphman::config::Config,
registry: Arc<MetricsRegistry>,
endpoint_metrics: Arc<EndpointMetrics>,
provider_checks: &[Arc<dyn ProviderCheck>],
2 changes: 1 addition & 1 deletion node/src/opt.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ use clap::Parser;
use git_testament::{git_testament, render_testament};
use lazy_static::lazy_static;

use crate::config;
use graphman::config;

git_testament!(TESTAMENT);
lazy_static! {
2 changes: 1 addition & 1 deletion node/src/store_builder.rs
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ use graph_store_postgres::{
SubscriptionManager, PRIMARY_SHARD,
};

use crate::config::{Config, Shard};
use graphman::config::{Config, Shard};

pub struct StoreBuilder {
logger: Logger,
1 change: 1 addition & 0 deletions store/test-store/Cargo.toml
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@ lazy_static = "1.5"
hex-literal = "0.4"
diesel = { workspace = true }
prost-types = { workspace = true }
graphman = { workspace = true }

[dev-dependencies]
hex = "0.4.3"
2 changes: 1 addition & 1 deletion store/test-store/src/store.rs
Original file line number Diff line number Diff line change
@@ -23,14 +23,14 @@ use graph_graphql::prelude::{
execute_query, Query as PreparedQuery, QueryExecutionOptions, StoreResolver,
};
use graph_graphql::test_support::GraphQLMetrics;
use graph_node::config::{Config, Opt};
use graph_node::store_builder::StoreBuilder;
use graph_store_postgres::layout_for_tests::FAKE_NETWORK_SHARED;
use graph_store_postgres::{connection_pool::ConnectionPool, Shard, SubscriptionManager};
use graph_store_postgres::{
BlockStore as DieselBlockStore, DeploymentPlacer, SubgraphStore as DieselSubgraphStore,
PRIMARY_SHARD,
};
use graphman::config::{Config, Opt};
use hex_literal::hex;
use lazy_static::lazy_static;
use std::collections::BTreeSet;
8 changes: 6 additions & 2 deletions tests/Cargo.toml
Original file line number Diff line number Diff line change
@@ -9,16 +9,20 @@ assert-json-diff = "2.0.2"
async-stream = "0.3.5"
graph = { path = "../graph" }
graph-chain-ethereum = { path = "../chain/ethereum" }
graph-chain-substreams= {path = "../chain/substreams"}
graph-chain-substreams = { path = "../chain/substreams" }
graph-node = { path = "../node" }
graph-core = { path = "../core" }
graph-graphql = { path = "../graphql" }
graph-store-postgres = { path = "../store/postgres" }
graph-server-index-node = { path = "../server/index-node" }
graph-runtime-wasm = { path = "../runtime/wasm" }
graphman = { workspace = true }
serde = { workspace = true }
serde_yaml = { workspace = true }
slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] }
slog = { version = "2.7.0", features = [
"release_max_level_trace",
"max_level_trace",
] }
tokio = { version = "1.38.0", features = ["rt", "macros", "process"] }
# Once graph upgrades to web3 0.19, we don't need this anymore. The version
# here needs to be kept in sync with the web3 version that the graph crate
3 changes: 2 additions & 1 deletion tests/src/fixture/mod.rs
Original file line number Diff line number Diff line change
@@ -51,10 +51,11 @@ use graph_core::{
SubgraphRegistrar as IpfsSubgraphRegistrar, SubgraphTriggerProcessor,
};
use graph_node::manager::PanicSubscriptionManager;
use graph_node::{config::Config, store_builder::StoreBuilder};
use graph_node::store_builder::StoreBuilder;
use graph_runtime_wasm::RuntimeHostBuilder;
use graph_server_index_node::IndexNodeService;
use graph_store_postgres::{ChainHeadUpdateListener, ChainStore, Store, SubgraphStore};
use graphman::config::Config;
use serde::Deserialize;
use slog::{crit, debug, info, o, Discard, Logger};
use std::env::VarError;