diff --git a/cmd/aws/batch/config.go b/cmd/aws/batch/config.go index af0ae69cb..39f1190ac 100644 --- a/cmd/aws/batch/config.go +++ b/cmd/aws/batch/config.go @@ -170,7 +170,7 @@ func DefaultConfig() Config { c.Funnel.DynamoDB.TableBasename = "funnel" c.Funnel.DynamoDB.Region = "" c.Funnel.Worker.WorkDir = "/opt/funnel-work-dir" - c.Funnel.Worker.LogUpdateRate = time.Minute * 5 + c.Funnel.Worker.LogUpdateRate = config.Duration(time.Minute * 5) c.Funnel.Worker.LogTailSize = 10000 return c diff --git a/cmd/util/flags.go b/cmd/util/flags.go index 3be4176fb..628f0defe 100644 --- a/cmd/util/flags.go +++ b/cmd/util/flags.go @@ -76,7 +76,7 @@ func serverFlags(flagConf *config.Config) *pflag.FlagSet { f.StringVar(&flagConf.Server.HTTPPort, "Server.HTTPPort", flagConf.Server.HTTPPort, "HTTP Port") f.StringVar(&flagConf.Server.RPCPort, "Server.RPCPort", flagConf.Server.RPCPort, "RPC Port") f.StringVar(&flagConf.Server.ServiceName, "Server.ServiceName", flagConf.Server.ServiceName, "Host name or IP") - f.DurationVar(&flagConf.Server.RPCClientTimeout, "Server.RPCClientTimeout", flagConf.Server.RPCClientTimeout, "Request timeout for RPC client connections") + f.Var(flagConf.Server.RPCClientTimeout, "Server.RPCClientTimeout", "Request timeout for RPC client connections") f.UintVar(&flagConf.Server.RPCClientMaxRetries, "Server.RPCClientMaxRetries", flagConf.Server.RPCClientMaxRetries, "Maximum number of times that a request will be retried for failures") return f @@ -86,8 +86,8 @@ func workerFlags(flagConf *config.Config) *pflag.FlagSet { f := pflag.NewFlagSet("", pflag.ContinueOnError) f.Int64Var(&flagConf.Worker.LogTailSize, "Worker.LogTailSize", flagConf.Worker.LogTailSize, "Max bytes to store for stdout/stderr") - f.DurationVar(&flagConf.Worker.LogUpdateRate, "Worker.LogUpdateRate", flagConf.Worker.LogUpdateRate, "How often to send stdout/stderr log updates") - f.DurationVar(&flagConf.Worker.PollingRate, "Worker.PollingRate", flagConf.Worker.PollingRate, "How often to poll for cancel signals") + f.Var(flagConf.Worker.LogUpdateRate, "Worker.LogUpdateRate", "How often to send stdout/stderr log updates") + f.Var(flagConf.Worker.PollingRate, "Worker.PollingRate", "How often to poll for cancel signals") f.StringVar(&flagConf.Worker.WorkDir, "Worker.WorkDir", flagConf.Worker.WorkDir, "Working directory") f.BoolVar(&flagConf.Worker.LeaveWorkDir, "Worker.LeaveWorkDir", flagConf.Worker.LeaveWorkDir, "Leave working directory after execution") @@ -101,8 +101,8 @@ func nodeFlags(flagConf *config.Config) *pflag.FlagSet { f.Uint32Var(&flagConf.Node.Resources.Cpus, "Node.Resources.Cpus", flagConf.Node.Resources.Cpus, "Cpus available to Node") f.Float64Var(&flagConf.Node.Resources.RamGb, "Node.Resources.RamGb", flagConf.Node.Resources.RamGb, "Ram (GB) available to Node") f.Float64Var(&flagConf.Node.Resources.DiskGb, "Node.Resources.DiskGb", flagConf.Node.Resources.DiskGb, "Free disk (GB) available to Node") - f.DurationVar(&flagConf.Node.Timeout, "Node.Timeout", flagConf.Node.Timeout, "Node timeout in seconds") - f.DurationVar(&flagConf.Node.UpdateRate, "Node.UpdateRate", flagConf.Node.UpdateRate, "Node update rate") + f.Var(flagConf.Node.Timeout, "Node.Timeout", "Node timeout in seconds") + f.Var(flagConf.Node.UpdateRate, "Node.UpdateRate", "Node update rate") // TODO Metadata return f @@ -143,7 +143,7 @@ func dbFlags(flagConf *config.Config) *pflag.FlagSet { // mongodb f.StringSliceVar(&flagConf.MongoDB.Addrs, "MongoDB.Addrs", flagConf.MongoDB.Addrs, "Address of a MongoDB seed server. This flag can be used multiple times") f.StringVar(&flagConf.MongoDB.Database, "MongoDB.Database", flagConf.MongoDB.Database, "Database name in MongoDB") - f.DurationVar(&flagConf.MongoDB.Timeout, "MongoDB.Timeout", flagConf.MongoDB.Timeout, "Timeout in seconds for initial connection and follow up operations") + f.Var(flagConf.MongoDB.Timeout, "MongoDB.Timeout", "Timeout in seconds for initial connection and follow up operations") return f } @@ -169,7 +169,7 @@ func storageFlags(flagConf *config.Config) *pflag.FlagSet { // HTTP storage f.BoolVar(&flagConf.HTTPStorage.Disabled, "HTTPStorage.Disabled", flagConf.HTTPStorage.Disabled, "Disable storage backend") - f.DurationVar(&flagConf.HTTPStorage.Timeout, "HTTPStorage.Timeout", flagConf.HTTPStorage.Timeout, "Timeout in seconds for request") + f.Var(flagConf.HTTPStorage.Timeout, "HTTPStorage.Timeout", "Timeout in seconds for request") return f } @@ -193,11 +193,11 @@ func computeFlags(flagConf *config.Config) *pflag.FlagSet { f.StringVar(&flagConf.PBS.Template, "PBS.Template", flagConf.PBS.Template, "Path to submit template file") // Scheduler - f.DurationVar(&flagConf.Scheduler.ScheduleRate, "Scheduler.ScheduleRate", flagConf.Scheduler.ScheduleRate, "How often to run a scheduler iteration") + f.Var(flagConf.Scheduler.ScheduleRate, "Scheduler.ScheduleRate", "How often to run a scheduler iteration") f.IntVar(&flagConf.Scheduler.ScheduleChunk, "Scheduler.ScheduleChunk", flagConf.Scheduler.ScheduleChunk, "How many tasks to schedule in one iteration") - f.DurationVar(&flagConf.Scheduler.NodePingTimeout, "Scheduler.NodePingTimeout", flagConf.Scheduler.NodePingTimeout, "How long to wait for a node ping before marking it as dead") - f.DurationVar(&flagConf.Scheduler.NodeInitTimeout, "Scheduler.NodeInitTimeout", flagConf.Scheduler.NodeInitTimeout, "How long to wait for node initialization before marking it dead") - f.DurationVar(&flagConf.Scheduler.NodeDeadTimeout, "Scheduler.NodeDeadTimeout", flagConf.Scheduler.NodeDeadTimeout, "How long to wait before deleting a dead node from the DB") + f.Var(flagConf.Scheduler.NodePingTimeout, "Scheduler.NodePingTimeout", "How long to wait for a node ping before marking it as dead") + f.Var(flagConf.Scheduler.NodeInitTimeout, "Scheduler.NodeInitTimeout", "How long to wait for node initialization before marking it dead") + f.Var(flagConf.Scheduler.NodeDeadTimeout, "Scheduler.NodeDeadTimeout", "How long to wait before deleting a dead node from the DB") // Slurm f.StringVar(&flagConf.Slurm.Template, "Slurm.Template", flagConf.Slurm.Template, "Path to submit template file") diff --git a/compute/batch/backend.go b/compute/batch/backend.go index 1eb33fec3..dbc696fc5 100644 --- a/compute/batch/backend.go +++ b/compute/batch/backend.go @@ -151,7 +151,7 @@ func (b *Backend) Cancel(ctx context.Context, taskID string) error { // In this context a "FAILED" state is being used as a generic term that captures // one or more terminal states for the backend. func (b *Backend) reconcile(ctx context.Context) { - ticker := time.NewTicker(b.conf.ReconcileRate) + ticker := time.NewTicker(time.Duration(b.conf.ReconcileRate)) for { select { diff --git a/compute/htcondor/backend.go b/compute/htcondor/backend.go index 27ef6f115..5d311400c 100644 --- a/compute/htcondor/backend.go +++ b/compute/htcondor/backend.go @@ -8,6 +8,7 @@ import ( "os/exec" "regexp" "strings" + "time" "github.com/ohsu-comp-bio/funnel/compute" "github.com/ohsu-comp-bio/funnel/config" @@ -27,7 +28,7 @@ func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServ Database: reader, ExtractID: extractID, MapStates: mapStates, - ReconcileRate: conf.HTCondor.ReconcileRate, + ReconcileRate: time.Duration(conf.HTCondor.ReconcileRate), } if !conf.HTCondor.DisableReconciler { diff --git a/compute/pbs/backend.go b/compute/pbs/backend.go index 17962f0cc..3a92a4fdf 100644 --- a/compute/pbs/backend.go +++ b/compute/pbs/backend.go @@ -6,6 +6,7 @@ import ( "encoding/xml" "fmt" "os/exec" + "time" "github.com/ohsu-comp-bio/funnel/compute" "github.com/ohsu-comp-bio/funnel/config" @@ -25,7 +26,7 @@ func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServ Database: reader, ExtractID: extractID, MapStates: mapStates, - ReconcileRate: conf.PBS.ReconcileRate, + ReconcileRate: time.Duration(conf.PBS.ReconcileRate), } if !conf.PBS.DisableReconciler { diff --git a/compute/scheduler/node.go b/compute/scheduler/node.go index 8fb1a5402..ee53ab132 100644 --- a/compute/scheduler/node.go +++ b/compute/scheduler/node.go @@ -33,7 +33,7 @@ func NewNodeProcess(ctx context.Context, conf config.Config, factory Worker, log log.Error("error detecting resources", "error", derr) } - timeout := util.NewIdleTimeout(conf.Node.Timeout) + timeout := util.NewIdleTimeout(time.Duration(conf.Node.Timeout)) state := NodeState_UNINITIALIZED return &NodeProcess{ @@ -71,7 +71,7 @@ func (n *NodeProcess) Run(ctx context.Context) { n.checkConnection(ctx) n.sync(ctx) - ticker := time.NewTicker(n.conf.Node.UpdateRate) + ticker := time.NewTicker(time.Duration(n.conf.Node.UpdateRate)) defer ticker.Stop() for { @@ -182,7 +182,7 @@ func (n *NodeProcess) runTask(ctx context.Context, id string) { // task cannot fully complete until it has successfully removed the // assigned ID from the node database. this helps prevent tasks from // running multiple times. - ticker := time.NewTicker(n.conf.Node.UpdateRate) + ticker := time.NewTicker(time.Duration(n.conf.Node.UpdateRate)) defer ticker.Stop() for { select { diff --git a/compute/scheduler/node_util.go b/compute/scheduler/node_util.go index ed0bac5b1..46de0cd6d 100644 --- a/compute/scheduler/node_util.go +++ b/compute/scheduler/node_util.go @@ -118,16 +118,16 @@ func UpdateNodeState(nodes []*Node, conf config.Scheduler) []*Node { if node.State == NodeState_UNINITIALIZED || node.State == NodeState_INITIALIZING { // The node is initializing, which has a more liberal timeout. - if d > conf.NodeInitTimeout { + if d > time.Duration(conf.NodeInitTimeout) { // Looks like the node failed to initialize. Mark it dead node.State = NodeState_DEAD } - } else if node.State == NodeState_DEAD && d > conf.NodeDeadTimeout { + } else if node.State == NodeState_DEAD && d > time.Duration(conf.NodeDeadTimeout) { // The node has been dead for long enough. node.State = NodeState_GONE - } else if d > conf.NodePingTimeout { + } else if d > time.Duration(conf.NodePingTimeout) { // The node hasn't pinged in awhile, mark it dead. node.State = NodeState_DEAD } diff --git a/compute/scheduler/scheduler.go b/compute/scheduler/scheduler.go index aa9cae1a6..2d9d93190 100644 --- a/compute/scheduler/scheduler.go +++ b/compute/scheduler/scheduler.go @@ -32,7 +32,7 @@ type Scheduler struct { // request the the configured backend schedule them, and // act on offers made by the backend. func (s *Scheduler) Run(ctx context.Context) error { - ticker := time.NewTicker(s.Conf.ScheduleRate) + ticker := time.NewTicker(time.Duration(s.Conf.ScheduleRate)) for { select { case <-ctx.Done(): diff --git a/compute/slurm/backend.go b/compute/slurm/backend.go index 1d021982b..c69a98453 100644 --- a/compute/slurm/backend.go +++ b/compute/slurm/backend.go @@ -9,6 +9,7 @@ import ( "os/exec" "regexp" "strings" + "time" "github.com/ohsu-comp-bio/funnel/compute" "github.com/ohsu-comp-bio/funnel/config" @@ -28,7 +29,7 @@ func NewBackend(ctx context.Context, conf config.Config, reader tes.ReadOnlyServ Database: reader, ExtractID: extractID, MapStates: mapStates, - ReconcileRate: conf.Slurm.ReconcileRate, + ReconcileRate: time.Duration(conf.Slurm.ReconcileRate), } if !conf.Slurm.DisableReconciler { diff --git a/config/config.go b/config/config.go index 7bcebb3f3..8edaea27d 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import ( "os" - "time" "github.com/ohsu-comp-bio/funnel/logger" ) @@ -57,7 +56,7 @@ type Server struct { // The timeout to use for making RPC client connections in nanoseconds // This timeout is Only enforced when used in conjunction with the // grpc.WithBlock dial option. - RPCClientTimeout time.Duration + RPCClientTimeout Duration // The maximum number of times that a request will be retried for failures. // Time between retries follows an exponential backoff starting at 5 seconds // up to 1 minute @@ -88,15 +87,15 @@ func (c *Server) RPCAddress() string { // Scheduler contains funnel's basic scheduler configuration. type Scheduler struct { // How often to run a scheduler iteration. - ScheduleRate time.Duration + ScheduleRate Duration // How many tasks to schedule in one iteration. ScheduleChunk int // How long to wait for a node ping before marking it as dead - NodePingTimeout time.Duration + NodePingTimeout Duration // How long to wait for node initialization before marking it dead - NodeInitTimeout time.Duration + NodeInitTimeout Duration // How long to wait before deleting a dead node from the DB. - NodeDeadTimeout time.Duration + NodeDeadTimeout Duration } // Node contains the configuration for a node. Nodes track available resources @@ -112,9 +111,9 @@ type Node struct { } // If the node has been idle for longer than the timeout, it will shut down. // -1 means there is no timeout. 0 means timeout immediately after the first task. - Timeout time.Duration + Timeout Duration // How often the node sends update requests to the server. - UpdateRate time.Duration + UpdateRate Duration Metadata map[string]string } @@ -123,11 +122,11 @@ type Worker struct { // Directory to write task files to WorkDir string // How often the worker should poll for cancel signals - PollingRate time.Duration + PollingRate Duration // How often to update stdout/stderr log fields. // Setting this to 0 will result in these fields being updated a single time // after the executor exits. - LogUpdateRate time.Duration + LogUpdateRate Duration // Max bytes of stdout/stderr to store in the database. // Setting this to 0 turns off stdout/stderr logging. LogTailSize int64 @@ -145,7 +144,7 @@ type HPCBackend struct { DisableReconciler bool // ReconcileRate is how often the compute backend compares states in Funnel's backend // to those reported by the backend - ReconcileRate time.Duration + ReconcileRate Duration Template string } @@ -164,7 +163,7 @@ type MongoDB struct { // first connecting and on follow up operations in the session. If // timeout is zero, the call may block forever waiting for a connection // to be established. - Timeout time.Duration + Timeout Duration // Username and Password inform the credentials for the initial authentication // done on the database defined by the Database field. Username string @@ -222,7 +221,7 @@ type AWSBatch struct { DisableReconciler bool // ReconcileRate is how often the compute backend compares states in Funnel's backend // to those reported by AWS Batch - ReconcileRate time.Duration + ReconcileRate Duration AWSConfig } @@ -326,7 +325,7 @@ func (s SwiftStorage) Valid() bool { type HTTPStorage struct { Disabled bool // Timeout duration for http GET calls - Timeout time.Duration + Timeout Duration } // Valid validates the HTTPStorage configuration. diff --git a/config/default-config.yaml b/config/default-config.yaml index e75a69d31..9a7b54fcd 100644 --- a/config/default-config.yaml +++ b/config/default-config.yaml @@ -39,8 +39,7 @@ Server: DisableHTTPCache: true # RPC client connection timeout. - # In nanoseconds. - RPCClientTimeout: 60000000000 # 30 seconds + RPCClientTimeout: 60s # The maximum number of times that a request will be retried for failures. # Time between retries follows an exponential backoff starting at 5 seconds @@ -51,16 +50,13 @@ Server: # The scheduler is used for the Manual compute backend. Scheduler: # How often to run a scheduler iteration. - # In nanoseconds. - ScheduleRate: 1000000000 # 1 second + ScheduleRate: 1s # How many tasks to schedule in one iteration. ScheduleChunk: 10 # How long to wait between updates before marking a node dead. - # In nanoseconds. - NodePingTimeout: 60000000000 # 1 minute + NodePingTimeout: 1m # How long to wait for a node to start, before marking the node dead. - # In nanoseconds. - NodeInitTimeout: 300000000000 # 5 minutes + NodeInitTimeout: 5m Node: # If empty, a node ID will be automatically generated. @@ -68,7 +64,7 @@ Node: # If the node has been idle for longer than the timeout, it will shut down. # -1 means there is no timeout. 0 means timeout immediately after the first task. - Timeout: -1 + Timeout: -1s # A Node will automatically try to detect what resources are available to it. # Defining Resources in the Node configuration overrides this behavior. @@ -84,8 +80,7 @@ Node: # For low-level tuning. # How often to sync with the Funnel server. - # In nanoseconds. - UpdateRate: 5000000000 # 5 seconds + UpdateRate: 5s Worker: # Files created during processing will be written in this directory. @@ -93,15 +88,13 @@ Worker: # For low-level tuning. # How often to poll for cancel signals - # In nanoseconds. - PollingRate: 5000000000 # 5 seconds + PollingRate: 5s # For low-level tuning. # How often to send stdout/err task log updates to the Funnel server. # Setting this to 0 will result in these fields being updated a single time # after the executor exits. - # In nanoseconds. - LogUpdateRate: 5000000000 # 5 seconds + LogUpdateRate: 5s # Max bytes to store for stdout/err in the task log. LogTailSize: 10000 # 10 KB @@ -152,7 +145,7 @@ MongoDB: # first connecting and on follow up operations in the session. If # timeout is zero, the call may block forever waiting for a connection # to be established. - Timeout: 300000000000 # 5 minutes + Timeout: 5m # Username and Password inform the credentials for the initial authentication # done on the database defined by the Database field. Username: "" @@ -174,7 +167,7 @@ HTCondor: DisableReconciler: true # ReconcileRate is how often the compute backend compares states in Funnel's backend # to those reported by the backend - ReconcileRate: 1800000000000 # 30 minutes + ReconcileRate: 30m Template: | universe = vanilla getenv = True @@ -204,7 +197,7 @@ PBS: DisableReconciler: true # ReconcileRate is how often the compute backend compares states in Funnel's backend # to those reported by the backend - ReconcileRate: 1800000000000 # 30 minutes + ReconcileRate: 30m Template: | #!bin/bash #PBS -N {{.NodeId}} @@ -248,7 +241,7 @@ Slurm: DisableReconciler: true # ReconcileRate is how often the compute backend compares states in Funnel's backend # to those reported by the backend - ReconcileRate: 1800000000000 # 30 minutes + ReconcileRate: 30m Template: | #!/bin/bash #SBATCH --job-name {{.NodeId}} @@ -274,7 +267,7 @@ AWSBatch: DisableReconciler: true # ReconcileRate is how often the compute backend compares states in Funnel's backend # to those reported by AWS Batch - ReconcileRate: 1800000000000 # 30 minutes + ReconcileRate: 30m # JobDefinition can be either a name or the Amazon Resource Name (ARN). JobDefinition: "funnel-job-def" # JobQueue can be either a name or the Amazon Resource Name (ARN). @@ -300,8 +293,7 @@ LocalStorage: # HTTPStorage is used to download public files on the web via a GET request. HTTPStorage: # Timeout for http(s) GET requests. - # In nanoseconds. - Timeout: 30000000000 # 30 seconds + Timeout: 30s AmazonS3: Disabled: false diff --git a/config/default.go b/config/default.go index cac945d2a..987e52c1b 100644 --- a/config/default.go +++ b/config/default.go @@ -29,7 +29,7 @@ func DefaultConfig() Config { RPCPort: "9090", ServiceName: "Funnel", DisableHTTPCache: true, - RPCClientTimeout: time.Second * 60, + RPCClientTimeout: Duration(time.Second * 60), RPCClientMaxRetries: 10, } @@ -40,21 +40,21 @@ func DefaultConfig() Config { // funnel components Server: server, Scheduler: Scheduler{ - ScheduleRate: time.Second, + ScheduleRate: Duration(time.Second), ScheduleChunk: 10, - NodePingTimeout: time.Minute, - NodeInitTimeout: time.Minute * 5, - NodeDeadTimeout: time.Minute * 5, + NodePingTimeout: Duration(time.Minute), + NodeInitTimeout: Duration(time.Minute * 5), + NodeDeadTimeout: Duration(time.Minute * 5), }, Node: Node{ Timeout: -1, - UpdateRate: time.Second * 5, + UpdateRate: Duration(time.Second * 5), Metadata: map[string]string{}, }, Worker: Worker{ WorkDir: workDir, - PollingRate: time.Second * 5, - LogUpdateRate: time.Second * 5, + PollingRate: Duration(time.Second * 5), + LogUpdateRate: Duration(time.Second * 5), LogTailSize: 10000, }, Logger: logger.DefaultConfig(), @@ -71,7 +71,7 @@ func DefaultConfig() Config { }, MongoDB: MongoDB{ Addrs: []string{"localhost"}, - Timeout: time.Minute * 5, + Timeout: Duration(time.Minute * 5), Database: "funnel", }, Kafka: Kafka{ @@ -82,7 +82,7 @@ func DefaultConfig() Config { AllowedDirs: allowedDirs, }, HTTPStorage: HTTPStorage{ - Timeout: time.Second * 60, + Timeout: Duration(time.Second * 60), }, AmazonS3: AmazonS3Storage{ AWSConfig: AWSConfig{ @@ -95,7 +95,7 @@ func DefaultConfig() Config { } // compute - reconcile := time.Minute * 10 + reconcile := Duration(time.Minute * 10) htcondorTemplate, _ := intern.Asset("config/htcondor-template.txt") c.HTCondor.Template = string(htcondorTemplate) diff --git a/config/duration.go b/config/duration.go new file mode 100644 index 000000000..fcfb67152 --- /dev/null +++ b/config/duration.go @@ -0,0 +1,50 @@ +package config + +import ( + "time" +) + +// Duration is a wrapper type for time.Duration which provides human-friendly +// text (un)marshaling. +// See https://github.com/golang/go/issues/16039 +type Duration time.Duration + +// String returns the string representation of the duration. +func (d Duration) String() string { + return time.Duration(d).String() +} + +// UnmarshalText parses text into a duration value. +func (d *Duration) UnmarshalText(text []byte) error { + // Ignore if there is no value set. + if len(text) == 0 { + return nil + } + + // Otherwise parse as a duration formatted string. + duration, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + + // Set duration and return. + *d = Duration(duration) + return nil +} + +// MarshalText converts a duration to text. +func (d Duration) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// Set sets the duration from the given string. +// Implements the pflag.Value interface. +func (d Duration) Set(raw string) error { + return d.UnmarshalText([]byte(raw)) +} + +// Type returns the name of this type. +// Implements the pflag.Value interface. +func (d Duration) Type() string { + return "duration" +} diff --git a/database/mongodb/new.go b/database/mongodb/new.go index 455533357..63711bb49 100644 --- a/database/mongodb/new.go +++ b/database/mongodb/new.go @@ -2,6 +2,7 @@ package mongodb import ( "fmt" + "time" "github.com/ohsu-comp-bio/funnel/config" "gopkg.in/mgo.v2" @@ -23,7 +24,7 @@ func NewMongoDB(conf config.MongoDB) (*MongoDB, error) { Username: conf.Username, Password: conf.Password, Database: conf.Database, - Timeout: conf.Timeout, + Timeout: time.Duration(conf.Timeout), // DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { // return tls.Dial("tcp", addr.String(), &tls.Config{}) // }, diff --git a/storage/http.go b/storage/http.go index d7764938d..11d66e732 100644 --- a/storage/http.go +++ b/storage/http.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/ohsu-comp-bio/funnel/config" "github.com/ohsu-comp-bio/funnel/tes" @@ -21,7 +22,7 @@ type HTTPBackend struct { // NewHTTPBackend creates a new HTTPBackend instance. func NewHTTPBackend(conf config.HTTPStorage) (*HTTPBackend, error) { client := &http.Client{ - Timeout: conf.Timeout, + Timeout: time.Duration(conf.Timeout), } return &HTTPBackend{client}, nil } diff --git a/util/rpc/rpc.go b/util/rpc/rpc.go index 4fc7e811c..61ba60a01 100644 --- a/util/rpc/rpc.go +++ b/util/rpc/rpc.go @@ -38,7 +38,7 @@ func (c *loginCreds) RequireTransportSecurity() bool { // Dial returns a new gRPC ClientConn with some default dial and call options set func Dial(pctx context.Context, conf config.Server, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - ctx, cancel := context.WithTimeout(pctx, conf.RPCClientTimeout) + ctx, cancel := context.WithTimeout(pctx, time.Duration(conf.RPCClientTimeout)) defer cancel() defaultOpts := []grpc.DialOption{ diff --git a/website/content/docs/compute/deployment.md b/website/content/docs/compute/deployment.md index e3fc31d83..b07c713b1 100644 --- a/website/content/docs/compute/deployment.md +++ b/website/content/docs/compute/deployment.md @@ -37,19 +37,16 @@ Compute: manual Scheduler: # How often to run a scheduler iteration. - # In nanoseconds. - ScheduleRate: 1000000000 # 1 second + ScheduleRate: 1s # How many tasks to schedule in one iteration. ScheduleChunk: 10 # How long to wait between updates before marking a node dead. - # In nanoseconds. - NodePingTimeout: 60000000000 # 1 minute + NodePingTimeout: 1m # How long to wait for a node to start, before marking the node dead. - # In nanoseconds. - NodeInitTimeout: 300000000000 # 5 minutes + NodeInitTimeout: 5m Node: @@ -58,7 +55,7 @@ Node: # If the node has been idle for longer than the timeout, it will shut down. # -1 means there is no timeout. 0 means timeout immediately after the first task. - Timeout: -1 + Timeout: -1s # A Node will automatically try to detect what resources are available to it. # Defining Resources in the Node configuration overrides this behavior. @@ -72,8 +69,7 @@ Node: # For low-level tuning. # How often to sync with the Funnel server. - # In nanoseconds. - UpdateRate: 5000000000 # 5 seconds + UpdateRate: 5s Logger: # Logging levels: debug, info, error @@ -81,7 +77,3 @@ Logger: # Write logs to this path. If empty, logs are written to stderr. OutputFile: "" ``` - -### Known issues - -The config uses nanoseconds for duration values. See [issue #342](https://github.com/ohsu-comp-bio/funnel/issues/342). diff --git a/website/content/docs/storage/http.md b/website/content/docs/storage/http.md index 2eb7f19c7..3bbc29c35 100644 --- a/website/content/docs/storage/http.md +++ b/website/content/docs/storage/http.md @@ -18,8 +18,7 @@ worker config: HTTPStorage: Disabled: false # Timeout for http(s) GET requests. - # In nanoseconds. - Timeout: 60000000000 # 60 seconds + Timeout: 30s ``` ### Example task diff --git a/worker/step.go b/worker/step.go index e9174c56f..52af3ec0a 100644 --- a/worker/step.go +++ b/worker/step.go @@ -31,7 +31,7 @@ func (s *stepWorker) Run(ctx context.Context) error { // Tail the stdout/err log streams. if s.Conf.LogTailSize > 0 { if s.Conf.LogUpdateRate > 0 { - stdout, stderr = s.Event.StreamLogTail(subctx, s.Conf.LogTailSize, s.Conf.LogUpdateRate) + stdout, stderr = s.Event.StreamLogTail(subctx, s.Conf.LogTailSize, time.Duration(s.Conf.LogUpdateRate)) } else { stdout, stderr = s.Event.LogTail(subctx, s.Conf.LogTailSize) } diff --git a/worker/worker.go b/worker/worker.go index 3979b3477..559eaf9a0 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -336,7 +336,7 @@ func (r *DefaultWorker) pollForCancel(pctx context.Context, taskID string, cance // Start a goroutine that polls the server to watch for a canceled state. // If a cancel state is found, "taskctx" is canceled. go func() { - ticker := time.NewTicker(r.Conf.PollingRate) + ticker := time.NewTicker(time.Duration(r.Conf.PollingRate)) defer ticker.Stop() for {