Configuration

在 resources 文件夹下，可以通过配置 application.conf 文件，来配置爬虫和爬虫引擎。

HOCON

spider: {
  config: {
    autoProxy: false
    initialDelay: 0
    maxRetries: 3
    retryDelayMillis: 1000
    usePrintRequestPipeline: true
    useConsolePipeline: false
  }

  request: {
    sleepTime: 0
    autoSleepTime: true
    downloadDelay: 0
    autoDownloadDelay: true
    domainDelay: 0
    autoDomainDelay: true
  }

  queue: {
    // default、disruptor
    type: default
  }

  pipeline: {
    pipelineDelay: 0
    autoPipelineDelay: false
  }

  downloader: {
    # vertx、urlConnection、file
    type: vertx
    vertx: {
      options: {
        keepAlive: true
        reuseAddress: true
        followRedirects: true
        connectTimeout: 10000
        idleTimeout: 10
        maxWaitQueueSize: 10
      }
    }
  }

}


spiderEngine: {
  config: {
    port: 8715,
    useMonitor: false
  }

  registry: {
    zookeeper: {
      zkStr: "localhost:2181"
      zkPath: "/netdiscovery"
    }
    etcd: {
      etcdStr: "http://127.0.0.1:2379"
      etcdPath: "/netdiscovery"
    }
  }
}