Merge branch 'master' into jaegertracing#1921

.gitignore

@@ -29,3 +29,5 @@ cmd/docs/*.1
 cmd/docs/*.yaml
 crossdock/crossdock-*
 run-crossdock.log
+
+__pycache__

Makefile

@@ -95,6 +95,11 @@ storage-integration-test: go-gen
 	go clean -testcache
 	bash -c "set -e; set -o pipefail; $(GOTEST) $(STORAGE_PKGS) | $(COLORIZE)"
 
+.PHONE: test-compile-es-scripts
+test-compile-es-scripts:
+	docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
+	docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py
+
 .PHONY: index-cleaner-integration-test
 index-cleaner-integration-test: docker-images-elastic
 	# Expire tests results for storage integration tests since the environment might change

pkg/es/config/config.go

@@ -302,14 +302,16 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp
 	} else {
 		httpTransport := &http.Transport{
 			Proxy: http.ProxyFromEnvironment,
+			// #nosec G402
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: c.TLS.SkipHostVerify},
 		}
 		if c.TLS.CaPath != "" {
 			ctls := &TLSConfig{CaPath: c.TLS.CaPath}
 			ca, err := ctls.loadCertificate()
 			if err != nil {
 				return nil, err
 			}
-			httpTransport.TLSClientConfig = &tls.Config{RootCAs: ca}
+			httpTransport.TLSClientConfig.RootCAs = ca
 		}
 
 		token := ""

plugin/storage/es/esCleaner.py

@@ -23,20 +23,10 @@ def main():
         print('ES_TLS_CA ... Path to TLS CA file.')
         print('ES_TLS_CERT ... Path to TLS certificate file.')
         print('ES_TLS_KEY ... Path to TLS key file.')
+        print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
         sys.exit(1)
 
-    username = os.getenv("ES_USERNAME")
-    password = os.getenv("ES_PASSWORD")
-
-    if username is not None and password is not None:
-        client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
-    elif str2bool(os.getenv("ES_TLS", 'false')):
-        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
-        context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
-        client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
-    else:
-        client = elasticsearch.Elasticsearch(sys.argv[2:])
-
+    client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
     ilo = curator.IndexList(client)
     empty_list(ilo, 'Elasticsearch has no indices')
 
@@ -102,5 +92,21 @@ def str2bool(v):
     return v.lower() in ('true', '1')
 
 
+def create_client(username, password, tls, ca, cert, key, skipHostVerify):
+    context = ssl.create_default_context()
+    if ca is not None:
+        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
+    elif skipHostVerify:
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+    if username is not None and password is not None:
+        return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
+    elif tls:
+        context.load_cert_chain(certfile=cert, keyfile=key)
+        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
+    else:
+        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
+
+
 if __name__ == "__main__":
     main()

plugin/storage/es/esRollover.py

@@ -35,6 +35,7 @@ def main():
         print('ES_TLS_CA ... Path to TLS CA file.')
         print('ES_TLS_CERT ... Path to TLS certificate file.')
         print('ES_TLS_KEY ... Path to TLS key file.')
+        print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
         print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.')
         print('init configuration:')
         print('\tSHARDS ...  the number of shards per index in Elasticsearch (default {}).'.format(SHARDS))
@@ -46,18 +47,7 @@ def main():
         print('\tUNIT_COUNT ... count of UNITs (default {}).'.format(UNIT_COUNT))
         sys.exit(1)
 
-    username = os.getenv("ES_USERNAME")
-    password = os.getenv("ES_PASSWORD")
-
-    if username is not None and password is not None:
-        client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
-    elif str2bool(os.getenv("ES_TLS", 'false')):
-        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
-        context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
-        client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
-    else:
-        client = elasticsearch.Elasticsearch(sys.argv[2:])
-
+    client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
     prefix = os.getenv('INDEX_PREFIX', '')
     if prefix != '':
         prefix += '-'
@@ -107,7 +97,7 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
 def create_index_template(template, template_name):
     print('Creating index template {}'.format(template_name))
     headers = {'Content-Type': 'application/json'}
-    s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"))
+    s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))
     r = s.put(sys.argv[2] + '/_template/' + template_name, headers=headers, data=template)
     print(r.text)
     r.raise_for_status()
@@ -202,8 +192,12 @@ def empty_list(ilo, error_msg):
         sys.exit(0)
 
 
-def get_request_session(username, password, tls, ca, cert, key):
+def get_request_session(username, password, tls, ca, cert, key, skipHostVerify):
     session = requests.Session()
+    if ca is not None:
+        session.verify = ca
+    elif skipHostVerify:
+        session.verify = False
     if username is not None and password is not None:
         session.auth = HTTPBasicAuth(username, password)
     elif tls:
@@ -221,6 +215,22 @@ def get_version(client):
     return esVersion
 
 
+def create_client(username, password, tls, ca, cert, key, skipHostVerify):
+    context = ssl.create_default_context()
+    if ca is not None:
+        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
+    elif skipHostVerify:
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+    if username is not None and password is not None:
+        return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
+    elif tls:
+        context.load_cert_chain(certfile=cert, keyfile=key)
+        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
+    else:
+        return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
+
+
 if __name__ == "__main__":
     logging.getLogger().setLevel(logging.DEBUG)
     main()

scripts/travis/es-integration-test.sh

@@ -21,6 +21,7 @@ echo "Executing token propatagion test"
 make build-crossdock-ui-placeholder
 GOOS=linux make build-query
 
+make test-compile-es-scripts
 SPAN_STORAGE_TYPE=elasticsearch ./cmd/query/query-linux --es.server-urls=http://127.0.0.1:9200 --es.tls=false --es.version=7 --query.bearer-token-propagation=true &
 PID=$(echo $!)
 make token-propagation-integration-test