Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support insecure TLS and only CA cert for Elasticsearch #1918

Merged
merged 4 commits into from
Nov 14, 2019
+53 −27
Merged

Support insecure TLS and only CA cert for Elasticsearch #1918

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
64226c5
Support insecure TLS and only CA cert in Elasticsearch scripts
pavolloffay Nov 12, 2019
f129762
nosec
pavolloffay Nov 13, 2019
2106a86
fmt
pavolloffay Nov 13, 2019
abf0df3
Test compilation of ES scripts and use CA in precedence of insecure
pavolloffay Nov 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pkg/es/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,14 +302,16 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp
} else {
httpTransport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
// #nosec G402
TLSClientConfig: &tls.Config{InsecureSkipVerify: c.TLS.SkipHostVerify},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when no TLS is involved at all, would non-nill TLSClientConfig cause issues?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should not godoc from http.Transport.TLSClientConfig

	// TLSClientConfig specifies the TLS configuration to use with
	// tls.Client.
	// If nil, the default configuration is used.
	// If non-nil, HTTP/2 support may not be enabled by default.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this godoc doesn't seem to clarify what happens to tls / no-tls. Perhaps it's just driven off the http scheme

}
if c.TLS.CaPath != "" {
ctls := &TLSConfig{CaPath: c.TLS.CaPath}
ca, err := ctls.loadCertificate()
if err != nil {
return nil, err
}
httpTransport.TLSClientConfig = &tls.Config{RootCAs: ca}
httpTransport.TLSClientConfig.RootCAs = ca
}

token := ""
Expand Down
28 changes: 16 additions & 12 deletions plugin/storage/es/esCleaner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,10 @@ def main():
print('ES_TLS_CA ... Path to TLS CA file.')
print('ES_TLS_CERT ... Path to TLS certificate file.')
print('ES_TLS_KEY ... Path to TLS key file.')
print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
sys.exit(1)

username = os.getenv("ES_USERNAME")
password = os.getenv("ES_PASSWORD")

if username is not None and password is not None:
client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
elif str2bool(os.getenv("ES_TLS", 'false')):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
client = elasticsearch.Elasticsearch(sys.argv[2:])

client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
ilo = curator.IndexList(client)
empty_list(ilo, 'Elasticsearch has no indices')

Expand Down Expand Up @@ -102,5 +92,19 @@ def str2bool(v):
return v.lower() in ('true', '1')


def create_client(username, password, tls, ca, cert, key, skipHostVerify):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) if ca is not None else ssl.create_default_context()
if skipHostVerify:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
if username is not None and password is not None:
return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
elif tls:
context.load_cert_chain(certfile=cert, keyfile=key)
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)


if __name__ == "__main__":
main()
36 changes: 22 additions & 14 deletions plugin/storage/es/esRollover.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def main():
print('ES_TLS_CA ... Path to TLS CA file.')
print('ES_TLS_CERT ... Path to TLS certificate file.')
print('ES_TLS_KEY ... Path to TLS key file.')
print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.')
print('init configuration:')
print('\tSHARDS ... the number of shards per index in Elasticsearch (default {}).'.format(SHARDS))
Expand All @@ -46,18 +47,7 @@ def main():
print('\tUNIT_COUNT ... count of UNITs (default {}).'.format(UNIT_COUNT))
sys.exit(1)

username = os.getenv("ES_USERNAME")
password = os.getenv("ES_PASSWORD")

if username is not None and password is not None:
client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password))
elif str2bool(os.getenv("ES_TLS", 'false')):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA"))
context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY"))
client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
client = elasticsearch.Elasticsearch(sys.argv[2:])

client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
prefix = os.getenv('INDEX_PREFIX', '')
if prefix != '':
prefix += '-'
Expand Down Expand Up @@ -107,7 +97,7 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
def create_index_template(template, template_name):
print('Creating index template {}'.format(template_name))
headers = {'Content-Type': 'application/json'}
s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"))
s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))
r = s.put(sys.argv[2] + '/_template/' + template_name, headers=headers, data=template)
print(r.text)
r.raise_for_status()
Expand Down Expand Up @@ -202,8 +192,12 @@ def empty_list(ilo, error_msg):
sys.exit(0)


def get_request_session(username, password, tls, ca, cert, key):
def get_request_session(username, password, tls, ca, cert, key, skipHostVerify):
session = requests.Session()
if skipHostVerify:
session.verify = False
if ca is not None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: should this be elif?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point I think they are mutually exclusive.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have changed it a bit. if CA is provided it will use that in the precedence of skip verify. I want to avoid accidentally using insecure connections.

session.verify = ca
if username is not None and password is not None:
session.auth = HTTPBasicAuth(username, password)
elif tls:
Expand All @@ -221,6 +215,20 @@ def get_version(client):
return esVersion


def create_client(username, password, tls, ca, cert, key, skipHostVerify):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) if ca is not None else ssl.create_default_context()
if skipHostVerify:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
if username is not None and password is not None:
return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
elif tls:
context.load_cert_chain(certfile=cert, keyfile=key)
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
else:
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)


if __name__ == "__main__":
logging.getLogger().setLevel(logging.DEBUG)
main()