-
Notifications
You must be signed in to change notification settings - Fork 14
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
accomodated review comments from JHutar #106
Conversation
opl/investigator/config.py
Outdated
@@ -75,6 +75,10 @@ def load_config(conf, fp): | |||
assert not conf.history_es_server.endswith("/") | |||
conf.history_es_index = data["history"]["es_index"] | |||
conf.history_es_query = data["history"]["es_query"] | |||
if "es_server_user" in data["history"]: | |||
conf.es_server_user = data["history"]["es_server_user"] | |||
conf.es_server_pass = data["history"]["es_server_pass"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
conf.es_server_pass = data["history"]["es_server_pass"] | |
conf.es_server_pass_env_var = data["history"]["es_server_pass_env_var"] |
opl/investigator/config.py
Outdated
@@ -88,6 +92,10 @@ def load_config(conf, fp): | |||
conf.decisions_es_server = data["decisions"]["es_server"] | |||
assert not conf.decisions_es_server.endswith("/") | |||
conf.decisions_es_index = data["decisions"]["es_index"] | |||
if "es_server_user" in data["decisions"]: | |||
conf.decisions_es_server_user = data["decisions"]["es_server_user"] | |||
conf.decisions_es_server_pass = data["decisions"]["es_server_pass"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Again, just env var
opl/http.py
Outdated
@@ -12,6 +12,7 @@ def disable_insecure_request_warnings(disable_it): | |||
if disable_it: | |||
logging.debug("Disabling insecure request warnings") | |||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) | |||
session.verify = False |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I would not put it even here. I would leave it on user to explicitly do http.session.verify = False
or so. I admit this function name in unfortunate. Also this http module should have been a class or so, this sucks.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Or maybe just create function like this:
def insecure():
session.verify = False
disable_insecure_request_warnings(True)
else: | ||
response = opl.http.get(url, headers=headers, json=data) | ||
|
||
if response: |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why is this if response:
required?
opl/pass_or_fail.py
Outdated
@@ -114,12 +114,18 @@ def main(): | |||
if args.history_type == "csv": | |||
history = opl.investigator.csv_loader.load(args.history_file, args.sets) | |||
elif args.history_type == "elasticsearch": | |||
if hasattr(args, "es_server_user"): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if hasattr(args, "es_server_user"): | |
if hasattr(args, "es_server_verify"): |
opl/pass_or_fail.py
Outdated
@@ -200,8 +206,17 @@ def main(): | |||
if not args.dry_run: | |||
for d_type in args.decisions_type: | |||
if d_type == "elasticsearch": | |||
if hasattr(args, "es_server_user"): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if hasattr(args, "es_server_user"): | |
if hasattr(args, "es_server_verify"): |
opl/http.py
Outdated
@@ -8,6 +8,12 @@ | |||
session = requests.Session() | |||
|
|||
|
|||
def insecure(): | |||
session.verify = False | |||
logging.debug("Disabling insecure request warnings") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is not needed, it will be printed in that disable_insecure_request_warnings
. Or maybe you meant:
logging.debug("Disabling insecure request warnings") | |
logging.debug("Disabling SSL verifications for this session") |
opl/pass_or_fail.py
Outdated
history = opl.investigator.elasticsearch_loader.load( | ||
args.history_es_server, | ||
args.history_es_index, | ||
args.history_es_query, | ||
args.sets, | ||
es_server_user=getattr(args, "es_server_user", None), | ||
es_server_pass=getattr(args, "es_server_pass_env_var", None), |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Pls rename es_server_pass
as well to be consistent.
opl/pass_or_fail.py
Outdated
args.decisions_es_index, | ||
info_all, | ||
es_server_user=getattr(args, "decisions_es_server_user", None), | ||
es_server_pass=getattr(args, "decisions_es_server_pass_env_var", None), |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same here. Pls either use es_server_pass_env_var
or call os.environ.get(...)
here.
opl/pass_or_fail.py
Outdated
@@ -114,12 +114,18 @@ def main(): | |||
if args.history_type == "csv": | |||
history = opl.investigator.csv_loader.load(args.history_file, args.sets) | |||
elif args.history_type == "elasticsearch": | |||
if hasattr(args, "es_server_verify"): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I assume you need this, or am I missing something?
if hasattr(args, "es_server_verify"): | |
if hasattr(args, "es_server_verify") and args.es_server_verify == False: |
opl/pass_or_fail.py
Outdated
@@ -200,8 +206,17 @@ def main(): | |||
if not args.dry_run: | |||
for d_type in args.decisions_type: | |||
if d_type == "elasticsearch": | |||
if hasattr(args, "es_server_verify"): |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same as above.
opl/pass_or_fail.py
Outdated
args.decisions_es_index, | ||
info_all, | ||
es_server_user=getattr(args, "decisions_es_server_user", None), | ||
decisions_es_server_pass_env_var=getattr( |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Tihs is not a requirement, but to align with es_server_user
I would rename decisions_es_server_pass_env_var
to es_server_pass_env_var
. Or rename es_server_user
to decisions_es_server_user
opl/investigator/config.py
Outdated
if "es_server_user" in data["history"]: | ||
conf.es_server_user = data["history"]["es_server_user"] | ||
conf.es_server_pass_env_var = data["history"]["es_server_pass_env_var"] | ||
conf.es_server_verify = data["history"]["es_server_verify"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Pls make this option optional, defaulting to True
. Also this option is valid for setups without auth, so should be outside of this if
.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If I have removed If condition
then most of services fail at this line because investigator.conf file is not updated with username, password and verify flag. The plan is to start with front-end service and slowly propagate changes to other services, once it is complete then If condition
is not longer required.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I will keep it for now until all investigator.conf files are updated across the services.
Pls also add these new fields (commented out) to the sample config, maybe with some explanation. |
opl/investigator/config.py
Outdated
@@ -75,6 +75,9 @@ def load_config(conf, fp): | |||
assert not conf.history_es_server.endswith("/") | |||
conf.history_es_index = data["history"]["es_index"] | |||
conf.history_es_query = data["history"]["es_query"] | |||
conf.history_es_server_user = data["history"]["es_server_user"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hello! conf.history_es_server_user and pass needs to stay in that if because they will not be configured every time. Just conf.history_es_server_verify should be excluded from it because it can be used for unauthenticated ES.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Maybe something like this:
if conf.history_type == "elasticsearch":
conf.history_es_server = data["history"]["es_server"]
assert not conf.history_es_server.endswith("/")
conf.history_es_index = data["history"]["es_index"]
conf.history_es_query = data["history"]["es_query"]
if "es_server_user" in data["history"]:
conf.history_es_server_user = data["history"]["es_server_user"]
conf.history_es_server_pass_env_var = data["history"]["es_server_pass_env_var"]
if "es_server_verify" in data["history"]:
conf.history_es_server_verify = data["history"]["es_server_verify"]
else:
conf.history_es_server_verify = True
opl/investigator/config.py
Outdated
@@ -75,6 +75,9 @@ def load_config(conf, fp): | |||
assert not conf.history_es_server.endswith("/") | |||
conf.history_es_index = data["history"]["es_index"] | |||
conf.history_es_query = data["history"]["es_query"] | |||
conf.history_es_server_user = data["history"]["es_server_user"] | |||
conf.history_es_server_pass_env_var = data["history"]["es_server_pass_env_var"] | |||
conf.history_es_server_verify = data["history"]["es_server_verify"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Make it so that if es_server_verify
is not configured in the config, conf.history_es_server_verify
is set to true.
opl/investigator/config.py
Outdated
conf.decisions_es_server_pass_env_var = data["decisions"][ | ||
"decisions_es_server_pass_env_var" | ||
] | ||
conf.decisions_es_server_verify = data["decisions"]["es_server_verify"] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same here.
opl/pass_or_fail.py
Outdated
decisions_es_server_user=getattr(args, "decisions_es_server_user", None), | ||
es_server_pass_env_var=getattr( |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Either call both with decisions_*
or without.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Probably without, as you are doing that bit above in call to history = opl.investigator.elasticsearch_loader.load(...)
Pls also fix linter complaints and add commented example to https://github.com/redhat-performance/opl/blob/main/opl/investigator/sample_config.yaml |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thank you.
core/opl/shovel.py
Outdated
@@ -1,22 +1,42 @@ | |||
import argparse |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hello. This file should not be in this PR. Pls remove it.
No description provided.