diff --git a/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py b/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py index 67e748c87e9..c05c9360634 100644 --- a/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py +++ b/ambari-common/src/main/python/ambari_commons/ambari_metrics_helper.py @@ -65,18 +65,18 @@ def get_metric_collectors_from_properties_file(sink_name): try: hadoop_conf_dir = conf_select.get_hadoop_conf_dir() except Exception as e: - raise Exception("Couldn't define hadoop_conf_dir: {0}".format(e)) + raise Exception(f"Couldn't define hadoop_conf_dir: {e}") properties_filepath = os.path.join(hadoop_conf_dir, DEFAULT_METRICS2_PROPERTIES_FILE_NAME) if not os.path.exists(properties_filepath): - raise Exception("Properties file doesn't exist : {0}. Can't define metric collector hosts".format(properties_filepath)) + raise Exception(f"Properties file doesn't exist : {properties_filepath}. Can't define metric collector hosts") props = load_properties_from_file(properties_filepath) property_key = sink_name + DEFAULT_COLLECTOR_SUFFIX if property_key in props: return props.get(property_key) else: - raise Exception("Properties file doesn't contain {0}. Can't define metric collector hosts".format(property_key)) + raise Exception(f"Properties file doesn't contain {property_key}. Can't define metric collector hosts") def load_properties_from_file(filepath, sep='=', comment_char='#'): """ @@ -105,12 +105,11 @@ def create_ams_client(alert_id, ams_app_id, configurations, parameters): else: # ams-site/timeline.metrics.service.webapp.address is required if not METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY in configurations: - raise Exception('{0} is a required parameter for the script'.format(METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)) + raise Exception(f'{METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY} is a required parameter for the script') collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") if not _valid_collector_webapp_address(collector_webapp_address): - raise Exception('{0} value should be set as "fqdn_hostname:port", but set to {1}'.format( - METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY])) + raise Exception(f'{METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY} value should be set as "fqdn_hostname:port", but set to {configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY]}') ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) if not ams_collector_hosts: @@ -154,7 +153,7 @@ def load_metric(self, ams_metric, host_filter): break except Exception as exception: if logger.isEnabledFor(logging.DEBUG): - logger.exception("[Alert][{0}] Unable to retrieve metrics from AMS ({1}:{2}): {3}".format(self.alert_id, ams_collector_host, self.ams_collector_port, str(exception))) + logger.exception(f"[Alert][{self.alert_id}] Unable to retrieve metrics from AMS ({ams_collector_host}:{self.ams_collector_port}): {str(exception)}") if not http_code: raise Exception("Ambari metrics is not available: no response") @@ -199,29 +198,28 @@ def _load_metric(self, ams_collector_host, ams_metric, host_filter): data = response.read() except Exception as exception: if logger.isEnabledFor(logging.DEBUG): - logger.exception("[Alert][{0}] Unable to retrieve metrics from AMS: {1}".format(self.alert_id, str(exception))) + logger.exception(f"[Alert][{self.alert_id}] Unable to retrieve metrics from AMS: {str(exception)}") status = response.status if response else None return None, status finally: if logger.isEnabledFor(logging.DEBUG): - logger.debug(""" - AMS request parameters - {0} - AMS response - {1} - """.format(encoded_get_metrics_parameters, data)) + logger.debug(f""" + AMS request parameters - {encoded_get_metrics_parameters} + AMS response - {data} + """) # explicitly close the connection as we've seen python hold onto these if conn is not None: try: conn.close() except: - logger.debug("[Alert][{0}] Unable to close URL connection to {1}".format(self.alert_id, url)) + logger.debug(f"[Alert][{self.alert_id}] Unable to close URL connection to {url}") data_json = None try: data_json = json.loads(data) except Exception as exception: if logger.isEnabledFor(logging.DEBUG): - logger.exception("[Alert][{0}] Convert response to json failed or json doesn't contain needed data: {1}". - format(self.alert_id, str(exception))) + logger.exception(f"[Alert][{self.alert_id}] Convert response to json failed or json doesn't contain needed data: {str(exception)}") if not data_json: return None, response.status diff --git a/ambari-common/src/main/python/ambari_commons/credential_store_helper.py b/ambari-common/src/main/python/ambari_commons/credential_store_helper.py index a5fc1b94cc7..a95d184bbcc 100644 --- a/ambari-common/src/main/python/ambari_commons/credential_store_helper.py +++ b/ambari-common/src/main/python/ambari_commons/credential_store_helper.py @@ -48,7 +48,7 @@ def get_password_from_credential_store(alias, provider_path, cs_lib_path, java_h downloadjar(cs_lib_path, jdk_location) # Execute a get command on the CredentialUtil CLI to get the password for the specified alias - java_bin = '{java_home}/bin/java'.format(java_home=java_home) + java_bin = f'{java_home}/bin/java' cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias, '-provider', provider_path) cmd_result, std_out_msg = checked_call(cmd, quiet=True) std_out_lines = std_out_msg.split('\n') @@ -59,7 +59,7 @@ def list_aliases_from_credential_store(provider_path, cs_lib_path, java_home, jd downloadjar(cs_lib_path, jdk_location) # Execute a get command on the CredentialUtil CLI to list all the aliases - java_bin = '{java_home}/bin/java'.format(java_home=java_home) + java_bin = f'{java_home}/bin/java' cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'list', '-provider', provider_path) cmd_result, std_out_msg = checked_call(cmd, quiet=True) std_out_lines = std_out_msg.split('\n') @@ -70,7 +70,7 @@ def delete_alias_from_credential_store(alias, provider_path, cs_lib_path, java_h downloadjar(cs_lib_path, jdk_location) #Execute the creation and overwrite password - java_bin = '{java_home}/bin/java'.format(java_home=java_home) + java_bin = f'{java_home}/bin/java' cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'delete', alias, '-provider', provider_path, '-f') Execute(cmd) @@ -79,6 +79,6 @@ def create_password_in_credential_store(alias, provider_path, cs_lib_path, java_ downloadjar(cs_lib_path, jdk_location) #Execute the creation and overwrite password - java_bin = '{java_home}/bin/java'.format(java_home=java_home) + java_bin = f'{java_home}/bin/java' cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'create', alias, '-value', PasswordString(password) ,'-provider', provider_path, '-f') Execute(cmd) diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py index e58ca3520de..13f69c89bf2 100644 --- a/ambari-common/src/main/python/ambari_commons/firewall.py +++ b/ambari-common/src/main/python/ambari_commons/firewall.py @@ -88,7 +88,7 @@ def run_command(self): self.stdoutdata = out self.stderrdata = err except Exception as ex: - print_warning_msg("Unable to check firewall status: {0}".format(ex)) + print_warning_msg(f"Unable to check firewall status: {ex}") def check_firewall(self): try: @@ -188,7 +188,7 @@ def run_command(self): def check_result(self): if self.returncode != 0: - print_warning_msg("Unable to check firewall status:{0}".format(self.stderrdata)) + print_warning_msg(f"Unable to check firewall status:{self.stderrdata}") return False profiles_status = [i for i in self.stdoutdata.split("\n") if not i == ""] if "1" in profiles_status: @@ -200,7 +200,6 @@ def check_result(self): if profiles_status[2] == "1": enabled_profiles.append("PublicProfile") print_warning_msg( - "Following firewall profiles are enabled:{0}. Make sure that the firewall is properly configured.".format( - ",".join(enabled_profiles))) + f'Following firewall profiles are enabled:{",".join(enabled_profiles)}. Make sure that the firewall is properly configured.') return True return False diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py b/ambari-common/src/main/python/ambari_commons/inet_utils.py index 657ba39f6d7..0ce09c584d3 100644 --- a/ambari-common/src/main/python/ambari_commons/inet_utils.py +++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py @@ -60,35 +60,35 @@ def openurl(url, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *args, **kwargs): def download_file(link, destination, chunk_size=16 * 1024, progress_func = None): - print_info_msg("Downloading {0} to {1}".format(link, destination)) + print_info_msg(f"Downloading {link} to {destination}") if os.path.exists(destination): - print_warning_msg("File {0} already exists, assuming it was downloaded before".format(destination)) + print_warning_msg(f"File {destination} already exists, assuming it was downloaded before") return force_download_file(link, destination, chunk_size, progress_func = progress_func) def download_file_anyway(link, destination, chunk_size=16 * 1024, progress_func = None): - print_info_msg("Trying to download {0} to {1} with python lib [urllib2].".format(link, destination)) + print_info_msg(f"Trying to download {link} to {destination} with python lib [urllib2].") if os.path.exists(destination): - print_warning_msg("File {0} already exists, assuming it was downloaded before".format(destination)) + print_warning_msg(f"File {destination} already exists, assuming it was downloaded before") return try: force_download_file(link, destination, chunk_size, progress_func = progress_func) except: - print_error_msg("Download {0} with python lib [urllib2] failed with error: {1}".format(link, str(sys.exc_info()))) + print_error_msg(f"Download {link} with python lib [urllib2] failed with error: {str(sys.exc_info())}") if not os.path.exists(destination): - print("Trying to download {0} to {1} with [curl] command.".format(link, destination)) - #print_info_msg("Trying to download {0} to {1} with [curl] command.".format(link, destination)) + print(f"Trying to download {link} to {destination} with [curl] command.") + #print_info_msg(f"Trying to download {link} to {destination} with [curl] command.") curl_command = "curl --fail -k -o %s %s" % (destination, link) retcode, out, err = os_run_os_command(curl_command) if retcode != 0: - print_error_msg("Download file {0} with [curl] command failed with error: {1}".format(link, out + err)) + print_error_msg(f"Download file {link} with [curl] command failed with error: {out + err}") if not os.path.exists(destination): - print_error_msg("Unable to download file {0}!".format(link)) + print_error_msg(f"Unable to download file {link}!") print("ERROR: unable to donwload file %s!" % (link)) @@ -125,7 +125,7 @@ def find_range_components(meta): if len(range_comp1) > 1: range_comp2 = range_comp1[0].split(' ') #split away the "bytes" prefix if len(range_comp2) == 0: - raise FatalException(12, 'Malformed Content-Range response header: "{0}".'.format(hdr_range)) + raise FatalException(12, f'Malformed Content-Range response header: "{hdr_range}".') range_comp3 = range_comp2[1].split('-') seek_pos = int(range_comp3[0]) if range_comp1[1] != '*': #'*' == unknown length @@ -146,7 +146,7 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func if os.path.exists(destination) and not os.path.isfile(destination): #Directory specified as target? Must be a mistake. Bail out, don't assume anything. - err = 'Download target {0} is a directory.'.format(destination) + err = f'Download target {destination} is a directory.' raise FatalException(1, err) (dest_path, file_name) = os.path.split(destination) @@ -203,11 +203,11 @@ def force_download_file(link, destination, chunk_size = 16 * 1024, progress_func sys.stdout.write('\n') sys.stdout.flush() - print_info_msg("Finished downloading {0} to {1}".format(link, destination)) + print_info_msg(f"Finished downloading {link} to {destination}") downloaded_size = os.stat(temp_dest).st_size if downloaded_size != file_size: - err = 'Size of downloaded file {0} is {1} bytes, it is probably damaged or incomplete'.format(destination, downloaded_size) + err = f'Size of downloaded file {destination} is {downloaded_size} bytes, it is probably damaged or incomplete' raise FatalException(1, err) # when download is complete -> mv temp_dest destination diff --git a/ambari-common/src/main/python/ambari_commons/network.py b/ambari-common/src/main/python/ambari_commons/network.py index 1707c96caf3..dbcb0f1e3d8 100644 --- a/ambari-common/src/main/python/ambari_commons/network.py +++ b/ambari-common/src/main/python/ambari_commons/network.py @@ -54,8 +54,7 @@ def check_ssl_certificate_and_return_ssl_version(host, port, ca_certs, ssl_versi try: ssl.get_server_certificate((host, port), ssl_version=ssl_version, ca_certs=ca_certs) except ssl.SSLError as ssl_error: - raise Fail("Failed to verify the SSL certificate for https://{0}:{1} with CA certificate in {2}. Error : {3}" - .format(host, port, ca_certs, str(ssl_error))) + raise Fail(f"Failed to verify the SSL certificate for https://{host}:{port} with CA certificate in {ca_certs}. Error : {str(ssl_error)}") return ssl_version diff --git a/ambari-common/src/main/python/ambari_commons/os_check.py b/ambari-common/src/main/python/ambari_commons/os_check.py index 71e03aee405..9df654a8ca4 100644 --- a/ambari-common/src/main/python/ambari_commons/os_check.py +++ b/ambari-common/src/main/python/ambari_commons/os_check.py @@ -127,7 +127,7 @@ def initialize_data(cls): f.close() if JSON_OS_MAPPING not in json_data: - raise Exception("Invalid {0}".format(OSFAMILY_JSON_RESOURCE)) + raise Exception(f"Invalid {OSFAMILY_JSON_RESOURCE}") json_mapping_data = json_data[JSON_OS_MAPPING] @@ -188,7 +188,7 @@ def os_distribution(): release = REL_2012 elif minor == 3: release = REL_2012R2 - distribution = (release, "{0}.{1}".format(major,minor),"WindowsServer") + distribution = (release, f"{major}.{minor}","WindowsServer") else: # we are on unsupported desktop os distribution = ("", "", "") diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py index fe5d20ca988..7f3a0c92c25 100644 --- a/ambari-common/src/main/python/ambari_commons/os_utils.py +++ b/ambari-common/src/main/python/ambari_commons/os_utils.py @@ -88,8 +88,8 @@ def copy_file(src, dest_file): try: shutil.copyfile(src, dest_file) except Exception as e: - err = "Can not copy file {0} to {1} due to: {2} . Please check file " \ - "permissions and free disk space.".format(src, dest_file, e) + err = f"Can not copy file {src} to {dest_file} due to: {e} . Please check file " \ + "permissions and free disk space." raise FatalException(1, err) def copy_files(files, dest_dir): @@ -142,7 +142,7 @@ def is_service_exist(serviceName): def find_in_path(file): full_path = _search_file(file, os.environ["PATH"], os.pathsep) if full_path is None: - raise Exception("File {0} not found in PATH".format(file)) + raise Exception(f"File {file} not found in PATH") return full_path def extract_path_component(path, path_fragment): diff --git a/ambari-common/src/main/python/ambari_commons/os_windows.py b/ambari-common/src/main/python/ambari_commons/os_windows.py index 062353cf402..c52e05bc67a 100644 --- a/ambari-common/src/main/python/ambari_commons/os_windows.py +++ b/ambari-common/src/main/python/ambari_commons/os_windows.py @@ -360,7 +360,7 @@ def run_powershell_script(script_content): script_file = open(os.path.join(tmp_dir,random_filename+".ps1"),"w") script_file.write(script_content) script_file.close() - result = os_run_os_command("powershell -ExecutionPolicy unrestricted -File {0}".format(script_file.name)) + result = os_run_os_command(f"powershell -ExecutionPolicy unrestricted -File {script_file.name}") os.remove(script_file.name) return result @@ -378,8 +378,8 @@ def os_is_root(): ''' retcode, out, err = os_run_os_command(WHOAMI_GROUPS) if retcode != 0: - err_msg = "Unable to check the current user's group memberships. " \ - "Command {0} returned exit code {1} with message: {2}".format(WHOAMI_GROUPS, retcode, err) + err_msg = f"Unable to check the current user's group memberships. " \ + f"Command {WHOAMI_GROUPS} returned exit code {retcode} with message: {err}" print_warning_msg(err_msg) raise FatalException(retcode, err_msg) @@ -407,10 +407,10 @@ def os_set_file_permissions(file, mod, recursive, user): # print_warning_msg(WARN_MSG.format(command, file, err)) # rights = mod - # acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) + # acls_remove_cmd = f"icacls {file} /remove {user}" # retcode, out, err = os_run_os_command(acls_remove_cmd) # if retcode == 0: - # acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) + # acls_modify_cmd = f"icacls {file} /grant {user}:{rights}" # retcode, out, err = os_run_os_command(acls_modify_cmd) return retcode diff --git a/ambari-common/src/main/python/ambari_commons/repo_manager/__init__.py b/ambari-common/src/main/python/ambari_commons/repo_manager/__init__.py index 4f36f252ba2..689b96471f9 100644 --- a/ambari-common/src/main/python/ambari_commons/repo_manager/__init__.py +++ b/ambari-common/src/main/python/ambari_commons/repo_manager/__init__.py @@ -69,4 +69,4 @@ def get_new_instance(cls, os_family=None): if OSCheck.is_in_family(os_family, OSConst.WINSRV_FAMILY): return ChocoManager() - raise RuntimeError("Not able to create Repository Manager object for unsupported OS family {0}".format(os_family)) \ No newline at end of file + raise RuntimeError(f"Not able to create Repository Manager object for unsupported OS family {os_family}") \ No newline at end of file diff --git a/ambari-common/src/main/python/ambari_commons/repo_manager/apt_manager.py b/ambari-common/src/main/python/ambari_commons/repo_manager/apt_manager.py index 71ac3917188..ce3640ae76a 100644 --- a/ambari-common/src/main/python/ambari_commons/repo_manager/apt_manager.py +++ b/ambari-common/src/main/python/ambari_commons/repo_manager/apt_manager.py @@ -75,7 +75,7 @@ class AptManagerProperties(GenericManagerProperties): class AptManager(GenericManager): def get_installed_package_version(self, package_name): - r = shell.subprocess_executor("dpkg -s {0} | grep Version | awk '{{print $2}}'".format(package_name)) + r = shell.subprocess_executor(f"dpkg -s {package_name} | grep Version | awk '{{print $2}}'") return r.out.strip(os.linesep) @property @@ -184,7 +184,7 @@ def get_available_packages_in_repos(self, repos): repo_ids.append(self.transform_baseurl_to_repoid(repo.base_url)) if repos.feat.scoped: - Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + Logger.info(f"Looking for matching packages in the following repositories: {', '.join(repo_ids)}") for repo_id in repo_ids: for package in packages: if repo_id in package[2]: @@ -203,7 +203,7 @@ def get_available_packages_in_repos(self, repos): filtered_packages.append(package[0]) if len(filtered_packages) > 0: - Logger.info("Found packages for repo {}".format(str(filtered_packages))) + Logger.info(f"Found packages for repo {str(filtered_packages)}") return filtered_packages else: return [package[0] for package in packages] @@ -261,34 +261,34 @@ def install_package(self, name, context): if 'base' in context.use_repos: use_repos = set([v for k, v in context.use_repos.items() if k != 'base']) else: - cmd = cmd + ['-o', 'Dir::Etc::SourceList={0}'.format(self.properties.empty_file)] + cmd = cmd + ['-o', f'Dir::Etc::SourceList={self.properties.empty_file}'] use_repos = set(context.use_repos.values()) if use_repos: is_tmp_dir_created = True apt_sources_list_tmp_dir = tempfile.mkdtemp(suffix="-ambari-apt-sources-d") - Logger.info("Temporary sources directory was created: {}".format(apt_sources_list_tmp_dir)) + Logger.info(f"Temporary sources directory was created: {apt_sources_list_tmp_dir}") for repo in use_repos: new_sources_file = os.path.join(apt_sources_list_tmp_dir, repo + '.list') - Logger.info("Temporary sources file will be copied: {0}".format(new_sources_file)) + Logger.info(f"Temporary sources file will be copied: {new_sources_file}") sudo.copy(os.path.join(self.properties.repo_definition_location, repo + '.list'), new_sources_file) copied_sources_files.append(new_sources_file) - cmd = cmd + ['-o', 'Dir::Etc::SourceParts='.format(apt_sources_list_tmp_dir)] + cmd = cmd + ['-o', f'Dir::Etc::SourceParts={apt_sources_list_tmp_dir}'] cmd = cmd + [name] - Logger.info("Installing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Installing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context, env=self.properties.install_cmd_env) if is_tmp_dir_created: for temporary_sources_file in copied_sources_files: - Logger.info("Removing temporary sources file: {0}".format(temporary_sources_file)) + Logger.info(f"Removing temporary sources file: {temporary_sources_file}") os.remove(temporary_sources_file) if apt_sources_list_tmp_dir: - Logger.info("Removing temporary sources directory: {0}".format(apt_sources_list_tmp_dir)) + Logger.info(f"Removing temporary sources directory: {apt_sources_list_tmp_dir}") os.rmdir(apt_sources_list_tmp_dir) else: - Logger.info("Skipping installation of existing package {0}".format(name)) + Logger.info(f"Skipping installation of existing package {name}") @replace_underscores def upgrade_package(self, name, context): @@ -318,10 +318,10 @@ def remove_package(self, name, context, ignore_dependencies=False): raise ValueError("Installation command were executed with no package name passed") elif self._check_existence(name): cmd = self.properties.remove_cmd[context.log_output] + [name] - Logger.info("Removing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Removing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context) else: - Logger.info("Skipping removal of non-existing package {0}".format(name)) + Logger.info(f"Skipping removal of non-existing package {name}") @replace_underscores def _check_existence(self, name): diff --git a/ambari-common/src/main/python/ambari_commons/repo_manager/generic_manager.py b/ambari-common/src/main/python/ambari_commons/repo_manager/generic_manager.py index c429fad8df1..35ee484ebc0 100644 --- a/ambari-common/src/main/python/ambari_commons/repo_manager/generic_manager.py +++ b/ambari-common/src/main/python/ambari_commons/repo_manager/generic_manager.py @@ -248,5 +248,4 @@ def _executor_error_handler(self, command, error_log, exit_code): if isinstance(command, (list, tuple)): command = " ".join(command) - Logger.error("Command execution error: command = \"{0}\", exit code = {1}, stderr = {2}".format( - command, exit_code, "\n".join(error_log))) + Logger.error(f"Command execution error: command = \"{command}\", exit code = {exit_code}, stderr = " + "\n".join(error_log)) diff --git a/ambari-common/src/main/python/ambari_commons/repo_manager/yum_manager.py b/ambari-common/src/main/python/ambari_commons/repo_manager/yum_manager.py index 7a1e5faec9e..297e524d782 100644 --- a/ambari-common/src/main/python/ambari_commons/repo_manager/yum_manager.py +++ b/ambari-common/src/main/python/ambari_commons/repo_manager/yum_manager.py @@ -87,7 +87,7 @@ def get_available_packages_in_repos(self, repos): repo_ids = [repo.repo_id for repo in repos.items] if repos.feat.scoped: - Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + Logger.info(f"Looking for matching packages in the following repositories: {', '.join(repo_ids)}") else: Logger.info("Packages will be queried using all available repositories on the system.") @@ -101,7 +101,7 @@ def get_available_packages_in_repos(self, repos): if repos.feat.scoped: fallback_repo_ids = set(repo_ids) ^ self._build_repos_ids(repos) # no reason to scan the same repos again if fallback_repo_ids: - Logger.info("Adding fallback repositories: {0}".format(", ".join(fallback_repo_ids))) + Logger.info(f"Adding fallback repositories: {', '.join(fallback_repo_ids)}") for repo in fallback_repo_ids: available_packages.extend(self.available_packages(repo_filter=repo)) @@ -190,8 +190,7 @@ def verify_dependencies(self): pattern = re.compile("has missing requires|Error:") if ret.code or (ret.out and pattern.search(ret.out)): - err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '{0}' returned {1}. {2}".format( - self.properties.verify_dependency_cmd, ret.code, ret.out)) + err_msg = Logger.filter_text(f"Failed to verify package dependencies. Execution of '{self.properties.verify_dependency_cmd}' returned {ret.code}. {ret.out}") Logger.error(err_msg) return False @@ -216,10 +215,10 @@ def install_package(self, name, context): disable_repo_option = '--disablerepo=' + "*" if not context.skip_repos or len(context.skip_repos) == 0 else ','.join(context.skip_repos) cmd = cmd + [disable_repo_option, enable_repo_option] cmd = cmd + [name] - Logger.info("Installing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Installing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context) else: - Logger.info("Skipping installation of existing package {0}".format(name)) + Logger.info(f"Skipping installation of existing package {name}") def upgrade_package(self, name, context): """ @@ -250,10 +249,10 @@ def remove_package(self, name, context, ignore_dependencies=False): cmd = self.properties.remove_without_dependencies_cmd + [name] else: cmd = self.properties.remove_cmd[context.log_output] + [name] - Logger.info("Removing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Removing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context) else: - Logger.info("Skipping removal of non-existing package {0}".format(name)) + Logger.info(f"Skipping removal of non-existing package {name}") def _check_existence(self, name): """ @@ -405,13 +404,9 @@ def check_uncompleted_transactions(self): transactions = list(self.uncomplete_transactions()) if len(transactions) > 0: - Logger.info("Yum non-completed transactions check failed, found {0} non-completed transaction(s):".format(len(transactions))) + Logger.info(f"Yum non-completed transactions check failed, found {len(transactions)} non-completed transaction(s):") for tr in transactions: - Logger.info("[{0}] Packages broken: {1}; Packages not-installed {2}".format( - tr.transaction_id, - ", ".join(tr.pkgs_done), - ", ".join(tr.pkgs_aborted) - )) + Logger.info(f"[{tr.transaction_id}] Packages broken: {', '.join(tr.pkgs_done)}; Packages not-installed {', '.join(tr.pkgs_aborted)}") return True diff --git a/ambari-common/src/main/python/ambari_commons/repo_manager/zypper_manager.py b/ambari-common/src/main/python/ambari_commons/repo_manager/zypper_manager.py index a99c63f32e7..e7d9f2cbea4 100644 --- a/ambari-common/src/main/python/ambari_commons/repo_manager/zypper_manager.py +++ b/ambari-common/src/main/python/ambari_commons/repo_manager/zypper_manager.py @@ -76,7 +76,7 @@ def get_available_packages_in_repos(self, repos): # as result repository would be matched if it contains package with same meta info if repos.feat.scoped: - Logger.info("Looking for matching packages in the following repositories: {0}".format(", ".join(repo_ids))) + Logger.info(f"Looking for matching packages in the following repositories: {', '.join(repo_ids)}") for repo in repo_ids: available_packages.extend(self.all_packages(repo_filter=repo)) else: @@ -166,8 +166,8 @@ def verify_dependencies(self): pattern = re.compile("\d+ new package(s)? to install") if r.code or (r.out and pattern.search(r.out)): - err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '{0}' returned {1}. {2}".format( - self.properties.verify_dependency_cmd, r.code, r.out)) + err_msg = Logger.filter_text(f"Failed to verify package dependencies. Execution of '{self.properties.verify_dependency_cmd}' " + f"returned {r.code}. {r.out}") Logger.error(err_msg) return False @@ -199,11 +199,11 @@ def install_package(self, name, context): cmd = cmd + use_repos_options cmd = cmd + [name] - Logger.info("Installing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Installing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context) else: - Logger.info("Skipping installation of existing package {0}".format(name)) + Logger.info(f"Skipping installation of existing package {name}") def upgrade_package(self, name, context): """ @@ -231,10 +231,10 @@ def remove_package(self, name, context, ignore_dependencies=False): raise ValueError("Installation command were executed with no package name passed") elif self._check_existence(name): cmd = self.properties.remove_cmd[context.log_output] + [name] - Logger.info("Removing package {0} ('{1}')".format(name, shell.string_cmd_from_args_list(cmd))) + Logger.info(f"Removing package {name} ('{shell.string_cmd_from_args_list(cmd)}')") shell.repository_manager_executor(cmd, self.properties, context) else: - Logger.info("Skipping removal of non-existing package {0}".format(name)) + Logger.info(f"Skipping removal of non-existing package {name}") def get_active_base_repos(self): enabled_repos = [] diff --git a/ambari-common/src/main/python/ambari_commons/shell.py b/ambari-common/src/main/python/ambari_commons/shell.py index 5ec162982dc..dddebf251c1 100644 --- a/ambari-common/src/main/python/ambari_commons/shell.py +++ b/ambari-common/src/main/python/ambari_commons/shell.py @@ -229,7 +229,7 @@ def quote_bash_args(command): return "''" if not isinstance(command, str): - raise ValueError("Command should be a list of strings, found '{0}' in command list elements".format(str(command))) + raise ValueError(f"Command should be a list of strings, found '{str(command)}' in command list elements") valid = set(string.ascii_letters + string.digits + '@%_-+=:,./') for char in command: @@ -277,9 +277,9 @@ def _geometry_helper(): # no way to execute shell command with bash pipes under sudo, it is fully dev responsibility if not is_under_root() and not isinstance(command, str): - command = "{0} -H -E {1}".format(AMBARI_SUDO_BINARY, string_cmd_from_args_list(command)) # core.shell.as_sudo + command = f"{AMBARI_SUDO_BINARY} -H -E {string_cmd_from_args_list(command)}" # core.shell.as_sudo elif not is_under_root() and isinstance(command, str): - _logger.debug("Warning, command \"{0}\" doesn't support sudo appending".format(command)) + _logger.debug(f"Warning, command \"{command}\" doesn't support sudo appending") is_shell = not isinstance(command, (list, tuple)) environ = copy.deepcopy(os.environ) if env: @@ -507,7 +507,7 @@ def error_handler(command, error_log, exit_code): elif strategy == ReaderStrategy.BufferedChunks: yield chunks_reader(cmd, kill_timer) else: - raise TypeError("Unknown reader strategy selected: {0}".format(strategy)) + raise TypeError(f"Unknown reader strategy selected: {strategy}") _exit_code = cmd.poll() if _exit_code is None: @@ -517,7 +517,7 @@ def error_handler(command, error_log, exit_code): error_callback(command, cmd.stderr.readlines(), cmd.returncode) except Exception as e: if not silent: - _logger.error("Exception during command '{0}' execution: {1}".format(command, str(e))) + _logger.error(f"Exception during command '{command}' execution: {str(e)}") if error_callback: error_callback(command, [str(e)], -1) @@ -664,16 +664,14 @@ def kill_process_with_children(base_pid): wait_for_process_list_kill(pids_to_kill) still_existing_pids = get_existing_pids(pids_to_kill) if still_existing_pids: - _logger.warning("These PIDs {0} did not respond to {1} signal. Detailed commands list:\n {2}".format( - ", ".join([str(i) for i in still_existing_pids]), - sig_name, - "\n".join([i[2] for i in full_child_pids if i[0] in still_existing_pids]) - )) + _logger.warning(f'These PIDs {", ".join([str(i) for i in still_existing_pids])} ' + f'did not respond to {sig_name} signal. Detailed commands list:\n ' + + "\n".join([i[2] for i in full_child_pids if i[0] in still_existing_pids])) if get_existing_pids(all_child_pids) and error_log: # we're unable to kill all requested PIDs _logger.warn("Process termination error log:\n") for error_item in error_log: - _logger.warn("PID: {0}, Process: {1}, Exception message: {2}".format(*error_item)) + _logger.warn(f"PID: {error_item[0]}, Process: {error_item[1]}, Exception message: {error_item[2]}") def __handle_retries(cmd, repo_properties, context, call_result, is_first_time, is_last_time): @@ -693,7 +691,7 @@ def __handle_retries(cmd, repo_properties, context, call_result, is_first_time, err_log_msg = None if context.retry_on_locked and repo_properties.locked_output and repo_properties.locked_output in out: - err_log_msg = __PACKAGE_MANAGER_LOCK_ACQUIRED_MSG.format(context.retry_sleep, call_result.out) + err_log_msg =f"{ __PACKAGE_MANAGER_LOCK_ACQUIRED_MSG} {context.retry_sleep} {call_result.out}" elif context.retry_on_repo_unavailability and repo_properties.repo_error: for err in repo_properties.repo_error: if err in call_result.all_out: @@ -745,9 +743,8 @@ def repository_manager_executor(cmd, repo_properties, context=RepoCallContext(), should_stop_retries = __handle_retries(cmd, repo_properties, context, call_result, is_first_time, is_last_time) if (is_last_time or should_stop_retries) and call_result.code != 0: - message = "Failed to execute command '{0}', exited with code '{1}', message: '{2}'".format( - cmd if not isinstance(cmd, (list, tuple)) else " ".join(cmd), - call_result.code, call_result.error) + message = (f"Failed to execute command '{cmd if not isinstance(cmd, (list, tuple)) else ' '.join(cmd)}', " + f"exited with code '{call_result.code}', message: '{call_result.error}'") if context.ignore_errors: _logger.warning(message) diff --git a/ambari-common/src/main/python/ambari_commons/urllib_handlers.py b/ambari-common/src/main/python/ambari_commons/urllib_handlers.py index bb840a67365..84f3202dd5f 100644 --- a/ambari-common/src/main/python/ambari_commons/urllib_handlers.py +++ b/ambari-common/src/main/python/ambari_commons/urllib_handlers.py @@ -91,7 +91,7 @@ def http_response(self, request, response): redirect_url = redirect_url_key_value_pair[equals_index+1:] if key.strip().lower() != REFRESH_HEADER_URL_KEY: - logger.warning("Unable to parse refresh header {0}".format(refresh_header)) + logger.warning(f"Unable to parse refresh header {refresh_header}") return response # extract out just host:port @@ -110,8 +110,7 @@ def http_response(self, request, response): # follow the new new and return the response return self.parent.open(redirect_url) except Exception as exception: - logger.error("Unable to follow refresh header {0}. {1}".format( - refresh_header, str(exception))) + logger.error(f"Unable to follow refresh header {refresh_header}. {str(exception)}") # return the original response return response diff --git a/ambari-common/src/main/python/ambari_simplejson/c_extension.py b/ambari-common/src/main/python/ambari_simplejson/c_extension.py index 00dc6b3188e..6fdb51f4886 100644 --- a/ambari-common/src/main/python/ambari_simplejson/c_extension.py +++ b/ambari-common/src/main/python/ambari_simplejson/c_extension.py @@ -45,7 +45,7 @@ def get(): for _import_path in _import_paths: try: - return __import__("{}.{}".format(_path, _import_path), fromlist=["_speedups"])._speedups + return __import__(f"{_path}.{_import_path}", fromlist=["_speedups"])._speedups except (ImportError, AttributeError): pass diff --git a/ambari-common/src/main/python/ambari_stomp/adapter/websocket.py b/ambari-common/src/main/python/ambari_stomp/adapter/websocket.py index 371f670884f..2495e5ca057 100644 --- a/ambari-common/src/main/python/ambari_stomp/adapter/websocket.py +++ b/ambari-common/src/main/python/ambari_stomp/adapter/websocket.py @@ -100,7 +100,7 @@ def receive(self): try: msg = self.ws.receive() msg = str(msg) if msg is not None else msg - logger.debug("Incoming STOMP message:\n<<< {0}".format(msg)) + logger.debug(f"Incoming STOMP message:\n<<< {msg}") return msg except: # exceptions from this method are hidden by the framework so implementing logging by ourselves diff --git a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py index 08fba91cf30..c59a32f6123 100644 --- a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py +++ b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py @@ -282,7 +282,7 @@ def process_upgrade_xml(file_path, target_version, config_data, stack_version_ch new_file_path = file_path if target_version in stack_version_changes: new_file_path = os.path.join(os.path.dirname(file_path), - 'upgrade-{0}.xml'.format(stack_version_changes[target_version])) + f'upgrade-{stack_version_changes[target_version]}.xml') os.rename(file_path, new_file_path) return new_file_path diff --git a/ambari-common/src/main/python/resource_management/core/environment.py b/ambari-common/src/main/python/resource_management/core/environment.py index 02f2e5db52b..701af88d39b 100644 --- a/ambari-common/src/main/python/resource_management/core/environment.py +++ b/ambari-common/src/main/python/resource_management/core/environment.py @@ -114,7 +114,7 @@ def set_params(self, arg): not hasattr(value, '__file__'): self.config.params[variable] = value except Exception as e: - Logger.info("Skipping param: {0}, due to {1}".format(variable, e)) + Logger.info(f"Skipping param: {variable}, due to {e}") def run_action(self, resource, action): provider_class = find_provider(self, resource.__class__.__name__, @@ -150,12 +150,12 @@ def run(self): if resource.not_if is not None and self._check_condition( resource.not_if): - Logger.info("Skipping {0} due to not_if".format(resource)) + Logger.info(f"Skipping {resource} due to not_if") continue if resource.only_if is not None and not self._check_condition( resource.only_if): - Logger.info("Skipping {0} due to only_if".format(resource)) + Logger.info(f"Skipping {resource} due to only_if") continue for action in resource.action: @@ -165,7 +165,7 @@ def run(self): try: self.run_action(resource, action) except Exception as ex: - Logger.info("Skipping failure of {0} due to ignore_failures. Failure reason: {1}".format(resource, ex)) + Logger.info(f"Skipping failure of {resource} due to ignore_failures. Failure reason: {ex}") pass # Run delayed actions diff --git a/ambari-common/src/main/python/resource_management/core/global_lock.py b/ambari-common/src/main/python/resource_management/core/global_lock.py index 6933c55d152..9906bcdc7fa 100644 --- a/ambari-common/src/main/python/resource_management/core/global_lock.py +++ b/ambari-common/src/main/python/resource_management/core/global_lock.py @@ -41,6 +41,6 @@ def get_lock(lock_type): :rtype: threading.RLock() """ if lock_type not in __GLOBAL_LOCKS: - raise Fail("There is no global lock associated with {0}".format(str(lock_type))) + raise Fail(f"There is no global lock associated with {str(lock_type)}") return __GLOBAL_LOCKS[lock_type] diff --git a/ambari-common/src/main/python/resource_management/core/logger.py b/ambari-common/src/main/python/resource_management/core/logger.py index 922b587f025..79fc76094be 100644 --- a/ambari-common/src/main/python/resource_management/core/logger.py +++ b/ambari-common/src/main/python/resource_management/core/logger.py @@ -176,9 +176,9 @@ def get_function_repr(name, arguments, resource=None): else: val = repr(y) - arguments_str += "'{0}': {1}, ".format(x, val) + arguments_str += f"'{x}': {val}, " if arguments_str: arguments_str = arguments_str[:-2] - return str("{0} {{{1}}}").format(name, arguments_str) + return str(f"{name} {{{arguments_str}}}") diff --git a/ambari-common/src/main/python/resource_management/core/providers/system.py b/ambari-common/src/main/python/resource_management/core/providers/system.py index 6f3ba875d6b..236f4c12b33 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/system.py @@ -50,7 +50,7 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_own try: _user_entity = pwd.getpwnam(user) except KeyError: - raise Fail("User '{0}' doesn't exist".format(user)) + raise Fail(f"User '{user}' doesn't exist") if stat.st_uid != _user_entity.pw_uid: user_entity = _user_entity @@ -61,7 +61,7 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_own try: _group_entity = grp.getgrnam(group) except KeyError: - raise Fail("Group '{0}' doesn't exist".format(group)) + raise Fail(f"Group '{group}' doesn't exist") if stat.st_gid != _group_entity.gr_gid: group_entity = _group_entity @@ -78,7 +78,7 @@ def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_own if not isinstance(recursive_mode_flags, dict): raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)") - regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" ) + regexp_to_match = f"^({'[ugoa]+[+=-][rwx]+'},)*({'[ugoa]+[+=-][rwx]+'})$" for key, flags in recursive_mode_flags.items(): if key != 'd' and key != 'f': raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key))) @@ -189,7 +189,7 @@ def action_create(self): path = os.path.join(os.path.dirname(prev_path), path) if path != self.resource.path: - Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path)) + Logger.info(f"Following the link {self.resource.path} to {path} to create the directory") if self.resource.create_parents: sudo.makedirs(path, self.resource.mode or 0o755) diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/service.py b/ambari-common/src/main/python/resource_management/core/providers/windows/service.py index e9b8ebd1358..b5bf4cb02ea 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/windows/service.py +++ b/ambari-common/src/main/python/resource_management/core/providers/windows/service.py @@ -36,7 +36,7 @@ def safe_open_scmanager(): try: _schSCManager = win32service.OpenSCManager(None, None, win32service.SC_MANAGER_ALL_ACCESS) except win32api.error as details: - raise Fail("Error opening Service Control Manager on the local machine: {0}".format(details.winerror)) + raise Fail(f"Error opening Service Control Manager on the local machine: {details.winerror}") return _schSCManager @@ -46,9 +46,9 @@ def safe_open_service(hSCM, service_name): win32service.SERVICE_ALL_ACCESS) except win32api.error as details: if details.winerror == winerror.ERROR_SERVICE_DOES_NOT_EXIST: - err_msg = "Invalid service name: {0}".format(service_name) + err_msg = f"Invalid service name: {service_name}" else: - err_msg = "Error configuring service {0}: {1}".format(service_name, details.winerror) + err_msg = f"Error configuring service {service_name}: {details.winerror}" raise Fail(err_msg) return hSvc @@ -97,7 +97,7 @@ def enable(self): None) win32service.CloseServiceHandle(hSvc) except win32api.error as details: - raise Fail("Error enabling service {0}: {1}".format(self.resource.service_name, details.winerror)) + raise Fail(f"Error enabling service {self.resource.service_name}: {details.winerror}") finally: win32service.CloseServiceHandle(hSCM) @@ -147,7 +147,7 @@ def action_install(self): win32service.CloseServiceHandle(hSvc) except win32api.error as details: - raise Fail("Error creating service {0}: {1}".format(self.resource.service_name, details.winerror)) + raise Fail(f"Error creating service {self.resource.service_name}: {details.winerror}") finally: win32service.CloseServiceHandle(hSCM) @@ -177,7 +177,7 @@ def action_configure(self): except NotImplementedError: pass ## ChangeServiceConfig2 and description do not exist on NT except win32api.error as details: - raise Fail("Error configuring service {0}: {1}".format(self.resource.service_name, details.winerror)) + raise Fail(f"Error configuring service {self.resource.service_name}: {details.winerror}") finally: win32service.CloseServiceHandle(hSvc) finally: @@ -204,7 +204,7 @@ def action_change_user(self): self.resource.password, None) except win32api.error as details: - raise Fail("Error changing user for service {0}: {1}".format(self.resource.service_name, details.winerror)) + raise Fail(f"Error changing user for service {self.resource.service_name}: {details.winerror}") finally: win32service.CloseServiceHandle(hSvc) finally: @@ -222,7 +222,7 @@ def action_uninstall(self): # Nothing to do return else: - raise Fail("Error removing service {0}: {1}".format(self.resource.service_name, details.winerror)) + raise Fail(f"Error removing service {self.resource.service_name}: {details.winerror}") try: win32service.DeleteService(hSvc) @@ -241,8 +241,8 @@ def _fix_start_type(self): win32service.SERVICE_AUTO_START, win32service.SERVICE_DISABLED, win32service.SERVICE_DEMAND_START]): - Logger.warning("Invalid service start type specified: service='{0}', start type='{1}'. Ignoring.".format( - self.resource.service_name, str(self.resource.start_type))) + Logger.warning(f"Invalid service start type specified: service='{self.resource.service_name}'," + f" start type='{str(self.resource.start_type)}'. Ignoring.") self.resource.start_type = win32service.SERVICE_NO_CHANGE def _fix_user_name(self): diff --git a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py index a3b7ef95047..8e4b4ee20e7 100644 --- a/ambari-common/src/main/python/resource_management/core/providers/windows/system.py +++ b/ambari-common/src/main/python/resource_management/core/providers/windows/system.py @@ -142,7 +142,7 @@ def QueryPrivilegeState(hToken, priv): for (id, attr) in privList: if id == privId: privState = attr - Logger.debug('Privilege state: {0}={1} ({2}) Enabled={3}'.format(privId, priv, LookupPrivilegeDisplayName(None, priv), privState)) + Logger.debug(f'Privilege state: {privId}={priv} ({LookupPrivilegeDisplayName(None, priv)}) Enabled={privState}') return privState # Execute command. As windows hdp stack heavily relies on proper environment it is better to reload fresh environment @@ -186,7 +186,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= if not ok: raise Exception("Unable to create StdErr for child process") - Logger.debug("Redirecting stdout to '{0}', stderr to '{1}'".format(out_file.name, err_file.name)) + Logger.debug(f"Redirecting stdout to '{out_file.name}', stderr to '{err_file.name}'") si.dwFlags = win32con.STARTF_USESTDHANDLES si.lpDesktop = "" @@ -225,14 +225,14 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish= # see msdn Icacls doc for rights def _set_file_acl(file, user, rights): - acls_modify_cmd = "icacls {0} /grant {1}:{2}".format(file, user, rights) - acls_remove_cmd = "icacls {0} /remove {1}".format(file, user) + acls_modify_cmd = f"icacls {file} /grant {user}:{rights}" + acls_remove_cmd = f"icacls {file} /remove {user}" code, out, err = _call_command(acls_remove_cmd) if code != 0: - raise Fail("Can not remove rights for path {0} and user {1}".format(file, user)) + raise Fail(f"Can not remove rights for path {file} and user {user}") code, out, err = _call_command(acls_modify_cmd) if code != 0: - raise Fail("Can not set rights {0} for path {1} and user {2}".format(file, user)) + raise Fail(f"Can not set rights for path {file} and user {user}") else: return diff --git a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py index 40944a317c1..7e77da64d79 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py +++ b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py @@ -52,9 +52,9 @@ def delete_node(self, znode, tries=3): tries=tries) def _acl_command(self, znode, acl): - return "{0} -Djava.security.auth.login.config={1} -jar {2} -connection-string {3} -znode {4} -acl {5}".format( \ - self.java_exec, self.jaas_file, self.zkmigrator_jar, self.zk_host, znode, acl) + return (f"{self.java_exec} -Djava.security.auth.login.config={self.jaas_file} -jar {self.zkmigrator_jar}" + f" -connection-string {self.zk_host} -znode {znode} -acl {acl}") def _delete_command(self, znode): - return "{0} -Djava.security.auth.login.config={1} -jar {2} -connection-string {3} -znode {4} -delete".format( \ - self.java_exec, self.jaas_file, self.zkmigrator_jar, self.zk_host, znode) + return (f"{self.java_exec} -Djava.security.auth.login.config={self.jaas_file} -jar {self.zkmigrator_jar}" + f" -connection-string {self.zk_host} -znode {znode} -delete") diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py index e24df14e832..e0688699ae9 100644 --- a/ambari-common/src/main/python/resource_management/core/shell.py +++ b/ambari-common/src/main/python/resource_management/core/shell.py @@ -58,7 +58,7 @@ def inner(command, **kwargs): if quiet == False or (quiet == None and not is_internal_call): command_repr = Logger._get_resource_name_repr(command) - log_msg = Logger.get_function_repr("{0}[{1}]".format(function.__name__, command_repr), kwargs) + log_msg = Logger.get_function_repr(f"{function.__name__}[{command_repr}]", kwargs) Logger.info(log_msg) # logoutput=False - never log @@ -73,7 +73,7 @@ def inner(command, **kwargs): result = function(command, **kwargs) if quiet == False or (quiet == None and not is_internal_call): - log_msg = "{0} returned {1}".format(function.__name__, result) + log_msg = f"{function.__name__} returned {result}" Logger.info(log_msg) return result @@ -85,7 +85,7 @@ def preexec_fn(): try: os.setpgid(processId, processId) except: - Logger.exception('setpgid({0}, {0}) failed'.format(processId)) + Logger.exception(f'setpgid({processId}, {processId}) failed') raise @log_function_call @@ -152,7 +152,7 @@ def _call_wrapper(command, **kwargs): break except ExecuteTimeoutException as ex: if on_timeout: - Logger.info("Executing '{0}'. Reason: {1}".format(on_timeout, str(ex))) + Logger.info(f"Executing '{on_timeout}'. Reason: {str(ex)}") result = checked_call(on_timeout) else: raise @@ -160,7 +160,7 @@ def _call_wrapper(command, **kwargs): if is_last_try: # last try raise else: - Logger.info("Retrying after {0} seconds. Reason: {1}".format(try_sleep, str(ex))) + Logger.info(f"Retrying after {try_sleep} seconds. Reason: {str(ex)}") time.sleep(try_sleep) return result @@ -286,7 +286,7 @@ def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE try: on_new_line(line, out_fd == proc.stderr) except Exception: - err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc()) + err_msg = f"Caused by on_new_line function failed with exception for input argument '{line}':\n{traceback.format_exc()}" raise Fail(err_msg) if logoutput: @@ -310,13 +310,13 @@ def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE timer.cancel() # timeout occurred else: - err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format(command, timeout) + err_msg = f"Execution of '{command}' was killed due timeout after {timeout} seconds" raise ExecuteTimeoutException(err_msg) code = proc.returncode if throw_on_failure and not code in returns: - err_msg = Logger.filter_text("Execution of '{0}' returned {1}. {2}".format(command_alias, code, all_output)) + err_msg = Logger.filter_text(f"Execution of '{command_alias}' returned {code}. {all_output}") raise ExecutionFailed(err_msg, code, out, err) # if separate stderr is enabled (by default it's redirected to out) @@ -341,19 +341,19 @@ def as_sudo(command, env=None, auto_escape=True): # # In that case while passing string, # any bash symbols eventually added to command like && || ; < > | << >> would cause problems. - err_msg = Logger.filter_text("String command '{0}' cannot be run as sudo. Please supply the command as a tuple of arguments".format(command)) + err_msg = Logger.filter_text(f"String command '{command}' cannot be run as sudo. Please supply the command as a tuple of arguments") raise Fail(err_msg) env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER - return "{0} {1} -H -E {2}".format(_get_sudo_binary(), env, command) + return f"{_get_sudo_binary()} {env} -H -E {command}" def as_user(command, user, env=None, auto_escape=True): if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command, auto_escape=auto_escape) - export_env = "export {0} ; ".format(_get_environment_str(_add_current_path_to_env(env))) if env else EXPORT_PLACEHOLDER - return "{0} su {1} -l -s /bin/bash -c {2}".format(_get_sudo_binary(), user, quote_bash_args(export_env + command)) + export_env = f"export {_get_environment_str(_add_current_path_to_env(env))} ; " if env else EXPORT_PLACEHOLDER + return f"{_get_sudo_binary()} su {user} -l -s /bin/bash -c {quote_bash_args(export_env + command)}" def quote_bash_args(command): @@ -361,7 +361,7 @@ def quote_bash_args(command): return "''" if not isinstance(command, str): - raise Fail("Command should be a list of strings, found '{0}' in command list elements".format(str(command))) + raise Fail(f"Command should be a list of strings, found '{str(command)}' in command list elements") valid = set(string.ascii_letters + string.digits + '@%_-+=:,./') for char in command: @@ -385,7 +385,7 @@ def _get_sudo_binary(): return AMBARI_SUDO_BINARY def _get_environment_str(env): - return reduce(lambda str,x: '{0} {1}={2}'.format(str,x,quote_bash_args(env[x])), env, '') + return reduce(lambda str,x: f'{str} {x}={quote_bash_args(env[x])}', env, '') def string_cmd_from_args_list(command, auto_escape=True): if auto_escape: diff --git a/ambari-common/src/main/python/resource_management/core/signal_utils.py b/ambari-common/src/main/python/resource_management/core/signal_utils.py index 901e48c41c1..a83878a8660 100644 --- a/ambari-common/src/main/python/resource_management/core/signal_utils.py +++ b/ambari-common/src/main/python/resource_management/core/signal_utils.py @@ -47,7 +47,7 @@ def terminate_process(proc, terminate_strategy): elif terminate_strategy == TerminateStrategy.KILL_PROCESS_TREE: kill_process_tree(proc) else: - raise Fail("Invalid timeout_kill_strategy = '{0}'. Use TerminateStrategy class constants as a value.".format(terminate_strategy)) + raise Fail(f"Invalid timeout_kill_strategy = '{terminate_strategy}'. Use TerminateStrategy class constants as a value.") def killpg_gracefully(proc, timeout=GRACEFUL_PG_KILL_TIMEOUT_SECONDS): """ @@ -67,7 +67,7 @@ def killpg_gracefully(proc, timeout=GRACEFUL_PG_KILL_TIMEOUT_SECONDS): break time.sleep(0.1) else: - Logger.info("Cannot gracefully kill process group {0}. Resorting to SIGKILL.".format(pgid)) + Logger.info(f"Cannot gracefully kill process group {pgid}. Resorting to SIGKILL.") sudo.kill(-pgid, signal.SIGKILL.value) proc.wait() # catch race condition if proc already dead @@ -86,7 +86,7 @@ def terminate_parent_process(proc): def kill_process_tree(proc): from resource_management.core import shell current_directory = os.path.dirname(os.path.abspath(__file__)) - kill_tree_script = "{0}/files/killtree.sh".format(current_directory) + kill_tree_script = f"{current_directory}/files/killtree.sh" if proc.poll() == None: shell.checked_call(["bash", kill_tree_script, str(proc.pid), str(signal.SIGKILL)]) \ No newline at end of file diff --git a/ambari-common/src/main/python/resource_management/core/source.py b/ambari-common/src/main/python/resource_management/core/source.py index d652e20f35f..69f25e2b47c 100644 --- a/ambari-common/src/main/python/resource_management/core/source.py +++ b/ambari-common/src/main/python/resource_management/core/source.py @@ -72,7 +72,7 @@ def get_content(self): path = os.path.join(basedir, "files", self.name) if not sudo.path_isfile(path): - raise Fail("{0} Source file {1} is not found".format(repr(self), path)) + raise Fail(f"{repr(self)} Source file {path} is not found") return self.read_file(path) @@ -171,17 +171,17 @@ def __init__(self, name, redownload_files=False, ignore_proxy=True): def get_content(self): if self.download_path and not os.path.exists(self.download_path): - raise Fail("Directory {0} doesn't exist, please provide valid download path".format(self.download_path)) + raise Fail(f"Directory {self.download_path} doesn't exist, please provide valid download path") if urllib.parse.urlparse(self.url).path: filename = os.path.basename(urllib.parse.urlparse(self.url).path) else: - filename = 'index.html.{0}'.format(time.time()) + filename = f'index.html.{time.time()}' filepath = os.path.join(self.download_path, filename) if not self.cache or not os.path.exists(filepath): - Logger.info("Downloading the file from {0}".format(self.url)) + Logger.info(f"Downloading the file from {self.url}") if self.ignore_proxy: opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -193,14 +193,14 @@ def get_content(self): try: web_file = opener.open(req) except urllib.error.HTTPError as ex: - raise Fail("Failed to download file from {0} due to HTTP error: {1}".format(self.url, str(ex))) + raise Fail(f"Failed to download file from {self.url} due to HTTP error: {str(ex)}") content = web_file.read() if self.cache: sudo.create_file(filepath, content) else: - Logger.info("Not downloading the file from {0}, because {1} already exists".format(self.url, filepath)) + Logger.info(f"Not downloading the file from {self.url}, because {filepath} already exists") content = sudo.read_file(filepath) return content diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py index e55f9f3b0b7..7bd9eb988ae 100644 --- a/ambari-common/src/main/python/resource_management/core/sudo.py +++ b/ambari-common/src/main/python/resource_management/core/sudo.py @@ -115,15 +115,15 @@ def makedirs(path, mode): if ex.errno == errno.ENOENT: dirname = os.path.dirname(ex.filename) if os.path.islink(dirname) and not os.path.exists(dirname): - raise Fail("Cannot create directory '{0}' as '{1}' is a broken symlink".format(path, dirname)) + raise Fail(f"Cannot create directory '{path}' as '{dirname}' is a broken symlink") elif ex.errno == errno.ENOTDIR: dirname = os.path.dirname(ex.filename) if os.path.isfile(dirname): - raise Fail("Cannot create directory '{0}' as '{1}' is a file".format(path, dirname)) + raise Fail(f"Cannot create directory '{path}' as '{dirname}' is a file") elif ex.errno == errno.ELOOP: dirname = os.path.dirname(ex.filename) if os.path.islink(dirname) and not os.path.exists(dirname): - raise Fail("Cannot create directory '{0}' as '{1}' is a looped symlink".format(path, dirname)) + raise Fail(f"Cannot create directory '{path}' as '{dirname}' is a looped symlink") raise @@ -280,7 +280,7 @@ def __init__(self, path): code, out, err = shell.checked_call(cmd, sudo=True, stderr=subprocess.PIPE) values = out.split(' ') if len(values) != 3: - raise Fail("Execution of '{0}' returned unexpected output. {2}\n{3}".format(cmd, code, err, out)) + raise Fail(f"Execution of '{cmd}' returned unexpected output. {err}\n{out}") uid_str, gid_str, mode_str = values self.st_uid, self.st_gid, self.st_mode = int(uid_str), int(gid_str), int(mode_str, 8) @@ -304,7 +304,7 @@ def copy(src, dst): # os.listdir replacement def listdir(path): if not path_isdir(path): - raise Fail("{0} is not a directory. Cannot list files of it.".format(path)) + raise Fail(f"{path} is not a directory. Cannot list files of it.") code, out, err = shell.checked_call(["ls", path], sudo=True, stderr=subprocess.PIPE) files = out.splitlines() diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py index ff31f9e95e5..89789c351b9 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py @@ -39,13 +39,13 @@ def check_process_status(pid_file): from resource_management.core import sudo if not pid_file or not os.path.isfile(pid_file): - Logger.info("Pid file {0} is empty or does not exist".format(str(pid_file))) + Logger.info(f"Pid file {str(pid_file)} is empty or does not exist") raise ComponentIsNotRunning() try: pid = int(sudo.read_file(pid_file)) except: - Logger.info("Pid file {0} does not exist or does not contain a process id number".format(pid_file)) + Logger.info(f"Pid file {pid_file} does not exist or does not contain a process id number") raise ComponentIsNotRunning() try: @@ -56,8 +56,7 @@ def check_process_status(pid_file): # process ID or process group ID. sudo.kill(pid, 0) except OSError: - Logger.info("Process with pid {0} is not running. Stale pid file" - " at {1}".format(pid, pid_file)) + Logger.info(f"Process with pid {pid} is not running. Stale pid file at {pid_file}") raise ComponentIsNotRunning() diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index 2b20b86be6d..8610428abe7 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@ -66,13 +66,13 @@ def get_package_dirs(): if stack_name not in data: raise Fail( - "Cannot find conf-select packages for the {0} stack".format(stack_name)) + f"Cannot find conf-select packages for the {stack_name} stack") conf_select_key = "conf-select" data = data[stack_name] if conf_select_key not in data: raise Fail( - "There are no conf-select packages defined for this command for the {0} stack".format(stack_name)) + f"There are no conf-select packages defined for this command for the {stack_name} stack") package_dirs = data[conf_select_key] @@ -101,9 +101,9 @@ def create(stack_name, package, version, dry_run = False): # clarify the logging of what we're doing ... if dry_run: Logger.info( - "Checking to see which directories will be created for {0} on version {1}".format(package, version)) + f"Checking to see which directories will be created for {package} on version {version}") else: - Logger.info("Creating /etc/{0}/{1}/0 if it does not exist".format(package, version)) + Logger.info(f"Creating /etc/{package}/{version}/0 if it does not exist") command = "dry-run-create" if dry_run else "create-conf-dir" @@ -149,8 +149,7 @@ def select(stack_name, package, version, ignore_errors=False): shell.checked_call(_get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True) except Exception as exception: if ignore_errors is True: - Logger.warning("Could not select the directory for package {0}. Error: {1}".format(package, - str(exception))) + Logger.warning(f"Could not select the directory for package {package}. Error: {str(exception)}") else: raise @@ -163,7 +162,7 @@ def get_hadoop_conf_dir(): this will fallback to using "current". """ hadoop_conf_dir = os.path.join(os.path.sep, "etc", "hadoop", "conf") - Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir)) + Logger.info(f"Using hadoop conf dir: {hadoop_conf_dir}") return hadoop_conf_dir @@ -190,8 +189,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs): stack_name = Script.get_stack_name() for directory_struct in dirs: if not os.path.exists(directory_struct['conf_dir']): - Logger.info("Skipping the conf-select tool on {0} since {1} does not exist.".format( - package, directory_struct['conf_dir'])) + Logger.info(f"Skipping the conf-select tool on {package} since {directory_struct['conf_dir']} does not exist.") return @@ -201,7 +199,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs): # if the dry run reported an error, then we must assume that the package does not exist in # the conf-select tool if len(dry_run_directory) == 0: - Logger.info("The conf-select tool reported an error for the package {0}. The configuration linking will be skipped.".format(package)) + Logger.info(f"The conf-select tool reported an error for the package {package}. The configuration linking will be skipped.") return @@ -212,8 +210,7 @@ def convert_conf_directories_to_symlinks(package, version, dirs): # log that we'll actually be creating some directories soon if len(need_dirs) > 0: - Logger.info("Package {0} will have the following new configuration directories created: {1}".format( - package, ", ".join(dry_run_directory))) + Logger.info(f"Package {package} will have the following new configuration directories created: {', '.join(dry_run_directory)}") # Create the versioned /etc/[component]/[version]/0 folder (using create-conf-dir) and then # set it for the installed component: @@ -231,17 +228,17 @@ def convert_conf_directories_to_symlinks(package, version, dirs): # it's already a link; make sure it's a link to where we want it if os.readlink(old_conf) != current_dir: # the link isn't to the right spot; re-link it - Logger.info("Re-linking symlink {0} to {1}".format(old_conf, current_dir)) + Logger.info(f"Re-linking symlink {old_conf} to {current_dir}") Link(old_conf, action = "delete") Link(old_conf, to = current_dir) else: - Logger.info("{0} is already linked to {1}".format(old_conf, current_dir)) + Logger.info(f"{old_conf} is already linked to {current_dir}") elif os.path.isdir(old_conf): # the /etc//conf directory is not a link, so turn it into one - Logger.info("{0} is a directory - it must be converted into a symlink".format(old_conf)) + Logger.info(f"{old_conf} is a directory - it must be converted into a symlink") backup_dir = _get_backup_conf_directory(old_conf) - Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir)) + Logger.info(f"Backing up {old_conf} to {backup_dir} if destination doesn't exist already.") Execute(("cp", "-R", "-p", old_conf, backup_dir), not_if = format("test -e {backup_dir}"), sudo = True) @@ -274,18 +271,16 @@ def convert_conf_directories_to_symlinks(package, version, dirs): Link(old_conf, to = current_dir) else: Logger.info( - "Will not create symlink from {0} to {1} because the destination's parent dir does not exist.".format( - old_conf, current_dir)) + f"Will not create symlink from {old_conf} to {current_dir} because the destination's parent dir does not exist.") else: Logger.info( - "Will not create symlink from {0} to {1} because Atlas is not installed on this host.".format( - old_conf, current_dir)) + f"Will not create symlink from {old_conf} to {current_dir} because Atlas is not installed on this host.") else: # Normal path for other packages Link(old_conf, to = current_dir) except Exception as e: - Logger.warning("Could not change symlink for package {0} to point to current directory. Error: {1}".format(package, e)) + Logger.warning(f"Could not change symlink for package {package} to point to current directory. Error: {e}") def get_restricted_packages(): @@ -329,12 +324,12 @@ def get_restricted_packages(): data = json.loads(stack_packages_config) if stack_name not in data: - Logger.info("Cannot find conf-select packages for the {0} stack".format(stack_name)) + Logger.info(f"Cannot find conf-select packages for the {stack_name} stack") return package_names conf_select_key = "conf-select-patching" if conf_select_key not in data[stack_name]: - Logger.info("There are no conf-select-patching elements defined for this command for the {0} stack".format(stack_name)) + Logger.info(f"There are no conf-select-patching elements defined for this command for the {stack_name} stack") return package_names service_dict = data[stack_name][conf_select_key] @@ -363,19 +358,18 @@ def _seed_new_configuration_directories(package, created_directories): """ package_dirs = get_package_dirs() if package not in package_dirs: - Logger.warning("Unable to seed newly created configuration directories for {0} because it is an unknown component".format(package)) + Logger.warning(f"Unable to seed newly created configuration directories for {package} because it is an unknown component") return # seed the directories with any existing configurations # this allows files which are not tracked by Ambari to be available after an upgrade - Logger.info("Seeding versioned configuration directories for {0}".format(package)) + Logger.info(f"Seeding versioned configuration directories for {package}") expected_directories = package_dirs[package] try: # if the expected directories don't match those created, we can't seed them if len(created_directories) != len(expected_directories): - Logger.warning("The known configuration directories for {0} do not match those created by conf-select: {1}".format( - package, str(created_directories))) + Logger.warning(f"The known configuration directories for {package} do not match those created by conf-select: {str(created_directories)}") return @@ -396,7 +390,7 @@ def _seed_new_configuration_directories(package, created_directories): _copy_configurations(source_seed_directory, target_seed_directory) except Exception as e: - Logger.warning("Unable to seed new configuration directories for {0}. {1}".format(package, str(e))) + Logger.warning(f"Unable to seed new configuration directories for {package}. {str(e)}") def _copy_configurations(source_directory, target_directory): diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py index 9f8438a82fd..08a73de40fa 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py @@ -67,20 +67,20 @@ def _prepare_tez_tarball(): sudo.chmod(mapreduce_temp_dir, 0o777) sudo.chmod(tez_temp_dir, 0o777) - Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir)) + Logger.info(f"Extracting {mapreduce_source_file} to {mapreduce_temp_dir}") tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir) - Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir)) + Logger.info(f"Extracting {tez_source_file} to {tez_temp_dir}") tar_archive.untar_archive(tez_source_file, tez_temp_dir) hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib", "native") tez_lib_dir = os.path.join(tez_temp_dir, "lib") if not os.path.exists(hadoop_lib_native_dir): - raise Fail("Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_dir)) + raise Fail(f"Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {hadoop_lib_native_dir} does not exist") if not os.path.exists(tez_lib_dir): - raise Fail("Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist".format(tez_lib_dir)) + raise Fail(f"Unable to seed the Tez tarball with native libraries since the target Tez lib directory {tez_lib_dir} does not exist") # copy native libraries from hadoop to tez Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo = True) @@ -98,11 +98,11 @@ def _prepare_tez_tarball(): hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version, "hadoop", "lib", "native") if not sudo.path_isdir(hadoop_lib_native_lzo_dir): - Logger.warning("Unable to located native LZO libraries at {0}, falling back to hadoop home".format(hadoop_lib_native_lzo_dir)) + Logger.warning(f"Unable to located native LZO libraries at {hadoop_lib_native_lzo_dir}, falling back to hadoop home") hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current", "hadoop-client", "lib", "native") if not sudo.path_isdir(hadoop_lib_native_lzo_dir): - raise Fail("Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}".format(hadoop_lib_native_lzo_dir)) + raise Fail(f"Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {hadoop_lib_native_lzo_dir}") Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir), sudo = True) @@ -123,7 +123,7 @@ def _prepare_tez_tarball(): recursive_ownership = True) tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir, "tez-native.tar.gz") - Logger.info("Creating a new Tez tarball at {0}".format(tez_tarball_with_native_lib)) + Logger.info(f"Creating a new Tez tarball at {tez_tarball_with_native_lib}") tar_archive.archive_dir_via_temp_file(tez_tarball_with_native_lib, tez_temp_dir) # ensure that the tarball can be read and uploaded @@ -161,9 +161,10 @@ def _prepare_mapreduce_tarball(): # calculate the source directory for LZO hadoop_lib_native_source_dir = os.path.join(os.path.dirname(mapreduce_source_file), "lib", "native") if not sudo.path_exists(hadoop_lib_native_source_dir): - raise Fail("Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_source_dir)) + raise Fail(f"Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib " + f"directory {hadoop_lib_native_source_dir} does not exist") - Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir)) + Logger.info(f"Extracting {mapreduce_source_file} to {mapreduce_temp_dir}") tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir) mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib") @@ -187,7 +188,7 @@ def _prepare_mapreduce_tarball(): recursive_ownership = True) mapreduce_tarball_with_native_lib = os.path.join(mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz") - Logger.info("Creating a new mapreduce tarball at {0}".format(mapreduce_tarball_with_native_lib)) + Logger.info(f"Creating a new mapreduce tarball at {mapreduce_tarball_with_native_lib}") tar_archive.archive_dir_via_temp_file(mapreduce_tarball_with_native_lib, mapreduce_temp_dir) # ensure that the tarball can be read and uploaded @@ -206,57 +207,57 @@ def _prepare_mapreduce_tarball(): # complicated to change during a Rolling/Express upgrade. TARBALL_MAP = { "yarn": { - "dirs": ("{0}/{1}/{2}/hadoop-yarn/lib/service-dep.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/yarn/service-dep.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/hadoop-yarn/lib/service-dep.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/yarn/service-dep.tar.gz"), "service": "YARN" }, "tez": { - "dirs": ("{0}/{1}/{2}/tez/lib/tez.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/tez/tez.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/tez/lib/tez.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/tez/tez.tar.gz"), "service": "TEZ" }, "tez_hive2": { - "dirs": ("{0}/{1}/{2}/tez_hive2/lib/tez.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/tez_hive2/tez.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/tez_hive2/lib/tez.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/tez_hive2/tez.tar.gz"), "service": "HIVE" }, "hive": { - "dirs": ("{0}/{1}/{2}/hive/hive.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/hive/hive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/hive/hive.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/hive/hive.tar.gz"), "service": "HIVE" }, "hadoop_streaming": { - "dirs": ("{0}/{1}/{2}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/mapreduce/hadoop-streaming.jar".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/hadoop-mapreduce/hadoop-streaming.jar", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/mpreduce/hadoop-streaming.jar"), "service": "MAPREDUCE2" }, "mapreduce": { - "dirs": ("{0}/{1}/{2}/hadoop/mapreduce.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, LIB_DIR), - "/{0}/apps/{1}/mapreduce/mapreduce.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/hadoop/mapreduce.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/mapreduce/mapreduce.tar.gz"), "service": "MAPREDUCE2", "prepare_function": _prepare_mapreduce_tarball }, "spark": { - "dirs": ("{0}/{1}/{3}/spark/lib/spark-{2}-assembly.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN, STACK_NAME_PATTERN, LIB_DIR), - "/{0}/apps/{1}/spark/spark-{0}-assembly.jar".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"{STACK_ROOT_PATTERN}/{STACK_VERSION_PATTERN}/{LIB_DIR}/spark/lib/spark-{STACK_NAME_PATTERN}-assembly.jar", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/spark/spark-{STACK_NAME_PATTERN}-assembly.jar"), "service": "SPARK" }, "spark2": { - "dirs": ("/tmp/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN), - "/{0}/apps/{1}/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"/tmp/spark2/spark2-{STACK_NAME_PATTERN}-yarn-archive.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/spark2/spark2-{STACK_NAME_PATTERN}-yarn-archive.tar.gz"), "service": "SPARK2" }, "spark2hive": { - "dirs": ("/tmp/spark2/spark2-{0}-hive-archive.tar.gz".format(STACK_NAME_PATTERN), - "/{0}/apps/{1}/spark2/spark2-{0}-hive-archive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)), + "dirs": (f"/tmp/spark2/spark2-{STACK_NAME_PATTERN}-hive-archive.tar.gz", + f"/{STACK_NAME_PATTERN}/apps/{STACK_VERSION_PATTERN}/spark2/spark2-{STACK_NAME_PATTERN}-hive-archive.tar.gz"), "service": "SPARK2" } @@ -295,23 +296,23 @@ def get_tarball_paths(name, use_upgrading_version_during_upgrade=True, custom_so stack_name = Script.get_stack_name() if not stack_name: - Logger.error("Cannot copy {0} tarball to HDFS because stack name could not be determined.".format(str(name))) + Logger.error(f"Cannot copy {str(name)} tarball to HDFS because stack name could not be determined.") return False, None, None if name is None or name.lower() not in TARBALL_MAP: - Logger.error("Cannot copy tarball to HDFS because {0} is not supported in stack {1} for this operation.".format(str(name), str(stack_name))) + Logger.error(f"Cannot copy tarball to HDFS because {str(name)} is not supported in stack {str(stack_name)} for this operation.") return False, None, None service = TARBALL_MAP[name.lower()]['service'] stack_version = get_current_version(service=service, use_upgrading_version_during_upgrade=use_upgrading_version_during_upgrade) if not stack_version: - Logger.error("Cannot copy {0} tarball to HDFS because stack version could be be determined.".format(str(name))) + Logger.error(f"Cannot copy {str(name)} tarball to HDFS because stack version could be be determined.") return False, None, None stack_root = Script.get_stack_root() if not stack_root: - Logger.error("Cannot copy {0} tarball to HDFS because stack root could be be determined.".format(str(name))) + Logger.error(f"Cannot copy {str(name)} tarball to HDFS because stack root could be be determined.") return False, None, None (source_file, dest_file) = TARBALL_MAP[name.lower()]['dirs'] @@ -357,8 +358,7 @@ def get_current_version(service=None, use_upgrading_version_during_upgrade=True) # if there is no upgrade, then use the command's version if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade: - Logger.info("Tarball version was calcuated as {0}. Use Command Version: {1}".format( - version, use_upgrading_version_during_upgrade)) + Logger.info(f"Tarball version was calcuated as {version}. Use Command Version: {use_upgrading_version_during_upgrade}") return version @@ -386,22 +386,22 @@ def _get_single_version_from_stack_select(): out = None stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) - get_stack_versions_cmd = "{0} versions > {1}".format(stack_selector_path, tmp_file) + get_stack_versions_cmd = f"{stack_selector_path} versions > {tmp_file}" try: code, stdoutdata = shell.call(get_stack_versions_cmd, logoutput=True) with open(tmp_file, 'r+') as file: out = file.read() except Exception as e: - Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e))) + Logger.logger.exception(f"Could not parse output of {str(tmp_file)}. Error: {str(e)}") finally: try: if os.path.exists(tmp_file): os.remove(tmp_file) except Exception as e: - Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e))) + Logger.logger.exception(f"Could not remove file {str(tmp_file)}. Error: {str(e)}") if code != 0 or out is None or out == "": - Logger.error("Could not verify stack version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_stack_versions_cmd, str(code), str(out))) + Logger.error(f"Could not verify stack version by calling '{get_stack_versions_cmd}'. Return Code: {str(code)}, Output: {str(out)}.") return None matches = re.findall(r"([\d\.]+(?:-\d+)?)", out) @@ -409,7 +409,7 @@ def _get_single_version_from_stack_select(): if matches and len(matches) == 1: stack_version = matches[0] elif matches and len(matches) > 1: - Logger.error("Found multiple matches for stack version, cannot identify the correct one from: {0}".format(", ".join(matches))) + Logger.error(f"Found multiple matches for stack version, cannot identify the correct one from: {', '.join(matches)}") return stack_version @@ -432,16 +432,16 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0o444, custom_source_file=No """ import params - Logger.info("Called copy_to_hdfs tarball: {0}".format(name)) + Logger.info(f"Called copy_to_hdfs tarball: {name}") (success, source_file, dest_file, prepare_function) = get_tarball_paths(name, use_upgrading_version_during_upgrade, custom_source_file, custom_dest_file) if not success: - Logger.error("Could not copy tarball {0} due to a missing or incorrect parameter.".format(str(name))) + Logger.error(f"Could not copy tarball {str(name)} due to a missing or incorrect parameter.") return False if skip: - Logger.warning("Skipping copying {0} to {1} for {2} as it is a sys prepped host.".format(str(source_file), str(dest_file), str(name))) + Logger.warning(f"Skipping copying {str(source_file)} to {str(dest_file)} for {str(name)} as it is a sys prepped host.") return True if not skip_component_check: @@ -449,14 +449,14 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0o444, custom_source_file=No config_name = SERVICE_TO_CONFIG_MAP.get(name) config = default("/configurations/"+config_name, None) if config is None: - Logger.info("{0} is not present on the cluster. Skip copying {1}".format(config_name, source_file)) + Logger.info(f"{config_name} is not present on the cluster. Skip copying {source_file}") return False - Logger.info("Source file: {0} , Dest file in HDFS: {1}".format(source_file, dest_file)) + Logger.info(f"Source file: {source_file} , Dest file in HDFS: {dest_file}") if not os.path.exists(source_file): - Logger.error("WARNING. Cannot copy {0} tarball because file does not exist: {1} . " - "It is possible that this component is not installed on this host.".format(str(name), str(source_file))) + Logger.error(f"WARNING. Cannot copy {str(name)} tarball because file does not exist: {str(source_file)} . " + "It is possible that this component is not installed on this host.") return False # Because CopyFromLocal does not guarantee synchronization, it's possible for two processes to first attempt to @@ -495,7 +495,7 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0o444, custom_source_file=No mode=0o444, replace_existing_files=replace_existing_files, ) - Logger.info("Will attempt to copy {0} tarball from {1} to DFS at {2}.".format(name, source_file, dest_file)) + Logger.info(f"Will attempt to copy {name} tarball from {source_file} to DFS at {dest_file}.") # For improved performance, force_execute should be False so that it is delayed and combined with other calls. # If still want to run the command now, set force_execute to True diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py b/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py index 7b84182a4db..ad7716c4d48 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/curl_krb_request.py @@ -98,14 +98,14 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix, # when executing curl. Use a hash of the combination of the principal and keytab file # to generate a (relatively) unique cache filename so that we can use it as needed. Scope # this file by user in order to prevent sharing of cache files by multiple users. - ccache_file_name = HASH_ALGORITHM("{0}|{1}".format(principal, keytab).encode('utf-8')).hexdigest() + ccache_file_name = HASH_ALGORITHM(f"{principal}|{keytab}".encode('utf-8')).hexdigest() curl_krb_cache_path = os.path.join(tmp_dir, "curl_krb_cache") if not os.path.exists(curl_krb_cache_path): os.makedirs(curl_krb_cache_path) os.chmod(curl_krb_cache_path, 0o1777) - ccache_file_path = "{0}{1}{2}_{3}_cc_{4}".format(curl_krb_cache_path, os.sep, cache_file_prefix, user, ccache_file_name) + ccache_file_path = f"{curl_krb_cache_path}{os.sep}{cache_file_prefix}_{user}_cc_{ccache_file_name}" kerberos_env = {'KRB5CCNAME': ccache_file_path} # concurrent kinit's can cause the following error: @@ -130,7 +130,7 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix, # if the time has not expired, double-check that the cache still has a valid ticket if not is_kinit_required: - klist_command = "{0} -s {1}".format(klist_path_local, ccache_file_path) + klist_command = f"{klist_path_local} -s {ccache_file_path}" is_kinit_required = (shell.call(klist_command, user=user)[0] != 0) # if kinit is required, the perform the kinit @@ -146,8 +146,7 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix, # kinit; there's no need to set a ticket timeout as this will use the default invalidation # configured in the krb5.conf - regenerating keytabs will not prevent an existing cache # from working correctly - shell.checked_call("{0} -c {1} -kt {2} {3} > /dev/null".format(kinit_path_local, - ccache_file_path, keytab, principal), user=user) + shell.checked_call(f"{kinit_path_local} -c {ccache_file_path} -kt {keytab} {principal} > /dev/null", user=user) # record kinit time _KINIT_CACHE_TIMES[ccache_file_name] = current_time @@ -202,7 +201,7 @@ def curl_krb_request(tmp_dir, keytab, principal, url, cache_file_prefix, except Fail: if logger.isEnabledFor(logging.DEBUG): - logger.exception("Unable to make a curl request for {0}.".format(caller_label)) + logger.exception(f"Unable to make a curl request for {caller_label}.") raise finally: if os.path.isfile(cookie_file): diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/download_from_hdfs.py b/ambari-common/src/main/python/resource_management/libraries/functions/download_from_hdfs.py index 94fc34961a7..945feffd1af 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/download_from_hdfs.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/download_from_hdfs.py @@ -47,11 +47,11 @@ def download_from_hdfs(source_file, dest_path, user_group, owner, download_type= """ import params - Logger.info("Called download_from_hdfs source in HDFS: {0} , local destination path: {1}".format(source_file, dest_path)) + Logger.info(f"Called download_from_hdfs source in HDFS: {source_file} , local destination path: {dest_path}") # The destination directory must already exist if not os.path.exists(dest_path): - Logger.error("Cannot copy {0} because destination directory {1} does not exist.".format(source_file, dest_path)) + Logger.error(f"Cannot copy {source_file} because destination directory {dest_path} does not exist.") return False filename = os.path.basename(source_file) @@ -67,7 +67,7 @@ def download_from_hdfs(source_file, dest_path, user_group, owner, download_type= replace_existing_files=replace_existing_files, ) - Logger.info("Will attempt to copy from DFS at {0} to local file system {1}.".format(source_file, dest_file)) + Logger.info(f"Will attempt to copy from DFS at {source_file} to local file system {dest_file}.") # For improved performance, force_execute should be False so that it is delayed and combined with other calls. if force_execute: diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/expect.py b/ambari-common/src/main/python/resource_management/libraries/functions/expect.py index 2a82ad5984d..ca809a50f8b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/expect.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/expect.py @@ -52,15 +52,15 @@ def expect(name, expected_type, default_value=None): elif value != None and value.lower() == "false": value = False else: - raise Fail("Configuration {0} expected to be boolean (true or false), but found '{1}'".format(name, value)) + raise Fail(f"Configuration {name} expected to be boolean (true or false), but found '{value}'") else: type_name = type(value).__name__ - raise Fail("Configuration {0} expected to be boolean (true or false), but found instance of unknown type '{1}'".format(name, type_name)) + raise Fail(f"Configuration {name} expected to be boolean (true or false), but found instance of unknown type '{type_name}'") elif expected_type in [int, int, float]: try: value = expected_type(value) except (ValueError, TypeError): - raise Fail("Configuration {0} expected to be number, but found '{1}'".format(name, value)) + raise Fail(f"Configuration {name} expected to be number, but found '{value}'") return value @@ -86,15 +86,14 @@ def expect_v2(name, expected_type, default_value=None): elif value != None and value.lower() == "false": value = False else: - raise Fail("Configuration {0} expected to be boolean (true or false), but found '{1}'".format(name, value)) + raise Fail(f"Configuration {name} expected to be boolean (true or false), but found '{value}'") else: type_name = type(value).__name__ raise Fail( - "Configuration {0} expected to be boolean (true or false), but found instance of unknown type '{1}'".format( - name, type_name)) + f"Configuration {name} expected to be boolean (true or false), but found instance of unknown type '{type_name}'") elif expected_type in [int, int, float]: try: value = expected_type(value) except (ValueError, TypeError): - raise Fail("Configuration {0} expected to be number, but found '{1}'".format(name, value)) + raise Fail(f"Configuration {name} expected to be number, but found '{value}'") return value \ No newline at end of file diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/fcntl_based_process_lock.py b/ambari-common/src/main/python/resource_management/libraries/functions/fcntl_based_process_lock.py index fcbbeadafd0..4c6095f771b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/fcntl_based_process_lock.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/fcntl_based_process_lock.py @@ -61,7 +61,7 @@ def blocking_lock(self): if not self.enabled: return import fcntl - Logger.info("Trying to acquire a lock on {0}".format(self.lock_file_name)) + Logger.info(f"Trying to acquire a lock on {self.lock_file_name}") if self.lock_file is None or self.lock_file.closed: self.lock_file = open(self.lock_file_name, 'a') try: @@ -74,7 +74,7 @@ def blocking_lock(self): raise else: self.acquired = True - Logger.info("Acquired the lock on {0}".format(self.lock_file_name)) + Logger.info(f"Acquired the lock on {self.lock_file_name}") def unlock(self): """ @@ -83,7 +83,7 @@ def unlock(self): if not self.enabled: return import fcntl - Logger.info("Releasing the lock on {0}".format(self.lock_file_name)) + Logger.info(f"Releasing the lock on {self.lock_file_name}") if self.acquired: try: fcntl.lockf(self.lock_file, fcntl.LOCK_UN) @@ -99,7 +99,7 @@ def unlock(self): self.lock_file.close() self.lock_file = None except IOError: - Logger.warning("Failed to close {0}".format(self.lock_file_name)) + Logger.warning(f"Failed to close {self.lock_file_name}") def __enter__(self): self.blocking_lock() diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_config.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_config.py index 76d32e057ab..e654e001b34 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_config.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_config.py @@ -35,11 +35,11 @@ def get_config(config_type, default=None): if all_configurations: config = all_configurations.get(config_type, default) if not config: - Logger.warning("No configurations for config type {0}. Use default instead.".format(config_type)) + Logger.warning(f"No configurations for config type {config_type}. Use default instead.") return config else: - Logger.warning("No service configurations available in the \"configurations\" section. Use default instead.".format(config_type)) + Logger.warning("No service configurations available in the \"configurations\" section. Use default instead.") return default else: - Logger.warning("No service configurations available. Use default instead.".format(config_type)) + Logger.warning("No service configurations available. Use default instead.") return default diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_not_managed_resources.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_not_managed_resources.py index 8cde6657ba2..33aa43781fd 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_not_managed_resources.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_not_managed_resources.py @@ -42,8 +42,8 @@ def get_not_managed_resources(): property_value = default('/configurations/' + property_name, None) if property_value == None: - Logger.warning(("Property {0} from cluster-env/managed_hdfs_resource_property_names not found in configurations. " - "Management of this DFS resource will not be forced.").format(property_name)) + Logger.warning((f"Property {property_name} from cluster-env/managed_hdfs_resource_property_names not found in configurations. " + "Management of this DFS resource will not be forced.")) else: while property_value in not_managed_hdfs_path_list: not_managed_hdfs_path_list.remove(property_value) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py index b527e4bcf17..e0d3eaa9ef8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py @@ -46,4 +46,4 @@ def get_port_from_url(address): elif address.isdigit(): return address - raise Fail("No port in URL:{0}".format(address)) + raise Fail(f"No port in URL:{address}") diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py index 27f458a70c2..5bc5edb9472 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_stack_version.py @@ -41,7 +41,7 @@ def get_stack_version(package_name): try: component_home_dir = os.environ[package_name.upper() + "_HOME"] except KeyError: - Logger.info('Skipping get_stack_version since the component {0} is not yet available'.format(package_name)) + Logger.info(f'Skipping get_stack_version since the component {package_name} is not yet available') return None # lazy fail #As a rule, component_home_dir is of the form \[\][\] @@ -54,7 +54,7 @@ def get_stack_version(package_name): # with package_version = #.#.# and stack_version=#.#.#.#- match = re.findall('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', home_dir_split[iSubdir]) if not match: - Logger.info('Failed to get extracted version for component {0}. Home dir not in expected format.'.format(package_name)) + Logger.info(f'Failed to get extracted version for component {package_name}. Home dir not in expected format.') return None # lazy fail return match[0] @@ -72,8 +72,7 @@ def get_stack_version(package_name): return None # lazy fail try: - command = 'ambari-python-wrap {stack_selector_path} status {package_name}'.format( - stack_selector_path=stack_selector_path, package_name=package_name) + command = f'ambari-python-wrap {stack_selector_path} status {package_name}' return_code, stack_output = shell.call(command, timeout=20) except Exception as e: Logger.error(str(e)) @@ -81,7 +80,7 @@ def get_stack_version(package_name): if return_code != 0: raise Fail( - 'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code))) + f'Unable to determine the current version because of a non-zero return code of {str(return_code)}') stack_version = re.sub(package_name + ' - ', '', stack_output) stack_version = stack_version.rstrip() diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py index 09192680f97..ee8d0dcffec 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py @@ -38,4 +38,4 @@ def get_unique_id_and_date(): now = datetime.datetime.now() date = now.strftime("%M%d%y") - return "id{id}_date{date}".format(id=id, date=date) + return f"id{id}_date{date}" diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_user_call_output.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_user_call_output.py index 10652930d90..2939818f66b 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_user_call_output.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_user_call_output.py @@ -68,7 +68,7 @@ def get_user_call_output(command, user, quiet=False, is_checked_call=True, **cal caller_filename = sys._getframe(1).f_code.co_filename is_internal_call = shell.NOT_LOGGED_FOLDER in caller_filename if quiet == False or (quiet == None and not is_internal_call): - log_msg = "{0} returned {1}".format(get_user_call_output.__name__, result) + log_msg = f"{get_user_call_output.__name__} returned {result}" Logger.info(log_msg) return result diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/install_jdbc_driver.py b/ambari-common/src/main/python/resource_management/libraries/functions/install_jdbc_driver.py index a67f45db58d..d24ee74c7d8 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/install_jdbc_driver.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/install_jdbc_driver.py @@ -35,8 +35,7 @@ def ensure_jdbc_driver_is_in_classpath(dest_dir, cache_location, driver_url, dri #If not, attempt to download it from the server resources URL for driver_file in driver_files: dest_path = os.path.join(dest_dir, driver_file) - Logger.info("JDBC driver file(s) {0}: Attempting to copy from {1} or download from {2} to {3}".format( - str(driver_files), cache_location, driver_url, dest_dir)) + Logger.info(f"JDBC driver file(s) {str(driver_files)}: Attempting to copy from {cache_location} or download from {driver_url} to {dest_dir}") if not os.path.exists(dest_path): search_path = os.environ["PATH"] if cache_location: diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py b/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py index 228b243ed9f..ff100852e7c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/install_windows_msi.py @@ -134,7 +134,7 @@ def _create_symlinks(stack_version): for link_pair in links_pairs: link, target = link_pair target = glob.glob(os.path.expandvars(target))[0].replace("\\\\", "\\") - Execute('cmd /c mklink "{0}" "{1}"'.format(link, target)) + Execute(f'cmd /c mklink "{link}" "{target}"') # check if services exists and marker file present @@ -187,7 +187,7 @@ def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_pass try: download_file(file_url, os.path.join(msi_save_dir, save_file)) except: - raise Fail("Failed to download {url}".format(url=file_url)) + raise Fail(f"Failed to download {file_url}") File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir, data_dir=data_dir, diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/log_process_information.py b/ambari-common/src/main/python/resource_management/libraries/functions/log_process_information.py index c8353e7d3cb..6719a11b863 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/log_process_information.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/log_process_information.py @@ -38,4 +38,4 @@ def log_process_information(logger): for cmd in cmd_list: ret = shell_runner.run(cmd) - logger.info("Command '{0}' returned {1}. {2}{3}".format(cmd, ret["exitCode"], ret["error"], ret["output"])) + logger.info(f"Command '{cmd}' returned {ret['exitCode']}. {ret['error']}{ret['output']}") diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py index 194161382a2..e0a8b6694b6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py @@ -47,7 +47,7 @@ def __init__(self, apache_major, apache_minor, internal_minor, internal_maint, h self.__build = int(build) def __repr__(self): - return "{0}.{1}.{2}.{3}-h{4}-b{5}".format(*self.to_list()) + return f"{self.to_list()[0]}.{self.to_list()[1]}.{self.to_list()[2]}.{self.to_list()[3]}-h{self.to_list()[4]}-b{self.to_list()[5]}" def to_list(self): """ @@ -71,7 +71,7 @@ def cmp_version(self, other): :raise TypeError """ if other and not isinstance(other, self.__class__): - raise TypeError("Operand type is different from {0}".format(self.__class__.__name__)) + raise TypeError(f"Operand type is different from {self.__class__.__name__}") r = 0 x = self.to_list() @@ -139,7 +139,7 @@ def validate(cls, module_version): matcher = cls.__module_version_regex.match(version) if not matcher: - raise ValueError("{0} is not a valid {1}".format(version, cls.__name__)) + raise ValueError(f"{version} is not a valid {cls.__name__}") return matcher diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py index 6655e45cf50..1f920ed206d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/mounted_dirs_helper.py @@ -139,8 +139,8 @@ def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True): folder_exists = dir_ in valid_existing_dirs if not folder_exists and ignore_bad_mounts: - Logger.debug("The directory {0} doesn't exist.".format(dir_)) - Logger.warning("Not creating {0} as cluster-env/ignore_bad_mounts is enabled.".format(dir_)) + Logger.debug(f"The directory {dir_} doesn't exist.") + Logger.warning(f"Not creating {dir_} as cluster-env/ignore_bad_mounts is enabled.") may_manage_this_dir = False else: may_manage_this_dir = _may_manage_folder(dir_, last_mount_point_for_dir, is_non_root_dir, dirs_unmounted, error_messages, manage_dirs_on_root, curr_mount_point) @@ -153,7 +153,7 @@ def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True): Logger.warning("Trying to create another directory on the following mount: " + str(curr_mount_point)) if may_manage_this_dir: - Logger.info("Forcefully ensuring existence and permissions of the directory: {0}".format(dir_)) + Logger.info(f"Forcefully ensuring existence and permissions of the directory: {dir_}") # Call the function func(dir_) used_mounts.add(curr_mount_point) @@ -175,7 +175,7 @@ def handle_mounted_dirs(func, dirs_string, history_filename, update_cache=True): header = " WARNING ".join(["*****"] * 6) header = "\n" + "\n".join([header, ] * 3) + "\n" msg = " ".join(error_messages) + \ - " Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of {0}.".format(history_filename) + f" Please ensure that mounts are healthy. If the mount change was intentional, you can update the contents of {history_filename}." Logger.error(header + msg + header) dir_to_mount = DIR_TO_MOUNT_HEADER @@ -194,24 +194,24 @@ def _may_manage_folder(dir_, last_mount_point_for_dir, is_non_root_dir, dirs_unm if manage_dirs_on_root: may_manage_this_dir = True else: - Logger.warning("Will not manage the directory {0} since it's on root mount and cluster-env/manage_dirs_on_root == {1}".format(dir_, str(manage_dirs_on_root))) + Logger.warning(f"Will not manage the directory {dir_} since it's on root mount and cluster-env/manage_dirs_on_root == {str(manage_dirs_on_root)}") may_manage_this_dir = False # Do not add to the history file: dirs_unmounted.add(dir_) else: - Logger.debug("Last mount for {0} in the history file is {1}".format(dir_, str(last_mount_point_for_dir))) + Logger.debug(f"Last mount for {dir_} in the history file is {str(last_mount_point_for_dir)}") if last_mount_point_for_dir == curr_mount_point: if is_non_root_dir or manage_dirs_on_root: - Logger.debug("Will manage {0} since it's on the same mount point: {1}".format(dir_, str(last_mount_point_for_dir))) + Logger.debug(f"Will manage {dir_} since it's on the same mount point: {str(last_mount_point_for_dir)}") may_manage_this_dir = True else: - Logger.warning("Will not manage {0} since it's on the root mount point and cluster-env/manage_dirs_on_root == {1}".format(dir_, str(manage_dirs_on_root))) + Logger.warning(f"Will not manage {dir_} since it's on the root mount point and cluster-env/manage_dirs_on_root == {str(manage_dirs_on_root)}") may_manage_this_dir = False else: may_manage_this_dir = False dirs_unmounted.add(dir_) - msg = "Directory {0} became unmounted from {1} . Current mount point: {2} .".format(dir_, last_mount_point_for_dir, curr_mount_point) + msg = f"Directory {dir_} became unmounted from {last_mount_point_for_dir} . Current mount point: {curr_mount_point} ." error_messages.append(msg) Logger.warning(msg) return may_manage_this_dir diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_version.py index 0fc836eea54..de3d006556c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_version.py @@ -48,7 +48,7 @@ def __init__(self, major, minor, maint, hotfix, build): self.__build = int(build) def __repr__(self): - return "{0}.{1}.{2}-h{3}-b{4}".format(*self.to_list()) + return f"{self.to_list()[0]}.{self.to_list()[1]}.{self.to_list()[2]}.{self.to_list()[3]}-h{self.to_list()[4]}-b{self.to_list()[5]}" def to_list(self): """ @@ -71,7 +71,7 @@ def cmp_version(self, other): :raise TypeError """ if other and not isinstance(other, self.__class__): - raise TypeError("Operand type is different from {0}".format(self.__class__.__name__)) + raise TypeError(f"Operand type is different from {self.__class__.__name__}") r = 0 x = self.to_list() @@ -159,10 +159,10 @@ def validate_stack_version(cls, stack_version): if not matcher: matcher = cls.__mpack_legacy_stack_version_regex.match(version) if not matcher: - raise ValueError("{0} is not a valid {1}".format(version, cls.__name__)) + raise ValueError(f"{version} is not a valid {cls.__name__}") else: if not matcher.group("hotfix"): - raise ValueError("{0} is not a valid {1}".format(version, cls.__name__)) + raise ValueError(f"{version} is not a valid {cls.__name__}") return matcher @@ -192,7 +192,7 @@ def validate(cls, mpack_version): matcher = cls.__mpack_version_regex.match(version) if not matcher: - raise ValueError("{0} is not a valid {1}".format(version, cls.__name__)) + raise ValueError(f"{version} is not a valid {cls.__name__}") return matcher diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py index 589810b9a8f..d05b69cc6de 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py @@ -59,12 +59,12 @@ def doRetries(hdfs_site, security_enabled, run_user): doRetries.attempt += 1 active_namenodes, standby_namenodes, unknown_namenodes = get_namenode_states_noretries(hdfs_site, security_enabled, run_user, doRetries.attempt == times, name_service=name_service) Logger.info( - "NameNode HA states: active_namenodes = {0}, standby_namenodes = {1}, unknown_namenodes = {2}".format( - active_namenodes, standby_namenodes, unknown_namenodes)) + f"NameNode HA states: active_namenodes = {active_namenodes}, standby_namenodes = {standby_namenodes}," + f" unknown_namenodes = {unknown_namenodes}") if active_namenodes: return active_namenodes, standby_namenodes, unknown_namenodes elif doRetries.attempt == times: - Logger.warning("No active NameNode was found after {0} retries. Will return current NameNode HA states".format(times)) + Logger.warning(f"No active NameNode was found after {times} retries. Will return current NameNode HA states") return active_namenodes, standby_namenodes, unknown_namenodes raise Fail('No active NameNode was found.') @@ -103,7 +103,7 @@ def _get_namenode_states_noretries_single_ns(hdfs_site, name_service, security_e state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled, last_retry) # If JMX parsing failed if not state: - check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(name_service, nn_unique_id) + check_service_cmd = f"hdfs haadmin -ns {name_service} -getServiceState {nn_unique_id}" code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: @@ -163,7 +163,7 @@ def get_hdfs_cluster_id_from_jmx(hdfs_site, security_enabled, run_user): if state: return state - Logger.info("Cannot get clusterId from {0}".format(jmx_uri)) + Logger.info(f"Cannot get clusterId from {jmx_uri}") raise Fail("Cannot get clsuterId from jmx, since none of the namenodes is running/accessible via jmx.") @@ -226,7 +226,7 @@ def get_property_for_active_namenode(hdfs_site, name_service, property_name, sec name_services = get_nameservices(hdfs_site) if name_service not in name_services: - raise Fail('Trying to get property for non-existing ns=\'{1}\'. Valid nameservices are {2}'.format(property_name, name_service, ','.join(name_services))) + raise Fail(f'Trying to get property for non-existing ns=\'{name_service}\'. Valid nameservices are {",".join(name_services)}') active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user, name_service=name_service)[0] @@ -378,9 +378,9 @@ def get_name_service_by_hostname(hdfs_site, host_name): raise ValueError('Host name required when using namenode federation') for ns in name_services: - ha_name_nodes = hdfs_site['dfs.ha.namenodes.{0}'.format(ns)].split(',') + ha_name_nodes = hdfs_site[f'dfs.ha.namenodes.{ns}'].split(',') for nn in ha_name_nodes: - nn_rpc_port = hdfs_site['dfs.namenode.rpc-address.{0}.{1}'.format(ns,nn)] + nn_rpc_port = hdfs_site[f'dfs.namenode.rpc-address.{ns}.{nn}'] nn_rpc = nn_rpc_port.split(':')[0] if nn_rpc == host_name: return ns diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/oozie_prepare_war.py b/ambari-common/src/main/python/resource_management/libraries/functions/oozie_prepare_war.py index 6138af1916f..b794f0a3ae7 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/oozie_prepare_war.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/oozie_prepare_war.py @@ -82,7 +82,7 @@ def prepare_war(params): output = "" if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower(): - message = "Unexpected Oozie WAR preparation output {0}".format(output) + message = f"Unexpected Oozie WAR preparation output {output}" Logger.error(message) raise Fail(message) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py index 32c1522a5e7..8754d89be5f 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py @@ -72,7 +72,7 @@ def get_repository_by_name_urllib2(self, name, component, status, usernamepasswo base64string = base64.b64encode(usernamepassword.encode()).decode().replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header(f"Authorization", "Basic {base64string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) @@ -86,9 +86,9 @@ def get_repository_by_name_urllib2(self, name, component, status, usernamepasswo return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error getting {0} repository for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) + raise Fail(f"Error getting {name} repository for component {component}. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error getting {0} repository for component {1}. Reason - {2}.".format(name, component, e.reason)) + raise Fail(f"Error getting {name} repository for component {component}. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -122,12 +122,12 @@ def create_ranger_repository(self, component, repo_name, repo_properties, while retryCount <= 5: repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: - Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) + Logger.info(f'{component.title()} Repository {repo["name"]} exist') break else: response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger, policy_user) if response is not None: - Logger.info('{0} Repository created in Ranger admin'.format(component.title())) + Logger.info(f'{component.title()} Repository created in Ranger admin') break else: if retryCount < 5: @@ -135,7 +135,7 @@ def create_ranger_repository(self, component, repo_name, repo_properties, time.sleep(15) # delay for 15 seconds retryCount += 1 else: - Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) + Logger.error(f'{component.title()} Repository creation failed in Ranger admin') break else: Logger.error('Ambari admin user creation failed') @@ -152,13 +152,13 @@ def create_repository_urllib2(self, data, usernamepassword, policy_user): """ try: searchRepoURL = self.urlReposPub - base64string = base64.b64encode('{0}'.format(usernamepassword).encode()).decode().replace('\n', '') + base64string = base64.b64encode(f'{usernamepassword}'.encode()).decode().replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib.request.Request(searchRepoURL, data, headers) - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header(f"Authorization", "Basic {base64string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) @@ -195,9 +195,9 @@ def create_repository_urllib2(self, data, usernamepassword, policy_user): return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Error creating repository. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error creating repository. Reason - {0}.".format(e.reason)) + raise Fail(f"Error creating repository. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -215,9 +215,9 @@ def check_ranger_login_urllib2(self, url): return response_code except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Connection to Ranger Admin failed. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason)) + raise Fail(f"Connection to Ranger Admin failed. Reason - {e.reason}.") except http.client.BadStatusLine as e: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -238,7 +238,7 @@ def get_policy_by_repo_name(self, name, component, status, usernamepassword): base64string = base64.b64encode(usernamepassword.encode()).decode().replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header(f"Authorization", "Basic {base64string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) @@ -248,9 +248,9 @@ def get_policy_by_repo_name(self, name, component, status, usernamepassword): return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error getting policy from repository {0} for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) + raise Fail(f"Error getting policy from repository {name} for component {component}. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error getting policy from repository {0} for component {1}. Reason - {2}.".format(name, component, e.reason)) + raise Fail(f"Error getting policy from repository {name} for component {component}. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -266,13 +266,13 @@ def update_ranger_policy(self, policyId, data, usernamepassword): """ try: searchRepoURL = self.urlPolicies + "/" + str(policyId) - base64string = base64.b64encode('{0}'.format(usernamepassword).encode()).decode().replace('\n', '') + base64string = base64.b64encode(f'{usernamepassword}'.encode()).decode().replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib.request.Request(searchRepoURL, data, headers) - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header("Authorization", f"Basic {base64string}") request.get_method = lambda: 'PUT' result = openurl(request, timeout=20) response_code = result.getcode() @@ -285,9 +285,9 @@ def update_ranger_policy(self, policyId, data, usernamepassword): return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error updating policy. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Error updating policy. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error updating policy. Reason - {0}.".format(e.reason)) + raise Fail(f"Error updating policy. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -336,7 +336,7 @@ def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,u base64string = base64.b64encode(usernamepassword.encode()).decode().replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header("Authorization", f"Basic {base64string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) @@ -362,13 +362,13 @@ def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,u admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) - base64string = base64.b64encode('{0}'.format(usernamepassword).encode()).decode().replace('\n', '') + base64string = base64.b64encode(f'{usernamepassword}'.encode()).decode().replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib.request.Request(url, data.encode(), headers) - request.add_header("Authorization", "Basic {0}".format(base64string)) + request.add_header("Authorization", f"Basic {base64string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) @@ -382,9 +382,9 @@ def create_ambari_admin_user(self,ambari_admin_username, ambari_admin_password,u return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Error creating ambari admin user. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason)) + raise Fail(f"Error creating ambari admin user. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py index ab4077c8eab..092adb484ab 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py @@ -71,7 +71,7 @@ def get_repository_by_name_urllib2(self, name, component, status, usernamepasswo base_64_string = base64.b64encode(usernamepassword.encode()).decode().replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") - request.add_header("Authorization", "Basic {0}".format(base_64_string)) + request.add_header("Authorization", f"Basic {base_64_string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) @@ -85,9 +85,9 @@ def get_repository_by_name_urllib2(self, name, component, status, usernamepasswo return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error getting {0} repository for component {1}. Http status code - {2}. \n {3}".format(name, component, e.code, e.read())) + raise Fail(f"Error getting {name} repository for component {component}. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error getting {0} repository for component {1}. Reason - {2}.".format(name, component, e.reason)) + raise Fail(f"Error getting {name} repository for component {component}. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -112,12 +112,12 @@ def create_ranger_repository(self, component, repo_name, repo_properties, while retryCount <= 5: repo = self.get_repository_by_name_urllib2(repo_name, component, 'true', ambari_username_password_for_ranger) if repo is not None: - Logger.info('{0} Repository {1} exist'.format(component.title(), repo['name'])) + Logger.info(f'{component.title()} Repository {repo["name"]} exist') break else: response = self.create_repository_urllib2(repo_data, ambari_username_password_for_ranger) if response is not None: - Logger.info('{0} Repository created in Ranger admin'.format(component.title())) + Logger.info(f'{component.title()} Repository created in Ranger admin') break else: if retryCount < 5: @@ -125,7 +125,7 @@ def create_ranger_repository(self, component, repo_name, repo_properties, time.sleep(30) # delay for 30 seconds retryCount += 1 else: - Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) + Logger.error(f'{component.title()} Repository creation failed in Ranger admin') break else: Logger.error('Ambari admin user creation failed') @@ -146,19 +146,19 @@ def create_ranger_repository(self, component, repo_name, repo_properties, while retryCount <= 5: response = self.get_repository_by_name_curl(component_user,component_user_keytab,component_user_principal,repo_name, component, 'true') if response is not None: - Logger.info('{0} Repository {1} exist'.format(component.title(), (response['name']))) + Logger.info(f'{component.title()} Repository {response["name"]} exist') break else: response = self.create_repository_curl(component_user,component_user_keytab,component_user_principal,repo_name, repo_data,policy_user) if response and len(response) > 0: - Logger.info('{0} Repository created in Ranger admin'.format(component.title())) + Logger.info(f'{component.title()} Repository created in Ranger admin') break else: if retryCount < 5: time.sleep(30) # delay for 30 seconds retryCount += 1 else: - Logger.error('{0} Repository creation failed in Ranger admin'.format(component.title())) + Logger.error(f'{component.title()} Repository creation failed in Ranger admin') break else: Logger.error("Connection failed to Ranger Admin !") @@ -172,13 +172,13 @@ def create_repository_urllib2(self, data, usernamepassword): """ try: search_repo_url = self.url_repos_pub - base_64_string = base64.b64encode('{0}'.format(usernamepassword).encode()).decode().replace('\n', '') + base_64_string = base64.b64encode(f'{usernamepassword}'.encode()).decode().replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib.request.Request(search_repo_url, data.encode(), headers) - request.add_header("Authorization", "Basic {0}".format(base_64_string)) + request.add_header("Authorization", f"Basic {base_64_string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) @@ -190,9 +190,9 @@ def create_repository_urllib2(self, data, usernamepassword): raise Fail('Repository creation failed') except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error creating repository. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Error creating repository. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error creating repository. Reason - {0}.".format(e.reason)) + raise Fail(f"Error creating repository. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -211,9 +211,9 @@ def check_ranger_login_urllib2(self, url): return response_code except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Connection failed to Ranger Admin. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason)) + raise Fail(f"Connection failed to Ranger Admin. Reason - {e.reason}.") except http.client.BadStatusLine as e: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -237,7 +237,7 @@ def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, base_64_string = base64.b64encode(usernamepassword.encode()).decode().replace('\n', '') request.add_header("Content-Type", "application/json") request.add_header("Accept", "application/json") - request.add_header("Authorization", "Basic {0}".format(base_64_string)) + request.add_header("Authorization", f"Basic {base_64_string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(result.read()) @@ -263,13 +263,13 @@ def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, admin_user['description'] = ambari_admin_username admin_user['firstName'] = ambari_admin_username data = json.dumps(admin_user) - base_64_string = base64.b64encode('{0}'.format(usernamepassword).encode()).decode().replace('\n', '') + base_64_string = base64.b64encode(f'{usernamepassword}'.encode()).decode().replace('\n', '') headers = { 'Accept': 'application/json', "Content-Type": "application/json" } request = urllib.request.Request(url, data.encode(), headers) - request.add_header("Authorization", "Basic {0}".format(base_64_string)) + request.add_header("Authorization", f"Basic {base_64_string}") result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) @@ -283,9 +283,9 @@ def create_ambari_admin_user(self, ambari_admin_username, ambari_admin_password, return None except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error creating ambari admin user. Http status code - {0}. \n {1}".format(e.code, e.read())) + raise Fail(f"Error creating ambari admin user. Http status code - {e.code}. \n {e.read()}") else: - raise Fail("Error creating ambari admin user. Reason - {0}.".format(e.reason)) + raise Fail(f"Error creating ambari admin user. Reason - {e.reason}.") except http.client.BadStatusLine: raise Fail("Ranger Admin service is not reachable, please restart the service and then try again") except TimeoutError: @@ -343,7 +343,7 @@ def get_repository_by_name_curl(self, component_user, component_user_keytab, com try: search_repo_url = self.url_repos_pub + "?serviceName=" + name + "&serviceType=" + component + "&isEnabled=" + status if is_keyadmin: - search_repo_url = '{0}&suser=keyadmin'.format(search_repo_url) + search_repo_url = f'{search_repo_url}&suser=keyadmin' response,error_message,time_in_millis = self.call_curl_request(component_user,component_user_keytab,component_user_principal,search_repo_url,False,request_method='GET') response_stripped = response[1:len(response) - 1] if response_stripped and len(response_stripped) > 0: @@ -355,7 +355,7 @@ def get_repository_by_name_curl(self, component_user, component_user_keytab, com else: return None except Exception as err: - raise Fail('Error in call for getting Ranger service:\n {0}'.format(err)) + raise Fail(f'Error in call for getting Ranger service:\n {err}') @safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None) def create_repository_curl(self, component_user, component_user_keytab, component_user_principal, name, data, policy_user, is_keyadmin = False): @@ -370,7 +370,7 @@ def create_repository_curl(self, component_user, component_user_keytab, componen try: search_repo_url = self.url_repos_pub if is_keyadmin: - search_repo_url = '{0}?suser=keyadmin'.format(search_repo_url) + search_repo_url = f'{search_repo_url}?suser=keyadmin' header = 'Content-Type: application/json' method = 'POST' @@ -381,7 +381,7 @@ def create_repository_curl(self, component_user, component_user_keytab, componen Logger.info('Repository created Successfully') return response_json elif 'exists' in response.lower(): - Logger.info('Repository {name} already exists'.format(name=name)) + Logger.info(f'Repository {name} already exists') return response_json else: Logger.info('Repository creation failed') @@ -390,7 +390,7 @@ def create_repository_curl(self, component_user, component_user_keytab, componen Logger.info('Repository creation failed') return None except Exception as err: - raise Fail('Error in call for creating Ranger service:\n {0}'.format(err)) + raise Fail(f'Error in call for creating Ranger service:\n {err}') @safe_retry(times=5, sleep_time=8, backoff_factor=1.5, err_class=Fail, return_on_fail=None) def update_repository_urllib2(self, component, repo_name, repo_properties, admin_user, admin_password, force_rename = False): @@ -408,30 +408,30 @@ def update_repository_urllib2(self, component, repo_name, repo_properties, admin update_repo_url = update_repo_url + "?forceRename=true" repo_update_data = json.dumps(repo_properties) usernamepassword = admin_user + ":" + admin_password - base_64_string = base64.b64encode("{0}".format(usernamepassword).encode()).decode().replace("\n", "") + base_64_string = base64.b64encode(f"{usernamepassword}".encode()).decode().replace("\n", "") headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' } request = urllib.request.Request(update_repo_url, repo_update_data.encode(), headers) - request.add_header("Authorization", "Basic {0}".format(base_64_string)) + request.add_header("Authorization", f"Basic {base_64_string}") request.get_method = lambda: 'PUT' result = openurl(request, timeout=20) response_code = result.getcode() response = json.loads(json.JSONEncoder().encode(result.read())) if response_code == 200: - Logger.info("Service name {0} updated successfully on Ranger Admin for service {1}".format(repo_name, component)) + Logger.info(f"Service name {repo_name} updated successfully on Ranger Admin for service {component}") return response else: - raise Fail("Service name {0} updation failed on Ranger Admin for service {1}".format(repo_name, component)) + raise Fail(f"Service name {repo_name} updation failed on Ranger Admin for service {component}") except urllib.error.URLError as e: if isinstance(e, urllib.error.HTTPError): - raise Fail("Error updating service name {0} on Ranger Admin for service {1}. Http status code - {2} \n {3}".format(repo_name, component, e.code, e.read())) + raise Fail(f"Error updating service name {repo_name} on Ranger Admin for service {component}. Http status code - {e.code} \n {e.read()}") else: - raise Fail("Error updating service name {0} on Ranger Admin for service {1}. Reason - {2}".format(repo_name, component, e.reason)) + raise Fail(f"Error updating service name {repo_name} on Ranger Admin for service {component}. Reason - {e.reason}") except http.client.BadStatusLine: - raise Fail("Ranger Admin is not reachable for updating service name {0} for service {1}".format(repo_name, component)) + raise Fail(f"Ranger Admin is not reachable for updating service name {repo_name} for service {component}") except TimeoutError: raise Fail("Connection to Ranger Admin failed. Reason - timeout") @@ -458,13 +458,13 @@ def update_repository_curl(self, component, repo_name, repo_properties, componen if response and len(response) > 0: response_json = json.loads(response) if "name" in response_json: - Logger.info("Service name {0} updated successfully on Ranger Admin for service {1}".format(repo_name, component)) + Logger.info(f"Service name {repo_name} updated successfully on Ranger Admin for service {component}") return response_json else: - Logger.info("Service name {0} updation failed on Ranger Admin for service {1}".format(repo_name, component)) + Logger.info(f"Service name {repo_name} updation failed on Ranger Admin for service {component}") return None else: - Logger.info("Service name {0} updation failed on Ranger Admin for service {1}".format(repo_name, component)) + Logger.info(f"Service name {repo_name} updation failed on Ranger Admin for service {component}") return None except Exception as err: - raise Fail('Error updating service name {0} on Ranger Admin for service {1}.\n Reason - {2}'.format(repo_name, component, err)) + raise Fail(f'Error updating service name {repo_name} on Ranger Admin for service {component}.\n Reason - {err}') diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py b/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py index 7e53613560a..d1ef685bb05 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repo_version_history.py @@ -91,8 +91,8 @@ def write_actual_version_to_history_file(repository_version, actual_version): f.write(repository_version + "," + actual_version + "\n") wrote_value = True if wrote_value: - Logger.info("Appended value \"{0}\" to file {1} to track this as a new version.".format(value, REPO_VERSION_HISTORY_FILE)) + Logger.info(f"Appended value \"{value}\" to file {REPO_VERSION_HISTORY_FILE} to track this as a new version.") except Exception as err: - Logger.error("Failed to write to file {0} the value: {1}. Error: {2}".format(REPO_VERSION_HISTORY_FILE, value, str(err))) + Logger.error(f"Failed to write to file {REPO_VERSION_HISTORY_FILE} the value: {value}. Error: {str(err)}") return wrote_value \ No newline at end of file diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py index beb740edd7a..87b62858a43 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -61,19 +61,17 @@ def create_repo_files(self): if 0 == len(self.command_repository.items): Logger.warning( - "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format( - self.command_repository.stack_name, self.command_repository.version_string)) + f"Repository for {self.command_repository.stack_name}/{self.command_repository.version_string} has no repositories. Ambari may not be managing this version.") return {} repo_files = {} for repository in self.command_repository.items: if repository.repo_id is None: - raise Fail("Repository with url {0} has no id".format(repository.base_url)) + raise Fail(f"Repository with url {repository.base_url} has no id") if not repository.ambari_managed: Logger.warning( - "Repository for {0}/{1}/{2} is not managed by Ambari".format( - self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id)) + f"Repository for {self.command_repository.stack_name}/{self.command_repository.version_string}/{repository.repo_id} is not managed by Ambari") else: Repository(repository.repo_id, action="prepare", @@ -134,7 +132,7 @@ def __init__(self, repo_object): elif isinstance(repo_object, str): json_dict = json.loads(repo_object) else: - raise Fail("Cannot deserialize command repository {0}".format(str(repo_object))) + raise Fail(f"Cannot deserialize command repository {str(repo_object)}") # version_id is the primary id of the repo_version table in the database self.version_id = _find_value(json_dict, 'repoVersionId') @@ -162,7 +160,7 @@ def __init__(self, repo_object): self.repo_tags_to_skip.add("GPL") for r in self.all_items: if self.repo_tags_to_skip & r.tags: - Logger.info("Repository with url {0} is not created due to its tags: {1}".format(r.base_url, r.tags)) + Logger.info(f"Repository with url {r.base_url} is not created due to its tags: {r.tags}") else: self.items.append(r) diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py b/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py index b2a2ba63da4..66d98fce56a 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/security_commons.py @@ -59,9 +59,9 @@ def update_credential_provider_path(config, config_type, dest_provider_path, fil config_copy = config.copy() # overwrite the provider path with the path specified if use_local_jceks: - config_copy[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'localjceks://file{0}'.format(dest_provider_path) + config_copy[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = f'localjceks://file{dest_provider_path}' else: - config_copy[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file{0}'.format(dest_provider_path) + config_copy[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = f'jceks://file{dest_provider_path}' return config_copy return config diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py index 401edaa93a4..2a76cf0ec44 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py @@ -180,7 +180,7 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir): if os.path.isfile(atlas_hook_file_name): Link(source_lib_file_name, to=atlas_hook_file_name) else: - Logger.info("Atlas hook directory path {0} doesn't exist".format(atlas_hook_dir)) + Logger.info(f"Atlas hook directory path {atlas_hook_dir} doesn't exist") def install_atlas_hook_packages(atlas_plugin_package, atlas_ubuntu_plugin_package, host_sys_prepped, agent_stack_retry_on_unavailability, agent_stack_retry_count): diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 8f2d8e09134..54209f38de4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -315,11 +315,11 @@ def get_policycache_service_name(service_name, repo_name, cache_service_list): with open(policycache_json_file) as json_file: json_data = json.load(json_file) if 'serviceName' in json_data and json_data['serviceName'] == repo_name: - Logger.info("Skipping Ranger API calls, as policy cache file exists for {0}".format(service_name)) - Logger.warning("If service name for {0} is not created on Ranger Admin, then to re-create it delete policy cache file: {1}".format(service_name, policycache_json_file)) + Logger.info(f"Skipping Ranger API calls, as policy cache file exists for {service_name}") + Logger.warning(f"If service name for {service_name} is not created on Ranger Admin, then to re-create it delete policy cache file: {policycache_json_file}") service_name_exist_flag = True break except Exception as err: - Logger.error("Error occurred while fetching service name from policy cache file.\nError: {0}".format(err)) + Logger.error(f"Error occurred while fetching service name from policy cache file.\nError: {err}") return service_name_exist_flag diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/simulate_perf_cluster_alert_behaviour.py b/ambari-common/src/main/python/resource_management/libraries/functions/simulate_perf_cluster_alert_behaviour.py index 7201f0d38f7..26e0d78165c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/simulate_perf_cluster_alert_behaviour.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/simulate_perf_cluster_alert_behaviour.py @@ -80,8 +80,8 @@ def simulate_perf_cluster_alert_behaviour(alert_behaviour_properties, configurat alert_timeout_secs = configurations[alert_timeout_secs_key] if alert_timeout_return_value and alert_timeout_secs: - logger.info("Sleeping for {0} seconds".format(alert_timeout_secs)) - print("Sleeping for {0} seconds".format(alert_timeout_secs)) + logger.info(f"Sleeping for {alert_timeout_secs} seconds") + print(f"Sleeping for {alert_timeout_secs} seconds") time.sleep(int(alert_timeout_secs)) return (return_values_map[alert_timeout_return_value][0], [return_values_map[alert_timeout_return_value][1]]) else: diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 099516ae45e..5167d3f16fc 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -59,7 +59,7 @@ def check_stack_feature(stack_feature, stack_version): data = json.loads(stack_features_config) if stack_name not in data: - Logger.warning("Cannot find stack features for the stack named {0}".format(stack_name)) + Logger.warning(f"Cannot find stack features for the stack named {stack_name}") return False data = data[stack_name] @@ -119,8 +119,7 @@ def get_stack_feature_version(config): # if this is not an upgrade, then we take the simple path if upgrade_direction is None: Logger.info( - "Stack Feature Version Info: Cluster Stack={0}, Command Stack={1}, Command Version={2} -> {3}".format( - stack_version, command_stack, command_version, version_for_stack_feature_checks)) + f"Stack Feature Version Info: Cluster Stack={stack_version}, Command Stack={command_stack}, Command Version={command_version} -> {version_for_stack_feature_checks}") return version_for_stack_feature_checks @@ -129,9 +128,8 @@ def get_stack_feature_version(config): is_stop_command = _is_stop_command(config) if not is_stop_command: Logger.info( - "Stack Feature Version Info: Cluster Stack={0}, Command Stack={1}, Command Version={2}, Upgrade Direction={3} -> {4}".format( - stack_version, command_stack, command_version, upgrade_direction, - version_for_stack_feature_checks)) + f"Stack Feature Version Info: Cluster Stack={stack_version}, Command Stack={command_stack}," + f" Command Version={command_version}, Upgrade Direction={upgrade_direction} -> {version_for_stack_feature_checks}") return version_for_stack_feature_checks @@ -145,9 +143,8 @@ def get_stack_feature_version(config): version_for_stack_feature_checks = command_version if command_version is not None else stack_version Logger.info( - "Stack Feature Version Info: Cluster Stack={0}, Command Stack={1}, Command Version={2}, Upgrade Direction={3}, stop_command={4} -> {5}".format( - stack_version, command_stack, command_version, upgrade_direction, - is_stop_command, version_for_stack_feature_checks)) + f"Stack Feature Version Info: Cluster Stack={stack_version}, Command Stack={command_stack}, Command Version={command_version}" + f", Upgrade Direction={upgrade_direction}, stop_command={is_stop_command} -> {version_for_stack_feature_checks}") return version_for_stack_feature_checks diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index ae6e396eb59..315ef95c173 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -151,7 +151,7 @@ def get_supported_packages(): code, stdout = shell.call(command, sudo = True, quiet = True) if code != 0 or stdout is None: - raise Fail("Unable to query for supported packages using {0}".format(stack_selector_path)) + raise Fail(f"Unable to query for supported packages using {stack_selector_path}") # turn the output into lines, stripping each line return [line.strip() for line in stdout.splitlines()] @@ -171,7 +171,7 @@ def get_packages(scope, service_name = None, component_name = None): from resource_management.libraries.functions.default import default if scope not in _PACKAGE_SCOPES: - raise Fail("The specified scope of {0} is not valid".format(scope)) + raise Fail(f"The specified scope of {scope} is not valid") config = Script.get_config() @@ -198,13 +198,13 @@ def get_packages(scope, service_name = None, component_name = None): if stack_name not in data: raise Fail( - "Cannot find stack-select packages for the {0} stack".format(stack_name)) + f"Cannot find stack-select packages for the {stack_name} stack") stack_select_key = "stack-select" data = data[stack_name] if stack_select_key not in data: raise Fail( - "There are no stack-select packages defined for this command for the {0} stack".format(stack_name)) + f"There are no stack-select packages defined for this command for the {stack_name} stack") # this should now be the dictionary of role name to package name data = data[stack_select_key] @@ -212,13 +212,13 @@ def get_packages(scope, service_name = None, component_name = None): component_name = component_name.upper() if service_name not in data: - Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(service_name)) + Logger.info(f"Skipping stack-select on {service_name} because it does not exist in the stack-select package structure.") return None data = data[service_name] if component_name not in data: - Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(component_name)) + Logger.info(f"Skipping stack-select on {component_name} because it does not exist in the stack-select package structure.") return None # this one scope is not an array, so transform it into one for now so we can @@ -236,12 +236,12 @@ def get_packages(scope, service_name = None, component_name = None): if _PACKAGE_SCOPE_LEGACY in data[component_name]: legacy_package = data[component_name][_PACKAGE_SCOPE_LEGACY] Logger.info( - "The package {0} is not supported by this version of the stack-select tool, defaulting to the legacy package of {1}".format(package, legacy_package)) + f"The package {package} is not supported by this version of the stack-select tool, defaulting to the legacy package of {legacy_package}") # use the legacy package packages[index] = legacy_package else: - raise Fail("The package {0} is not supported by this version of the stack-select tool.".format(package)) + raise Fail(f"The package {package} is not supported by this version of the stack-select tool.") # transform the array bcak to a single element if scope == PACKAGE_SCOPE_STACK_SELECT: @@ -265,7 +265,7 @@ def select_all(version_to_select): Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified")) return - Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select)) + Logger.info(f"Executing {stack_selector_name} set all on {version_to_select}") command = format('{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`') only_if_command = format('ls -d {stack_root}/{version_to_select}*') @@ -297,8 +297,7 @@ def select_packages(version): if stack_select_packages is None: return - Logger.info("The following packages will be stack-selected to version {0} using a {1} orchestration and {2} scope: {3}".format( - version, orchestration.upper(), package_scope, ", ".join(stack_select_packages))) + Logger.info(f"The following packages will be stack-selected to version {version} using a {orchestration.upper()} orchestration and {package_scope} scope: {', '.join(stack_select_packages)}") for stack_select_package_name in stack_select_packages: select(stack_select_package_name, version) @@ -331,7 +330,7 @@ def select(component, version): if moduleName in modules: module = modules.get(moduleName) importlib.reload(module) - Logger.info("After {0}, reloaded module {1}".format(command, moduleName)) + Logger.info(f"After {command}, reloaded module {moduleName}") def get_role_component_current_stack_version(): @@ -343,7 +342,7 @@ def get_role_component_current_stack_version(): role_command = default("/roleCommand", "") stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME) - Logger.info("Checking version for {0} via {1}".format(role, stack_selector_name)) + Logger.info(f"Checking version for {role} via {stack_selector_name}") if role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP: stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role] else: @@ -353,17 +352,15 @@ def get_role_component_current_stack_version(): if not role: Logger.error("No role information available.") elif not role.lower().endswith("client"): - Logger.error("Mapping unavailable for role {0}. Skip checking its version.".format(role)) + Logger.error(f"Mapping unavailable for role {role}. Skip checking its version.") return None current_stack_version = get_stack_version(stack_select_component) if current_stack_version is None: - Logger.warning("Unable to determine {0} version for {1}".format( - stack_selector_name, stack_select_component)) + Logger.warning(f"Unable to determine {stack_selector_name} version for {stack_select_component}") else: - Logger.info("{0} is currently at version {1}".format( - stack_select_component, current_stack_version)) + Logger.info(f"{stack_select_component} is currently at version {current_stack_version}") return current_stack_version @@ -380,7 +377,7 @@ def get_hadoop_dir(target): stack_version = Script.get_stack_version() if not target in HADOOP_DIR_DEFAULTS: - raise Fail("Target {0} not defined".format(target)) + raise Fail(f"Target {target} not defined") hadoop_dir = HADOOP_DIR_DEFAULTS[target] @@ -431,7 +428,7 @@ def get_hadoop_dir_for_stack_version(target, stack_version): stack_root = Script.get_stack_root() if not target in HADOOP_DIR_DEFAULTS: - raise Fail("Target {0} not defined".format(target)) + raise Fail(f"Target {target} not defined") # home uses a different template if target == "home": @@ -513,7 +510,7 @@ def get_stack_version_before_install(component_name): stack_version = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(os.readlink(component_dir))))) match = re.match('[0-9]+.[0-9]+.[0-9]+', stack_version) if match is None: - Logger.info('Failed to get extracted version with {0} in method get_stack_version_before_install'.format(stack_selector_name)) + Logger.info(f'Failed to get extracted version with {stack_selector_name} in method get_stack_version_before_install') return None # lazy fail return stack_version else: diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py index 3cf11b84fa3..869261d1100 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py @@ -55,14 +55,14 @@ def get_stack_tool(name): return None, None, None if stack_name not in stack_tools: - Logger.warning("Cannot find stack tools for the stack named {0}".format(stack_name)) + Logger.warning(f"Cannot find stack tools for the stack named {stack_name}") return None, None, None # load the stack tooks keyed by the stack name stack_tools = stack_tools[stack_name] if not stack_tools or not name or name.lower() not in stack_tools: - Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools))) + Logger.warning(f"Cannot find config for {str(name)} stack tool in {str(stack_tools)}") return None, None, None tool_config = stack_tools[name.lower()] @@ -110,13 +110,13 @@ def get_stack_root(stack_name, stack_root_json): from resource_management.libraries.functions.default import default if stack_root_json is None: - return "/usr/{0}".format(stack_name.lower()) + return f"/usr/{stack_name.lower()}" stack_root = json.loads(stack_root_json) if stack_name not in stack_root: - Logger.warning("Cannot determine stack root for stack named {0}".format(stack_name)) - return "/usr/{0}".format(stack_name.lower()) + Logger.warning(f"Cannot determine stack root for stack named {stack_name}") + return f"/usr/{stack_name.lower()}" return stack_root[stack_name] diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py index 6f14cf999e2..ca8649b44fb 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py @@ -98,7 +98,7 @@ def get_component_version_with_stack_selector(stack_selector_path, component_nam try: # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088 with open(tmpfile.name, 'r') as file: - get_stack_comp_version_cmd = '{0} status {1} > {2}' .format(stack_selector_path, component_name, tmpfile.name) + get_stack_comp_version_cmd = f'{stack_selector_path} status {component_name} > {tmpfile.name}' code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True) out = file.read() diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 699cc0e90d1..fe7a2b2de6c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -102,7 +102,7 @@ def action_delayed(self, action_name, main_resource): except namenode_ha_utils.NoActiveNamenodeException as ex: # one of ns can be down (during initial start forexample) no need to worry for federated cluster if len(nameservices) > 1: - Logger.exception("Cannot run HdfsResource for nameservice {0}. Due to no active namenode present".format(nameservice)) + Logger.exception(f"Cannot run HdfsResource for nameservice {nameservice}. Due to no active namenode present") else: raise @@ -112,7 +112,7 @@ def action_delayed_for_nameservice(self, nameservice, action_name, main_resource env_dict_key = 'hdfs_files_sudo' if main_resource.create_as_root else 'hdfs_files' if main_resource.create_as_root: - Logger.info("Will create {0} as root user".format(main_resource.resource.target)) + Logger.info(f"Will create {main_resource.resource.target} as root user") if not env_dict_key in env.config: @@ -240,7 +240,7 @@ def run_command(self, *args, **kwargs): raise while True: - Logger.info("Retrying after {0} seconds. Reason: {1}".format(try_sleep, str(last_exception))) + Logger.info(f"Retrying after {try_sleep} seconds. Reason: {str(last_exception)}") try_count -= 1 time.sleep(try_sleep) @@ -391,7 +391,7 @@ def action_delayed(self, action_name, main_resource): except namenode_ha_utils.NoActiveNamenodeException as ex: # one of ns can be down (during initial start forexample) no need to worry for federated cluster if len(nameservices) > 1: - Logger.exception("Cannot run HdfsResource for nameservice {0}. Due to no active namenode present".format(nameservice)) + Logger.exception(f"Cannot run HdfsResource for nameservice {nameservice}. Due to no active namenode present") else: raise @@ -703,7 +703,7 @@ def action_delayed(self, action_name): # so for them fast-hdfs-resource.jar should be used if path_protocol and path_protocol != self.default_protocol: self.can_use_webhdfs = False - Logger.info("Cannot use webhdfs for {0} defaultFs = {1} has different protocol".format(self.resource.target, self.resource.default_fs)) + Logger.info(f"Cannot use webhdfs for {self.resource.target} defaultFs = {self.resource.default_fs} has different protocol") else: self.can_use_webhdfs = False self.create_as_root = True @@ -714,7 +714,7 @@ def action_delayed(self, action_name): self.manage_if_exists = not parsed_path in parsed_not_managed_paths if parsed_path in self.ignored_resources_list: - Logger.info("Skipping '{0}' because it is in ignore file {1}.".format(self.resource, self.resource.hdfs_resource_ignore_file)) + Logger.info(f"Skipping '{self.resource}' because it is in ignore file {self.resource.hdfs_resource_ignore_file}.") return self.get_hdfs_resource_executor().action_delayed(action_name, self) @@ -741,7 +741,7 @@ def get_hdfs_resource_executor(self): def assert_parameter_is_set(self, parameter_name): if not getattr(self.resource, parameter_name): - raise Fail("Resource parameter '{0}' is not set.".format(parameter_name)) + raise Fail(f"Resource parameter '{parameter_name}' is not set.") return True def kinit(self): diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/modify_properties_file.py b/ambari-common/src/main/python/resource_management/libraries/providers/modify_properties_file.py index 90a902fb6a7..44cfa12ce70 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/modify_properties_file.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/modify_properties_file.py @@ -54,14 +54,14 @@ def action_create(self): if in_var_name in properties: value = InlineTemplate(str(properties[in_var_name])).get_content() - new_content_lines[line_num] = "{0}{1}{2}".format(str(in_var_name), delimiter, value) + new_content_lines[line_num] = f"{str(in_var_name)}{delimiter}{value}" unsaved_values.remove(in_var_name) else: Logger.info(format("Creating new properties file as {filename} doesn't exist")) for property_name in unsaved_values: value = InlineTemplate(str(properties[property_name])).get_content() - line = "{0}{1}{2}".format(str(property_name), delimiter, value) + line = f"{str(property_name)}{delimiter}{value}" new_content_lines.append(line) final_content_lines = "\n".join(new_content_lines) diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py index 89445b74676..c772a99db57 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py @@ -62,7 +62,7 @@ def action_create(self): configuration_attrs=configuration_attrs, xml_include_file=self.resource.xml_include_file) xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename) - Logger.info("Generating config: {0}".format(xml_config_dest_file_path)) + Logger.info(f"Generating config: {xml_config_dest_file_path}") File (xml_config_dest_file_path, content = config_content, diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 1dc44367a1c..7fbc549a60f 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -219,8 +219,8 @@ def save_component_version_to_structured_out(self, command_name): # this is needed in cases where an existing symlink is on the system and stack-select can't # change it on installation (because it's scared to in order to support parallel installs) if is_install_command and repository_resolved and repository_version is not None: - Logger.info("The repository with version {0} for this command has been marked as resolved."\ - " It will be used to report the version of the component which was installed".format(repository_version)) + Logger.info(f"The repository with version {repository_version} for this command has been marked as resolved."\ + " It will be used to report the version of the component which was installed") component_version = repository_version @@ -238,9 +238,9 @@ def save_component_version_to_structured_out(self, command_name): output, code, versions = stack_select.unsafe_get_stack_versions() if len(versions) == 1: component_version = versions[0] - Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging. " \ - "However, the stack-select tool was able to report a single version installed ({1}). " \ - "This is the version that will be reported.".format(stack_select_package_name, component_version)) + Logger.error(f"The '{stack_select_package_name}' component did not advertise a version. This may indicate a problem with the component packaging. " \ + f"However, the stack-select tool was able to report a single version installed ({component_version}). " \ + "This is the version that will be reported.") if component_version: self.put_structured_out({"version": component_version}) @@ -252,7 +252,7 @@ def save_component_version_to_structured_out(self, command_name): self.put_structured_out({"repository_version_id": repo_version_id}) else: if not self.is_hook(): - Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name)) + Logger.error(f"The '{stack_select_package_name}' component did not advertise a version. This may indicate a problem with the component packaging.") def should_expose_component_version(self, command_name): @@ -379,11 +379,11 @@ def execute_prefix_function(self, command_name, afix, env): example: command_name=start, afix=pre will result in execution of self.pre_start(env) if exists """ self_methods = dir(self) - method_name = "{0}_{1}".format(afix, command_name) + method_name = f"{afix}_{command_name}" if not method_name in self_methods: - Logger.logger.debug("Action afix '{0}' not present".format(method_name)) + Logger.logger.debug(f"Action afix '{method_name}' not present") return - Logger.logger.debug("Execute action afix: {0}".format(method_name)) + Logger.logger.debug(f"Execute action afix: {method_name}") method = getattr(self, method_name) method(env) @@ -427,11 +427,11 @@ def post_start(self, env=None): pids = [] for pid_file in pid_files: if not sudo.path_exists(pid_file): - raise Fail("Pid file {0} doesn't exist after starting of the component.".format(pid_file)) + raise Fail(f"Pid file {pid_file} doesn't exist after starting of the component.") pids.append(sudo.read_file(pid_file).strip()) - Logger.info("Component has started with pid(s): {0}".format(', '.join([x.decode('utf-8') for x in pids]))) + Logger.info(f"Component has started with pid(s): {', '.join([x.decode('utf-8') for x in pids])}") def post_stop(self, env): """ @@ -465,7 +465,7 @@ def choose_method_to_execute(self, command_name): """ self_methods = dir(self) if not command_name in self_methods: - raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name)) + raise Fail(f"Script '{sys.argv[0]}' has no method '{command_name}'") method = getattr(self, command_name) return method @@ -518,7 +518,7 @@ def get_package_from_available(self, name, available_packages_in_repos=None): from resource_management.libraries.functions.default import default package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' - package_regex = name.replace(STACK_VERSION_PLACEHOLDER, '(\d|{0})+'.format(package_delimiter)) + "$" + package_regex = name.replace(STACK_VERSION_PLACEHOLDER, f'(\d|{package_delimiter})+') + "$" repo = default('/repositoryFile', None) name_with_version = None @@ -532,9 +532,9 @@ def get_package_from_available(self, name, available_packages_in_repos=None): return package if name_with_version: - raise Fail("No package found for {0}(expected name: {1})".format(name, name_with_version)) + raise Fail(f"No package found for {name}(expected name: {name_with_version})") else: - raise Fail("Cannot match package for regexp name {0}. Available packages: {1}".format(name, self.available_packages_in_repos)) + raise Fail(f"Cannot match package for regexp name {name}. Available packages: {self.available_packages_in_repos}") def format_package_name(self, name): from resource_management.libraries.functions.default import default @@ -579,7 +579,7 @@ def format_package_name(self, name): if (package_version is None or '*' in package_version) \ and effective_version is not None and 'INSTALL' == role_command: package_version = effective_version.replace('.', package_delimiter).replace('-', package_delimiter) - Logger.info("Version {0} was provided as effective cluster version. Using package version {1}".format(effective_version, package_version)) + Logger.info(f"Version {effective_version} was provided as effective cluster version. Using package version {package_version}") if package_version: stack_version_package_formatted = package_version @@ -720,13 +720,13 @@ def get_stack_root(): stack_root_json = default("/configurations/cluster-env/stack_root", None) if stack_root_json is None: - return "/usr/{0}".format(stack_name.lower()) + return f"/usr/{stack_name.lower()}" stack_root = json.loads(stack_root_json) if stack_name not in stack_root: - Logger.warning("Cannot determine stack root for stack named {0}".format(stack_name)) - return "/usr/{0}".format(stack_name.lower()) + Logger.warning(f"Cannot determine stack root for stack named {stack_name}") + return f"/usr/{stack_name.lower()}" return stack_root[stack_name] @@ -830,10 +830,10 @@ def load_available_packages(self): repos = CommandRepository(config['repositoryFile']) repo_ids = [repo.repo_id for repo in repos.items] - Logger.info("Command repositories: {0}".format(", ".join(repo_ids))) + Logger.info(f"Command repositories: {', '.join(repo_ids)}") repos.items = [x for x in repos.items if (not x.applicable_services or service_name in x.applicable_services) ] applicable_repo_ids = [repo.repo_id for repo in repos.items] - Logger.info("Applicable repositories: {0}".format(", ".join(applicable_repo_ids))) + Logger.info(f"Applicable repositories: {', '.join(applicable_repo_ids)}") pkg_provider = ManagerFactory.get() @@ -910,7 +910,7 @@ def should_install_package(self, package): chooser_method = getattr(package_conditions, condition) except AttributeError: name = package['name'] - raise Fail("Condition with name '{0}', when installing package {1}. Please check package_conditions.py.".format(condition, name)) + raise Fail(f"Condition with name '{condition}', when installing package {name}. Please check package_conditions.py.") return chooser_method() @@ -1026,9 +1026,9 @@ def restart(self, env): #TODO Once the logic for pid is available from Ranger and Ranger KMS code, will remove the below if block. services_to_skip = ['RANGER', 'RANGER_KMS'] if service_name in services_to_skip: - Logger.info('Temporarily skipping status check for {0} service only.'.format(service_name)) + Logger.info(f'Temporarily skipping status check for {service_name} service only.') elif is_stack_upgrade: - Logger.info('Skipping status check for {0} service during upgrade'.format(service_name)) + Logger.info(f'Skipping status check for {service_name} service during upgrade') else: self.status(env) raise Fail("Stop command finished but process keep running.") diff --git a/ambari-common/src/main/python/urlinfo_processor/urlinfo_processor.py b/ambari-common/src/main/python/urlinfo_processor/urlinfo_processor.py index e90dc201431..5f16e32b4e0 100644 --- a/ambari-common/src/main/python/urlinfo_processor/urlinfo_processor.py +++ b/ambari-common/src/main/python/urlinfo_processor/urlinfo_processor.py @@ -75,10 +75,7 @@ def replace_url_in_repoinfo_xml(repoinfo_xml_path, repo_id, repo_info): baseurl_tag = repo_tag.find("baseurl") if baseurl_tag is not None and family in repo_info: if family in repo_info: - print("URLINFO_PROCESSOR: replacing {0} to {1} for repo id:{2} and family:{3}".format(baseurl_tag.text, - repo_info[family], - repo_id, - family)) + print(f"URLINFO_PROCESSOR: replacing {baseurl_tag.text} to {repo_info[family]} for repo id:{repo_id} and family:{family}") baseurl_tag.text = repo_info[family] with open(repoinfo_xml_path, "w") as out: diff --git a/ambari-common/src/main/repo/install_ambari_tarball.py b/ambari-common/src/main/repo/install_ambari_tarball.py index a09d97fb37c..35d4b3ef349 100644 --- a/ambari-common/src/main/repo/install_ambari_tarball.py +++ b/ambari-common/src/main/repo/install_ambari_tarball.py @@ -57,7 +57,7 @@ def os_call(command, logoutput=None, env={}): else: stdout = stderr = None - logger.info("Running '{0}'".format(command)) + logger.info(f"Running '{command}'") proc = subprocess.Popen(command, shell=shell, stdout=stdout, stderr=stderr, env=env, universal_newlines=True) if not print_output: @@ -78,7 +78,7 @@ def os_call(command, logoutput=None, env={}): def install_package(name): from os_check import OSCheck - logger.info("Checking for existance of {0} dependency package".format(name)) + logger.info(f"Checking for existance of {name} dependency package") is_rpm = not OSCheck.is_ubuntu_family() if is_rpm: @@ -90,9 +90,9 @@ def install_package(name): try: Utils.os_call(is_installed_cmd, logoutput=False) - logger.info("Package {0} is already installed. Skipping installation.".format(name)) + logger.info(f"Package {name} is already installed. Skipping installation.") except OsCallFailure: - logger.info("Package {0} is not installed. Installing it...".format(name)) + logger.info(f"Package {name} is not installed. Installing it...") Utils.os_call(install_cmd) @@ -104,7 +104,7 @@ class FakePropertiesHeader(object): FAKE_SECTION_NAME = 'section' def __init__(self, fp): self.fp = fp - self.sechead = '[{0}]\n'.format(FakePropertiesHeader.FAKE_SECTION_NAME) + self.sechead = f'[{FakePropertiesHeader.FAKE_SECTION_NAME}]\n' def readline(self): if self.sechead: try: @@ -132,8 +132,8 @@ def download_files(self, files_list): if dirname: Utils.os_call(["mkdir", "-p", dirname]) - url = "{0}/{1}".format(self.base_url, name) - logger.info("Downloading {0}".format(url)) + url = f"{self.base_url}/{name}" + logger.info(f"Downloading {url}") Utils.os_call(["wget", "-O", name, url]) def run(self): @@ -183,7 +183,7 @@ def check_dependencies(self): packages_string = re.sub('[()]','',packages_string) if self.skip_dependencies: - var = input("Please confirm you have the following packages installed {0} (y/n): ".format(packages_string)) + var = input(f"Please confirm you have the following packages installed {packages_string} (y/n): ") if var.lower() != "y" and var.lower() != "yes": raise Exception("User canceled the installation.") return diff --git a/ambari-common/src/test/python/coilmq/protocol/__init__.py b/ambari-common/src/test/python/coilmq/protocol/__init__.py index 7e75f0f1a3a..be8f53e2f39 100644 --- a/ambari-common/src/test/python/coilmq/protocol/__init__.py +++ b/ambari-common/src/test/python/coilmq/protocol/__init__.py @@ -87,7 +87,7 @@ def process_frame(self, frame): cmd_method = frame.cmd.lower() if not cmd_method in VALID_COMMANDS: - raise ProtocolError("Invalid STOMP command: {}".format(frame.cmd)) + raise ProtocolError(f"Invalid STOMP command: {frame.cmd}") method = getattr(self, cmd_method, None) @@ -271,8 +271,7 @@ def enable_heartbeat(self, cx, cy, response): self.timer.schedule(max(self.send_heartbeat_interval, datetime.timedelta(milliseconds=cx)).total_seconds(), self.receive_heartbeat) self.timer.start() - response.headers['heart-beat'] = '{0},{1}'.format(int(self.send_heartbeat_interval.microseconds / 1000), - int(self.receive_heartbeat_interval.microseconds / 1000)) + response.headers['heart-beat'] = f'{int(self.send_heartbeat_interval.microseconds / 1000)},{int(self.receive_heartbeat_interval.microseconds / 1000)}' def disable_heartbeat(self): self.timer.stop() @@ -284,7 +283,7 @@ def send_heartbeat(self): def receive_heartbeat(self): ago = datetime.datetime.now() - self.last_hb if ago > self.receive_heartbeat_interval: - self.engine.log.debug("No heartbeat was received for {0} seconds".format(ago.total_seconds())) + self.engine.log.debug(f"No heartbeat was received for {ago.total_seconds()} seconds") self.engine.unbind() def connect(self, frame, response=None): @@ -315,7 +314,7 @@ def _negotiate_protocol(self, frame, response): self.engine.connection.send_frame(Frame( frames.ERROR, headers={'version': versions, 'content-type': frames.TEXT_PLAIN}, - body='Supported protocol versions are {0}'.format(versions) + body=f'Supported protocol versions are {versions}' )) else: response.headers['version'] = max(common) diff --git a/ambari-common/src/test/python/coilmq/util/frames.py b/ambari-common/src/test/python/coilmq/util/frames.py index 3a3580e8b45..f39a5c1809d 100644 --- a/ambari-common/src/test/python/coilmq/util/frames.py +++ b/ambari-common/src/test/python/coilmq/util/frames.py @@ -85,12 +85,8 @@ def __init__(self, cmd, headers=None, body=None): self.body = body or '' def __str__(self): - return '{{cmd={0},headers=[{1}],body={2}}}'.format( - self.cmd, - self.headers, - self.body if isinstance( - self.body, six.binary_type) else six.b(self.body) - ) + return (f'{{cmd={self.cmd},headers=[{self.headers}]' + f',body={self.body if isinstance(self.body, six.binary_type) else six.b(self.body)}}}') def __eq__(self, other): """ Override equality checking to test for matching command, headers, and body. """ @@ -121,11 +117,11 @@ def pack(self): # Convert and append any existing headers to a string as the # protocol describes. - headerparts = ("{0}:{1}\n".format(key, value) + headerparts = (f"{key}:{value}\n" for key, value in self.headers.items()) # Frame is Command + Header + EOF marker. - return six.b("{0}\n{1}\n".format(self.cmd, "".join(headerparts))) + (self.body if isinstance(self.body, six.binary_type) else six.b(self.body)) + six.b('\x00') + return six.b(f"{self.cmd}\n{''.join(headerparts)}\n") + (self.body if isinstance(self.body, six.binary_type) else six.b(self.body)) + six.b('\x00') class ConnectedFrame(Frame):