Skip to content

Commit

Permalink
Promoter: use api call for batch aggregate promotion
Browse files Browse the repository at this point in the history 
https://softwarefactory-project.io/r/#/c/17748 and
https://softwarefactory-project.io/r/#/c/17749

introduced the new api call api_promote_batch_post to promote a set of
commitdistro hashes associated to an aggregate hash in a single call.
This patch uses the new call as the only way to promote aggregate hash.
As the default call is single now, the implementation has been slightly
simplified merging two methods and removing one.

Change-Id: I48da64dc8584736bb1517a9f40c72d89683015f9
  • Loading branch information
@gabrielecerami
gabrielecerami authored and Zuul CI committed Apr 9, 2020
1 parent 31e8817 commit 7cafc92
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 219 deletions.
164 changes: 61 additions & 103 deletions ci-scripts/dlrnapi_promoter/dlrn_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,28 +358,6 @@ def promote(self, dlrn_hash, target_label, candidate_label=None,
self.promote_hash(log_header, dlrn_hash, target_label,
candidate_label=candidate_label)

def get_promotion_commitdistro_hashes(self, log_header, dlrn_hash,
candidate_label,
target_label):
"""
Return the hashes needed for the promotion of a commidistro hash
:param log_header: the header for all logging messages
:param dlrn_hash: The hash to use as a starting point
:param candidate_label: The name the hash was recently promoted to,
if any
:param target_label: The name to promote the hash to
:return: A list with all the hashes needed to promote, to have
the starting hash correctly promoted. In commitdistro case, this is
just the starting hash
"""
promotion_hash_list = []
self.log.debug("%s adding '%s' to promotion list for single pipeline",
log_header, dlrn_hash)

promotion_hash_list.append(dlrn_hash)

return promotion_hash_list

def get_hash_from_component(self, log_header, component_name, base_url):
"""
Downloads the commit.yaml file relative to the component, and creates
Expand Down Expand Up @@ -437,7 +415,7 @@ def get_promotion_aggregate_hashes(self, log_header, dlrn_hash,
the starting hash correctly promoted. In aggregate case these are all
the hashes from the component that form the aggregate hash
"""
promotion_hash_list = []
promotion_parameters_list = []
# Aggregate hash cannot be promoted directly, we need to promote
# all the components the aggregate points to singularly

Expand Down Expand Up @@ -493,110 +471,90 @@ def get_promotion_aggregate_hashes(self, log_header, dlrn_hash,
promotion_hash = self.get_hash_from_component(log_header,
component_name,
base_url)
promotion_hash_list.append(promotion_hash)
promotion_parameters = copy.deepcopy(self.promote_params)
promotion_hash.dump_to_params(promotion_parameters)
promotion_parameters.promote_name = target_label
promotion_parameters_list.append(promotion_parameters)

return promotion_hash_list
return sorted(promotion_parameters_list, key=lambda x: x.timestamp)

def promote_hash(self, log_header, dlrn_hash, target_label,
candidate_label=None):
"""
Calls the proper list creation method depending on the hash type,
then passes the list to the api promotion wrapper
Select the promotion parameters creation method and the proper api
call to promote the type of hashes, then call the api.
:param log_header: Header of all logging message in this function
:param dlrn_hash: The dlrn hash to promote
:param target_label: The label/name to promote dlrn_hash to
:param candidate_label: The name/label the dlrn_hash was recently
promoted to, if any (mandatory for aggregate promotion)
:return: None
:return: the promoted hash. It's the same hash for commitdistro,
and the last component promoted for the aggregate hash
"""
hash_type = type(dlrn_hash)
self.log.debug("%s promoting a %s", log_header, hash_type)

promotion_hash_list = []

if hash_type is DlrnCommitDistroHash:
promotion_hash_list = \
self.get_promotion_commitdistro_hashes(log_header,
dlrn_hash,
candidate_label,
target_label)
promotion_parameters = copy.deepcopy(self.promote_params)
dlrn_hash.dump_to_params(promotion_parameters)
promotion_parameters.promote_name = target_label
api_call = self.api_instance.api_promote_post

elif hash_type is DlrnAggregateHash:
promotion_hash_list = \
promotion_parameters = \
self.get_promotion_aggregate_hashes(log_header,
dlrn_hash,
candidate_label,
target_label)
api_call = self.api_instance.api_promote_batch_post
else:
self.log.error("Unrecognized dlrn hash type: %s", hash_type)
raise PromotionError("Unknown hash type")

if not promotion_hash_list:
self.log.error("%s No hashes ended up in the list to promote",
log_header)
raise PromotionError("Dlrn promote: No hashes to promote")

self.promote_hash_list(log_header, promotion_hash_list,
target_label)
try:
promoted_info = api_call(promotion_parameters)
except ApiException as ae:
message = ae.body
try:
body = json.loads(ae.body)
message = body['message']
except JSONDecodeError:
pass
self.log.error(
"Exception while promoting hashes to API endpoint "
"(%s) %s: %s", ae.status, ae.reason, message)
self.log.error("------- -------- Promoter aborted")
raise ae

def promote_hash_list(self, log_header, promotion_hash_list, target_label):
"""
This method promotes a list of hashes to a target label
from another POV the hash is labeled as the target
from another yet POV the label becomes a link to the hash identifier
:param log_header:
:param promotion_hash_list: A list of DlrnHashes
:param target_label: The label to promote the hashes to
:return: The list of promoted hashes (used mainly for tests)
"""
# The promoted info will sometimes return the aggregate,
# and always the timestamp
# but we'll always be interested in comparing just commit and
# distro hashes
stored_timestamp = dlrn_hash.timestamp
dlrn_hash.timestamp = None
promoted_hash = DlrnHash(source=promoted_info)
promoted_hash.timestamp = None

# Here we compare request and response. We check that info returned by
# the api call corresponds to the info passed to the api call.
# This seemingly stupid check already helped find at least 3 bugs
# in code and tests, because we think we are promoting something when
# we are promoting something completely different
if dlrn_hash == promoted_hash:
self.log.info("%s (subhash %s) Successfully promoted",
log_header, dlrn_hash)
else:
self.log.error("%s (subhash %s) API returned different"
" promoted hash: '%s'", log_header,
dlrn_hash, promoted_hash)
raise PromotionError("API returned different promoted hash")

# Promote in the same order the components were promoted
# initially
promotion_hash_list.sort(key=lambda x: x.timestamp)
# For every hash promoted, we need to update the named hashes.
self.update_current_named_hashes(dlrn_hash, target_label)
# Add back timestamp
dlrn_hash.timestamp = stored_timestamp

promoted_list = []
for promotion_hash in promotion_hash_list:
params = copy.deepcopy(self.promote_params)
promotion_hash.dump_to_params(params)
params.promote_name = target_label
try:
promoted_info = self.api_instance.api_promote_post(params)
except ApiException as ae:
message = ae.body
try:
body = json.loads(ae.body)
message = body['message']
except JSONDecodeError:
pass
self.log.error(
"Exception while promoting hashes to API endpoint "
"(%s) %s: %s", ae.status, ae.reason, message)
self.log.error("------- -------- Promoter aborted")
raise ae

# The promoted info will sometimes return the aggregate,
# and always the timestamp
# but we'll always be interested in comparing just commit and
# distro hashes
promoted_info.timestamp = None
stored_timestamp = promotion_hash.timestamp
promotion_hash.timestamp = None
promoted_hash = DlrnCommitDistroHash(source=promoted_info)

# This seemingly stupid check already helped find at least 3 bugs
# in code and tests.
if promoted_hash == promotion_hash:
self.log.info("%s (subhash %s) Successfully promoted",
log_header, promotion_hash)
else:
self.log.error("%s (subhash %s) API returned different"
" promoted hash: '%s'", log_header,
promotion_hash, promoted_hash)
raise PromotionError("API returned different promoted hash")

# For every hash promoted, we need to update the named hashes.
self.update_current_named_hashes(promotion_hash, target_label)
# Add back timestamp
promotion_hash.timestamp = stored_timestamp
promoted_list.append(promotion_hash)

return promoted_list
return dlrn_hash

def vote(self, dlrn_hash, job_id, job_url, vote):
"""
Expand Down
2 changes: 1 addition & 1 deletion ci-scripts/dlrnapi_promoter/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dlrnapi_client==0.8.1
dlrnapi_client>=0.9.1
configparser
MarkupSafe # ansible soft-dependency
shyaml
Expand Down
Loading

0 comments on commit 7cafc92

Please sign in to comment.