Merge pull request #560 from ddps-lab/azure-collector-fix

폴더 이름 오타 수정, 멀티 호출를  as_completed 방식아닌 기존 방식으로 롤백

Author: krtaiyang

collector/spot-dataset/azure/lambda/current_collector/lambda_function_sps.py

@@ -146,7 +146,7 @@ def handle_res_df_for_research(sps_res_desired_count_1_df, sps_res_desired_count
                                                            data_type='desired_count_1')
 
         save_raw_az_true_desired_count_loop_success = save_raw(sps_res_desired_count_loop_df, time_datetime, availability_zones,
-                                                               data_type='desired_count_loop')
+                                                               data_type='multi')
         save_raw_specific_az_true_success = save_raw(sps_res_specific_df, time_datetime, availability_zones, data_type='specific')
 
 
@@ -173,4 +173,5 @@ def handle_response(status_code, body, action, time_datetime, error_message=None
     if error_message:
         response["error_message"] = error_message
 
+    Logger.info(f"Response: {response}")
     return response

collector/spot-dataset/azure/lambda/current_collector/load_sps.py

@@ -14,7 +14,7 @@
 from datetime import datetime, timezone
 from utils.azure_auth import get_sps_token_and_subscriptions
 from utils.pub_service import S3, AZURE_CONST, Logger
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
 
 availability_zones = os.environ.get("availability_zones", "False").lower() == "true"
 
@@ -196,7 +196,7 @@ def execute_spot_placement_score_task_by_parameter_pool_df(api_calls_df, desired
                 future_to_desired_count[future] = desired_count
 
         try:
-            for future in as_completed(futures):
+            for future in futures:
                 try:
                     result = future.result()
                     desired_count = future_to_desired_count[future]
@@ -283,7 +283,6 @@ def execute_spot_placement_score_api(region_chunk, instance_type_chunk, desired_
             if res is None:
                 return "NO_AVAILABLE_LOCATIONS"
             else:
-                SS_Resources.succeed_to_get_next_available_location_count += 1
                 subscription_id, location, history, over_limit_locations = res
                 SL_Manager.update_call_history(subscription_id, location, history)
 

collector/spot-dataset/azure/lambda/current_collector/sps_module/sps_location_manager.py

@@ -114,6 +114,7 @@ def get_next_available_location():
                 if validation_can_call(location, current_history, current_over_limit_locations):
                     SS_Resources.last_subscription_id_and_location_tmp['last_subscription_id'] = subscription_id
                     SS_Resources.last_subscription_id_and_location_tmp['last_location'] = location
+                    SS_Resources.succeed_to_get_next_available_location_count += 1
                     return subscription_id, location, current_history, current_over_limit_locations
 
         return None

collector/spot-dataset/azure/lambda/current_collector/utils/upload_data.py

@@ -236,7 +236,7 @@ def save_raw(all_data_dataframe, time_utc, az, data_type=None):
             else:
                 data_path = f"{base_path}/{data_type}/{az_str}/{s3_dir_name}/{s3_obj_name}.csv.gz"
 
-        elif data_type in {"desired_count_loop", "specific"}:
+        elif data_type in {"multi", "specific"}:
             data_path = f"{base_path}/{data_type}/{az_str}/{s3_dir_name}/{s3_obj_name}.csv.gz"
 
         else: