From 57cd0b95a4eb12f825b2d4e3a35cea1da9545a6d Mon Sep 17 00:00:00 2001 From: Harikrishna Yerra Date: Wed, 13 Dec 2023 17:21:27 +1100 Subject: [PATCH] Added Environment health status checks in clone and swap steps --- src/clone_blue_environment.py | 12 ++++++++++++ src/release_health_check.py | 4 ++-- src/swap_environment.py | 6 +++--- src/terminate_green_env.py | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/clone_blue_environment.py b/src/clone_blue_environment.py index 4408d2c..8bdd79f 100644 --- a/src/clone_blue_environment.py +++ b/src/clone_blue_environment.py @@ -1,6 +1,7 @@ import json import time import os +import swap_environment # noqa: E502 @@ -32,6 +33,17 @@ def main(BLUE_ENV_NAME, GREEN_ENV_NAME, BEANSTALK_APP_NAME, S3_ARTIFACTS_BUCKET, ) print("Green environment ID: " + green_env_id) + print("Wating for the Green environment to be Ready and Healthy\n") + + green_env_info, beanstalkclient = swap_environment.get_environment_information( + beanstalkclient, GREEN_ENV_NAME) + while ((green_env_info["Environments"][0]["Status"] != "Ready") and (green_env_info["Environments"][0]["Health"] != "Green") and (green_env_info["Environments"][0]["HealthStatus"] != "Ok")): + time.sleep(10) + green_env_info,beanstalkclient = swap_environment.get_environment_information( + beanstalkclient, GREEN_ENV_NAME) + + print("Green environment is Ready and Healthy\n") + if green_env_id and did_new_env_was_created: # Create a CNAME Config file blue_env_cname = blue_env_info['Environments'][0]['CNAME'] diff --git a/src/release_health_check.py b/src/release_health_check.py index 867f003..5b5d8fa 100644 --- a/src/release_health_check.py +++ b/src/release_health_check.py @@ -36,8 +36,8 @@ def get_env_info(beanstalkclient, env_name): def wait_until_env_be_ready(beanstalkclient, ENV_NAME): env_info = get_env_info(beanstalkclient, ENV_NAME) - while env_info["Environments"][0]["Status"] != "Ready": - print("Waiting the blue environment be Ready!") + while ((env_info["Environments"][0]["Status"] != "Ready") and (env_info["Environments"][0]["HealthStatus"] != "Ok")): + print("Waiting the blue environment to be Ready and Healthy!") time.sleep(10) env_info = get_env_info(beanstalkclient, ENV_NAME) return "Env is ready" diff --git a/src/swap_environment.py b/src/swap_environment.py index 9dfe50e..efd0175 100644 --- a/src/swap_environment.py +++ b/src/swap_environment.py @@ -38,7 +38,7 @@ def main(BLUE_ENV_NAME, GREEN_ENV_NAME, S3_ARTIFACTS_BUCKET, BEANSTALK_APP_NAME, if blue_env_url == green_env_cname: print("Nothing to swap") else: - while green_env_info["Environments"][0]["Status"] != "Ready": + while ((green_env_info["Environments"][0]["Status"] != "Ready") and (green_env_info["Environments"][0]["Health"] != "Green") and (green_env_info["Environments"][0]["HealthStatus"] != "Ok")): time.sleep(10) green_env_info = get_environment_information( beanstalkclient, GREEN_ENV_NAME) @@ -74,7 +74,7 @@ def get_environment_information(beanstalkclient, EnvName): time.sleep(60) if env_count == 3: - print(f"Waiting for {EnvName} env to be ready.") + print(f"Waiting for {EnvName} env to be ready and Healthy.") env_count = 0 else: env_count += 1 @@ -87,7 +87,7 @@ def swap_urls(beanstalkclient, SourceEnv, DestEnv): green_env_data = (beanstalkclient.describe_environments( EnvironmentNames=[SourceEnv, DestEnv], IncludeDeleted=False)) print(green_env_data) - if (((green_env_data["Environments"][0]["Status"]) == "Ready") and ((green_env_data["Environments"][1]["Status"]) == "Ready")): + if (((green_env_data["Environments"][0]["Status"]) == "Ready") and ((green_env_data["Environments"][0]["HealthStatus"]) == "Ok") and ((green_env_data["Environments"][1]["Status"]) == "Ready") and ((green_env_data["Environments"][1]["HealthStatus"]) == "Ok")): beanstalkclient.swap_environment_cnames( SourceEnvironmentName=SourceEnv, DestinationEnvironmentName=DestEnv) return ("Successful") diff --git a/src/terminate_green_env.py b/src/terminate_green_env.py index f83c7c5..ecc4f67 100644 --- a/src/terminate_green_env.py +++ b/src/terminate_green_env.py @@ -14,7 +14,7 @@ def main(BLUE_ENV_NAME, GREEN_ENV_NAME, BEANSTALK_APP_NAME, boto_authenticated_c DeleteConfigTemplate=delete_config_template_blue(beanstalkclient, AppName=(BEANSTALK_APP_NAME), TempName=(CREATE_CONFIG_TEMPLATE_NAME)) print(DeleteConfigTemplate) #re-swapping the urls - print("Swapping URL's") + print("Swapping URL's back!") reswap = swap_green_blue(beanstalkclient, SourceEnv=(BLUE_ENV_NAME), DestEnv=(GREEN_ENV_NAME)) if reswap == "Failure": print("Re-Swap did not happen") @@ -38,7 +38,7 @@ def delete_config_template_blue(beanstalkclient, AppName, TempName): def swap_green_blue(beanstalkclient, SourceEnv, DestEnv): GetEnvData = (beanstalkclient.describe_environments(EnvironmentNames=[SourceEnv,DestEnv],IncludeDeleted=False)) - if (((GetEnvData['Environments'][0]['Status']) == "Ready") and ((GetEnvData['Environments'][1]['Status']) == "Ready")): + if (((GetEnvData['Environments'][0]['Status']) == "Ready") and ((GetEnvData['Environments'][0]['HealthStatus']) == "Ok") and ((GetEnvData['Environments'][1]['Status']) == "Ready") and ((GetEnvData['Environments'][1]['HealthStatus']) == "Ok")): response = beanstalkclient.swap_environment_cnames(SourceEnvironmentName=SourceEnv,DestinationEnvironmentName=DestEnv) return ("Successful") else: