| # Python wrapper script for collecting Canary metrics, setting-up/tearing-down alarms, reporting metrics to Cloudwatch, |
| # checking the alarms to ensure everything is correct at the end of the run, and pushing the log to S3 if successful. |
| |
| # Needs to be installed prior to running |
| # Part of standard packages in Python 3.4+ |
| import argparse |
| import time |
| import datetime |
| # Dependencies in project folder |
| from CanaryWrapper_Classes import * |
| from CanaryWrapper_MetricFunctions import * |
| |
| # Code for command line argument parsing |
| # ================================================================================ |
| command_parser = argparse.ArgumentParser("CanaryWrapper") |
| command_parser.add_argument("--canary_executable", type=str, required=True, |
| help="The path to the canary executable (or program - like 'python3')") |
| command_parser.add_argument("--canary_arguments", type=str, default="", |
| help="The arguments to pass/launch the canary executable with") |
| command_parser.add_argument("--git_hash", type=str, required=True, |
| help="The Git commit hash that we are running the canary with") |
| command_parser.add_argument("--git_repo_name", type=str, required=True, |
| help="The name of the Git repository") |
| command_parser.add_argument("--git_hash_as_namespace", type=bool, default=False, |
| help="(OPTIONAL, default=False) If true, the git hash will be used as the name of the Cloudwatch namespace") |
| command_parser.add_argument("--output_log_filepath", type=str, default="output.log", |
| help="(OPTIONAL, default=output.log) The file to output log info to. Set to 'None' to disable") |
| command_parser.add_argument("--output_to_console", type=bool, default=True, |
| help="(OPTIONAL, default=True) If true, info will be output to the console") |
| command_parser.add_argument("--cloudwatch_region", type=str, default="us-east-1", |
| help="(OPTIONAL, default=us-east-1) The AWS region for Cloudwatch") |
| command_parser.add_argument("--s3_bucket_name", type=str, default="canary-wrapper-folder", |
| help="(OPTIONAL, default=canary-wrapper-folder) The name of the S3 bucket where success logs will be stored") |
| command_parser.add_argument("--snapshot_wait_time", type=int, default=600, |
| help="(OPTIONAL, default=600) The number of seconds between gathering and sending snapshot reports") |
| command_parser.add_argument("--ticket_category", type=str, default="AWS", |
| help="(OPTIONAL, default=AWS) The category to register the ticket under") |
| command_parser.add_argument("--ticket_type", type=str, default="SDKs and Tools", |
| help="(OPTIONAL, default='SDKs and Tools') The type to register the ticket under") |
| command_parser.add_argument("--ticket_item", type=str, default="IoT SDK for CPP", |
| help="(OPTIONAL, default='IoT SDK for CPP') The item to register the ticket under") |
| command_parser.add_argument("--ticket_group", type=str, default="AWS IoT Device SDK", |
| help="(OPTIONAL, default='AWS IoT Device SDK') The group to register the ticket under") |
| command_parser.add_argument("--dependencies", type=str, default="", |
| help="(OPTIONAL, default='') Any dependencies and their commit hashes. \ |
| Current expected format is '(name or path);(hash);(next name or path);(hash);(etc...)'.") |
| command_parser.add_argument("--lambda_name", type=str, default="iot-send-email-lambda", |
| help="(OPTIONAL, default='CanarySendEmailLambda') The name of the Lambda used to send emails") |
| command_parser.add_argument("--codebuild_log_path", type=str, default="", |
| help="The CODEBUILD_LOG_PATH environment variable. Leave blank to ignore") |
| command_parser_arguments = command_parser.parse_args() |
| |
| if (command_parser_arguments.output_log_filepath == "None"): |
| command_parser_arguments.output_log_filepath = None |
| if (command_parser_arguments.snapshot_wait_time <= 0): |
| command_parser_arguments.snapshot_wait_time = 60 |
| |
| # Deal with possibly empty values in semi-critical commands/arguments |
| if (command_parser_arguments.canary_executable == ""): |
| print ("ERROR - required canary_executable is empty!", flush=True) |
| exit (1) # cannot run without a canary executable |
| if (command_parser_arguments.git_hash == ""): |
| print ("ERROR - required git_hash is empty!", flush=True) |
| exit (1) # cannot run without git hash |
| if (command_parser_arguments.git_repo_name == ""): |
| print ("ERROR - required git_repo_name is empty!", flush=True) |
| exit (1) # cannot run without git repo name |
| if (command_parser_arguments.git_hash_as_namespace is not True and command_parser_arguments.git_hash_as_namespace is not False): |
| command_parser_arguments.git_hash_as_namespace = False |
| if (command_parser_arguments.output_log_filepath == ""): |
| command_parser_arguments.output_log_filepath = None |
| if (command_parser_arguments.output_to_console != True and command_parser_arguments.output_to_console != False): |
| command_parser_arguments.output_to_console = True |
| if (command_parser_arguments.cloudwatch_region == ""): |
| command_parser_arguments.cloudwatch_region = "us-east-1" |
| if (command_parser_arguments.s3_bucket_name == ""): |
| command_parser_arguments.s3_bucket_name = "canary-wrapper-folder" |
| if (command_parser_arguments.ticket_category == ""): |
| command_parser_arguments.ticket_category = "AWS" |
| if (command_parser_arguments.ticket_type == ""): |
| command_parser_arguments.ticket_type = "SDKs and Tools" |
| if (command_parser_arguments.ticket_item == ""): |
| command_parser_arguments.ticket_item = "IoT SDK for CPP" |
| if (command_parser_arguments.ticket_group == ""): |
| command_parser_arguments.ticket_group = "AWS IoT Device SDK" |
| |
| |
| |
| # ================================================================================ |
| |
| datetime_now = datetime.datetime.now() |
| datetime_string = datetime_now.strftime("%d-%m-%Y/%H-%M-%S") |
| print("Datetime string is: " + datetime_string, flush=True) |
| |
| # Make the snapshot class |
| data_snapshot = DataSnapshot( |
| git_hash=command_parser_arguments.git_hash, |
| git_repo_name=command_parser_arguments.git_repo_name, |
| datetime_string=datetime_string, |
| git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace, |
| git_fixed_namespace_text="mqtt5_canary", |
| output_log_filepath="output.txt", |
| output_to_console=command_parser_arguments.output_to_console, |
| cloudwatch_region="us-east-1", |
| cloudwatch_make_dashboard=False, |
| cloudwatch_teardown_alarms_on_complete=True, |
| cloudwatch_teardown_dashboard_on_complete=True, |
| s3_bucket_name=command_parser_arguments.s3_bucket_name, |
| s3_bucket_upload_on_complete=True, |
| lambda_name=command_parser_arguments.lambda_name, |
| metric_frequency=command_parser_arguments.snapshot_wait_time) |
| |
| # Make sure nothing failed |
| if (data_snapshot.abort_due_to_internal_error == True): |
| print ("INFO - Stopping application due to error caused by credentials") |
| print ("Please fix your credentials and then restart this application again", flush=True) |
| exit(0) |
| |
| # Register metrics |
| data_snapshot.register_metric( |
| new_metric_name="total_cpu_usage", |
| new_metric_function=get_metric_total_cpu_usage, |
| new_metric_unit="Percent", |
| new_metric_alarm_threshold=70, |
| new_metric_reports_to_skip=1, |
| new_metric_alarm_severity=5, |
| is_percent=True) |
| data_snapshot.register_metric( |
| new_metric_name="total_memory_usage_value", |
| new_metric_function=get_metric_total_memory_usage_value, |
| new_metric_unit="Bytes") |
| data_snapshot.register_metric( |
| new_metric_name="total_memory_usage_percent", |
| new_metric_function=get_metric_total_memory_usage_percent, |
| new_metric_unit="Percent", |
| new_metric_alarm_threshold=70, |
| new_metric_reports_to_skip=0, |
| new_metric_alarm_severity=5, |
| is_percent=True) |
| |
| # Print diagnosis information |
| data_snapshot.output_diagnosis_information(command_parser_arguments.dependencies) |
| |
| # Make the snapshot (metrics) monitor |
| snapshot_monitor = SnapshotMonitor( |
| wrapper_data_snapshot=data_snapshot, |
| wrapper_metrics_wait_time=command_parser_arguments.snapshot_wait_time) |
| |
| # Make sure nothing failed |
| if (snapshot_monitor.had_internal_error == True): |
| print ("INFO - Stopping application due to error caused by credentials") |
| print ("Please fix your credentials and then restart this application again", flush=True) |
| exit(0) |
| |
| # Make the application monitor |
| application_monitor = ApplicationMonitor( |
| wrapper_application_path=command_parser_arguments.canary_executable, |
| wrapper_application_arguments=command_parser_arguments.canary_arguments, |
| wrapper_application_restart_on_finish=False, |
| data_snapshot=data_snapshot # pass the data_snapshot for printing to the log |
| ) |
| |
| # Make sure nothing failed |
| if (application_monitor.error_has_occurred == True): |
| print ("INFO - Stopping application due to error caused by credentials") |
| print ("Please fix your credentials and then restart this application again", flush=True) |
| exit(0) |
| |
| # For tracking if we stopped due to a metric alarm |
| stopped_due_to_metric_alarm = False |
| |
| execution_sleep_time = 30 |
| def execution_loop(): |
| while True: |
| snapshot_monitor.monitor_loop_function( |
| time_passed=execution_sleep_time, psutil_process=application_monitor.application_process_psutil) |
| application_monitor.monitor_loop_function( |
| time_passed=execution_sleep_time) |
| |
| # Did a metric go into alarm? |
| if (snapshot_monitor.has_cut_ticket == True): |
| # Set that we had an 'internal error' so we go down the right code path |
| snapshot_monitor.had_internal_error = True |
| break |
| |
| # If an error has occurred or otherwise this thread needs to stop, then break the loop |
| if (application_monitor.error_has_occurred == True or snapshot_monitor.had_internal_error == True): |
| break |
| |
| time.sleep(execution_sleep_time) |
| |
| |
| def application_thread(): |
| |
| start_email_body = "MQTT5 Short Running Canary Wrapper has started for " |
| start_email_body += "\"" + command_parser_arguments.git_repo_name + "\" commit \"" + command_parser_arguments.git_hash + "\"" |
| start_email_body += "\nThe wrapper will run for the length the MQTT5 Canary application is set to run for, which is determined by " |
| start_email_body += "the arguments set. The arguments used for this run are listed below:" |
| start_email_body += "\n Arguments: " + command_parser_arguments.canary_arguments |
| snapshot_monitor.send_email(email_body=start_email_body, email_subject_text_append="Started") |
| |
| # Start the application going |
| snapshot_monitor.start_monitoring() |
| application_monitor.start_monitoring() |
| # Allow the snapshot monitor to cut tickets |
| snapshot_monitor.can_cut_ticket = True |
| |
| # Start the execution loop |
| execution_loop() |
| |
| # Make sure everything is stopped |
| snapshot_monitor.stop_monitoring() |
| application_monitor.stop_monitoring() |
| |
| # Track whether this counts as an error (and therefore we should cleanup accordingly) or not |
| wrapper_error_occurred = False |
| # Finished Email |
| send_finished_email = True |
| finished_email_body = "MQTT5 Short Running Canary Wrapper has stopped." |
| finished_email_body += "\n\n" |
| |
| try: |
| # Find out why we stopped |
| if (snapshot_monitor.had_internal_error == True): |
| if (snapshot_monitor.has_cut_ticket == True): |
| # We do not need to cut a ticket here - it's cut by the snapshot monitor! |
| print ("ERROR - Snapshot monitor stopped due to metric in alarm!", flush=True) |
| finished_email_body += "Failure due to required metrics being in alarm! A new ticket should have been cut!" |
| finished_email_body += "\nMetrics in Alarm: " + str(snapshot_monitor.cloudwatch_current_alarms_triggered) |
| wrapper_error_occurred = True |
| else: |
| print ("ERROR - Snapshot monitor stopped due to internal error!", flush=True) |
| cut_ticket_using_cloudwatch( |
| git_repo_name=command_parser_arguments.git_repo_name, |
| git_hash=command_parser_arguments.git_hash, |
| git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace, |
| git_fixed_namespace_text="mqtt5_canary", |
| cloudwatch_region="us-east-1", |
| ticket_description="Snapshot monitor stopped due to internal error! Reason info: " + snapshot_monitor.internal_error_reason, |
| ticket_reason="Snapshot monitor stopped due to internal error", |
| ticket_allow_duplicates=True, |
| ticket_category=command_parser_arguments.ticket_category, |
| ticket_item=command_parser_arguments.ticket_item, |
| ticket_group=command_parser_arguments.ticket_group, |
| ticket_type=command_parser_arguments.ticket_type, |
| ticket_severity=4) |
| wrapper_error_occurred = True |
| finished_email_body += "Failure due to Snapshot monitor stopping due to an internal error." |
| finished_email_body += " Reason given for error: " + snapshot_monitor.internal_error_reason |
| |
| elif (application_monitor.error_has_occurred == True): |
| if (application_monitor.error_due_to_credentials == True): |
| print ("INFO - Stopping application due to error caused by credentials") |
| print ("Please fix your credentials and then restart this application again", flush=True) |
| wrapper_error_occurred = True |
| send_finished_email = False |
| else: |
| # Is the error something in the canary failed? |
| if (application_monitor.error_code != 0): |
| cut_ticket_using_cloudwatch( |
| git_repo_name=command_parser_arguments.git_repo_name, |
| git_hash=command_parser_arguments.git_hash, |
| git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace, |
| git_fixed_namespace_text="mqtt5_canary", |
| cloudwatch_region="us-east-1", |
| ticket_description="The Short Running Canary exited with a non-zero exit code! This likely means something in the canary failed.", |
| ticket_reason="The Short Running Canary exited with a non-zero exit code", |
| ticket_allow_duplicates=True, |
| ticket_category=command_parser_arguments.ticket_category, |
| ticket_item=command_parser_arguments.ticket_item, |
| ticket_group=command_parser_arguments.ticket_group, |
| ticket_type=command_parser_arguments.ticket_type, |
| ticket_severity=4) |
| wrapper_error_occurred = True |
| finished_email_body += "Failure due to MQTT5 application exiting with a non-zero exit code! This means something in the Canary application itself failed" |
| else: |
| print ("INFO - Stopping application. No error has occurred, application has stopped normally", flush=True) |
| application_monitor.print_stdout() |
| finished_email_body += "Short Running Canary finished successfully and run without errors!" |
| wrapper_error_occurred = False |
| else: |
| print ("ERROR - Short Running Canary stopped due to unknown reason!", flush=True) |
| cut_ticket_using_cloudwatch( |
| git_repo_name=command_parser_arguments.git_repo_name, |
| git_hash=command_parser_arguments.git_hash, |
| git_hash_as_namespace=command_parser_arguments.git_hash_as_namespace, |
| git_fixed_namespace_text="mqtt5_canary", |
| cloudwatch_region="us-east-1", |
| ticket_description="The Short Running Canary stopped for an unknown reason!", |
| ticket_reason="The Short Running Canary stopped for unknown reason", |
| ticket_allow_duplicates=True, |
| ticket_category=command_parser_arguments.ticket_category, |
| ticket_item=command_parser_arguments.ticket_item, |
| ticket_group=command_parser_arguments.ticket_group, |
| ticket_type=command_parser_arguments.ticket_type, |
| ticket_severity=4) |
| wrapper_error_occurred = True |
| finished_email_body += "Failure due to unknown reason! This shouldn't happen and means something has gone wrong!" |
| except Exception as e: |
| print ("ERROR: Could not (possibly) cut ticket due to exception!") |
| print ("Exception: " + str(e), flush=True) |
| |
| # Clean everything up and stop |
| snapshot_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred) |
| application_monitor.cleanup_monitor(error_occurred=wrapper_error_occurred) |
| print ("Short Running Canary finished!", flush=True) |
| |
| finished_email_body += "\n\nYou can find the log file for this run at the following S3 location: " |
| finished_email_body += "https://s3.console.aws.amazon.com/s3/object/" |
| finished_email_body += command_parser_arguments.s3_bucket_name |
| finished_email_body += "?region=" + command_parser_arguments.cloudwatch_region |
| finished_email_body += "&prefix=" + command_parser_arguments.git_repo_name + "/" + datetime_string + "/" |
| if (wrapper_error_occurred == True): |
| finished_email_body += "Failed_Logs/" |
| finished_email_body += command_parser_arguments.git_hash + ".log" |
| if (command_parser_arguments.codebuild_log_path != ""): |
| print ("\n Codebuild log path: " + command_parser_arguments.codebuild_log_path + "\n") |
| |
| # Send the finish email |
| if (send_finished_email == True): |
| if (wrapper_error_occurred == True): |
| snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Had an error") |
| else: |
| snapshot_monitor.send_email(email_body=finished_email_body, email_subject_text_append="Finished") |
| |
| exit (application_monitor.error_code) |
| |
| |
| # Start the application! |
| application_thread() |
