Parallel deployments (#81)

* first review for parallel deploymnets

* deprecate pkg_resources package

* include parallel deployment option

* added new functionalities

* update for package release

* update test python version and changelog

---------

Co-authored-by: Duarte Castaño <duarte.castañ[email protected]>

Author: duarte-castano

CHANGELOG.md

@@ -7,6 +7,32 @@
 [//]: # (Features)
 [//]: # (BREAKING CHANGES)
 
+## Nov 14th, 2024
+
+### Parallel Deployments
+
+The following scripts have been updated to enable creating and running parallel deployment plans:
+
+* `deploy_latest_tags_to_target_env.py`
+* `deploy_package_to_target_env.py`
+* `deploy_tags_to_target_env_with_manifest.py`
+
+To enable this feature, use the following parameter:
+
+* `--allow_parallel_deployments`: Skips LifeTime validation for active deployment plans.
+
+### Enhanced Pipeline Operations
+
+New pipeline scripts have been added to streamline operations related to manifest files:
+
+* `generate_manifest_file.py`: Generates a trigger manifest file.
+* `validate_manifest_apps_exist_in_target_env.py`: Verifies that manifest applications exist in the target environment.
+
+### Updated Package Dependencies
+
+* Updated `requests` dependency to version 2.32.2
+* Added `packaging` dependency, version 24.1
+
 ## May 16th, 2024
 
 ### Download Application Source Code

INSTALL.md

@@ -17,4 +17,5 @@ To install all the dependencies you'll need:
 
 * **Dependencies**
   * To install the dependencies needed, you just need to install the requirements with pip.
-  * On the root dir run: `pip install -q -I -r cd_pipelines/requirements.txt`
+  * On the root dir run: `pip install -q -I -r cd_pipelines/requirements.txt`
+  

build-pipeline.yml

@@ -26,8 +26,8 @@ jobs:
     matrix:
       Python38:
         python.version: '3.8'
-      Python311:
-        python.version: '3.11'
+      Python312:
+        python.version: '3.12'
     maxParallel: 2
 
   steps:

build-requirements.txt

@@ -1,6 +1,7 @@
 python-dateutil==2.9.0.post0
-requests==2.31.0
+requests==2.32.2
 unittest-xml-reporting==3.2.0
 xunitparser==1.3.4
 toposort==1.10
-python-dotenv==1.0.1
+python-dotenv==1.0.1
+packaging==24.1

outsystems/pipeline/deploy_latest_tags_to_target_env.py

@@ -2,7 +2,7 @@
 import sys
 import os
 import argparse
-from pkg_resources import parse_version
+from packaging.version import Version
 from time import sleep
 
 # Workaround for Jenkins:
@@ -105,10 +105,10 @@ def check_if_can_deploy(artifact_dir: str, lt_endpoint: str, lt_api_version: str
                         # The version is not the one deployed -> need to compare the version tag
                         app_in_env_data = get_application_version(artifact_dir, lt_endpoint, lt_token, False, app_in_env["BaseApplicationVersionKey"], app_key=app["Key"])
                         # If the version in the environment is bigger than the one in the manifest -> stale pipeline -> abort
-                        if parse_version(app_in_env_data["Version"]) > parse_version(app["Version"]):
+                        if Version(app_in_env_data["Version"]) > Version(app["Version"]):
                             print("The deployment manifest is stale. The Application {} needs to be deployed with version {} but then environment {} has the version {}.\nReason: VersionTag is inferior to the VersionTag already deployed.\nAborting the pipeline.".format(app["Name"], app["Version"], env_name, app_in_env_data["Version"]), flush=True)
                             sys.exit(1)
-                        elif parse_version(app_in_env_data["Version"]) == parse_version(app["Version"]):
+                        elif Version(app_in_env_data["Version"]) == Version(app["Version"]):
                             print("Skipping application {} with version {}, since it's already deployed in {} environment.\nReason: VersionTag is equal.".format(app["Name"], app["Version"], env_name), flush=True)
                         else:
                             # Generated app_keys for deployment plan based on the running version
@@ -122,7 +122,7 @@ def check_if_can_deploy(artifact_dir: str, lt_endpoint: str, lt_api_version: str
     return app_keys
 
 
-def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, source_env: str, dest_env: str, apps: list, dep_manifest: list, dep_note: str):
+def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, source_env: str, dest_env: str, apps: list, dep_manifest: list, dep_note: str, allow_parallel_deployments: bool):
 
     app_data_list = []  # will contain the applications to deploy details from LT
     to_deploy_app_keys = []  # will contain the app keys for the apps tagged
@@ -167,17 +167,18 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
                 raise NotImplementedError("Please make sure the API version is compatible with the module.")
     print("Creating deployment plan from {} to {} including applications: {} ({}).".format(source_env, dest_env, to_deploy_app_names, to_deploy_app_info), flush=True)
 
-    wait_counter = 0
-    deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
-    while len(deployments) > 0:
-        if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
-            print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
-            sys.exit(1)
-        sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
-        sleep(sleep_value)
-        wait_counter += sleep_value
-        print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+    if not allow_parallel_deployments:
+        wait_counter = 0
         deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
+        while len(deployments) > 0:
+            if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
+                print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
+                sys.exit(1)
+            sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
+            sleep(sleep_value)
+            wait_counter += sleep_value
+            print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+            deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
 
     # LT is free to deploy
     # Send the deployment plan and grab the key
@@ -262,7 +263,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
                         help="(optional) Manifest file path, used to promote the same application versions throughout the pipeline execution.")
     parser.add_argument("-cf", "--config_file", type=str,
                         help="Config file path. Contains configuration values to override the default ones.")
-
+    parser.add_argument("-p", "--allow_parallel_deployments", action='store_true',
+                        help="Skip LifeTime validation for active deployment plans.")
     args = parser.parse_args()
 
     # Load config file if exists
@@ -302,6 +304,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
         manifest_file = None
     # Parse Deployment Message
     dep_note = args.deploy_msg
+    # Parse Allow Parallel Deployments
+    allow_parallel_deployments = args.allow_parallel_deployments
 
     # Calls the main script
-    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, source_env, dest_env, apps, manifest_file, dep_note)
+    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, source_env, dest_env, apps, manifest_file, dep_note, allow_parallel_deployments)

outsystems/pipeline/deploy_package_to_target_env.py

@@ -32,25 +32,26 @@
 
 
 # ############################################################# SCRIPT ##############################################################
-def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, dest_env_label: str, force_two_step_deployment: bool, package_path: str):
+def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, dest_env_label: str, force_two_step_deployment: bool, package_path: str, allow_parallel_deployments: bool):
 
     # Builds the LifeTime endpoint
     lt_endpoint = build_lt_endpoint(lt_http_proto, lt_url, lt_api_endpoint, lt_api_version)
 
     # Gets the environment key for the destination environment
     dest_env_key = get_environment_key(artifact_dir, lt_endpoint, lt_token, dest_env_label)
 
-    wait_counter = 0
-    deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
-    while len(deployments) > 0:
-        if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
-            print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
-            sys.exit(1)
-        sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
-        sleep(sleep_value)
-        wait_counter += sleep_value
-        print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+    if not allow_parallel_deployments:
+        wait_counter = 0
         deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
+        while len(deployments) > 0:
+            if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
+                print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
+                sys.exit(1)
+            sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
+            sleep(sleep_value)
+            wait_counter += sleep_value
+            print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+            deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_key)
 
     # LT is free to deploy
     # Validate if file has OutSystems package extension
@@ -156,6 +157,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
                         help="Force the execution of the 2-Step deployment.")
     parser.add_argument("-cf", "--config_file", type=str,
                         help="Config file path. Contains configuration values to override the default ones.")
+    parser.add_argument("-p", "--allow_parallel_deployments", action='store_true',
+                        help="Skip LifeTime validation for active deployment plans.")
 
     args = parser.parse_args()
 
@@ -187,6 +190,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
     package_path = args.package_path
     # Parse Force Two-step Deployment flag
     force_two_step_deployment = args.force_two_step_deployment
+    # Parse Allow Parallel Deployments
+    allow_parallel_deployments = args.allow_parallel_deployments
 
     # Calls the main script
-    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, dest_env_label, force_two_step_deployment, package_path)
+    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, dest_env_label, force_two_step_deployment, package_path, allow_parallel_deployments)

outsystems/pipeline/deploy_tags_to_target_env_with_manifest.py

@@ -2,7 +2,7 @@
 import sys
 import os
 import argparse
-from pkg_resources import parse_version
+from packaging.version import Version
 from time import sleep
 import json
 
@@ -97,7 +97,7 @@ def check_if_can_deploy(artifact_dir: str, lt_endpoint: str, lt_api_version: str
                         # The version is not the one deployed -> need to compare the version tag
                         app_in_env_data = get_application_version(artifact_dir, lt_endpoint, lt_token, False, app_in_env["BaseApplicationVersionKey"], app_key=app["Key"])
                         # If the version in the target environment has the same version number -> skip deployment
-                        if parse_version(app_in_env_data["Version"]) == parse_version(app["Version"]):
+                        if Version(app_in_env_data["Version"]) == Version(app["Version"]):
                             print("Skipping application {} with version {}, since it's already deployed in {} environment.\nReason: VersionTag is equal.".format(app["Name"], app["Version"], env_name), flush=True)
                         else:
                             # Generated app_keys for deployment plan based on the target version
@@ -127,7 +127,7 @@ def check_if_can_deploy(artifact_dir: str, lt_endpoint: str, lt_api_version: str
     return app_keys
 
 
-def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, source_env_label: str, dest_env_label: str, include_test_apps: bool, trigger_manifest: dict, force_two_step_deployment: bool, include_deployment_zones: bool):
+def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: str, lt_api_version: int, lt_token: str, source_env_label: str, dest_env_label: str, include_test_apps: bool, trigger_manifest: dict, force_two_step_deployment: bool, include_deployment_zones: bool, allow_parallel_deployments: bool):
 
     app_data_list = []  # will contain the applications to deploy details from LT
     to_deploy_app_keys = []  # will contain the app keys for the apps tagged
@@ -168,17 +168,18 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
                 raise NotImplementedError("Please make sure the API version is compatible with the module.")
     print("Creating deployment plan from {} (Label: {}) to {} (Label: {}) including applications: {} ({}).".format(src_env_tuple[0], source_env_label, dest_env_tuple[0], dest_env_label, to_deploy_app_names, to_deploy_app_info), flush=True)
 
-    wait_counter = 0
-    deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_tuple[1])
-    while len(deployments) > 0:
-        if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
-            print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
-            sys.exit(1)
-        sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
-        sleep(sleep_value)
-        wait_counter += sleep_value
-        print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+    if not allow_parallel_deployments:
+        wait_counter = 0
         deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_tuple[1])
+        while len(deployments) > 0:
+            if wait_counter >= get_configuration_value("QUEUE_TIMEOUT_IN_SECS", QUEUE_TIMEOUT_IN_SECS):
+                print("Timeout occurred while waiting for LifeTime to be free, to create the new deployment plan.", flush=True)
+                sys.exit(1)
+            sleep_value = get_configuration_value("SLEEP_PERIOD_IN_SECS", SLEEP_PERIOD_IN_SECS)
+            sleep(sleep_value)
+            wait_counter += sleep_value
+            print("Waiting for LifeTime to be free. Elapsed time: {} seconds...".format(wait_counter), flush=True)
+            deployments = get_running_deployment(artifact_dir, lt_endpoint, lt_token, dest_env_tuple[1])
 
     # LT is free to deploy
     # Send the deployment plan and grab the key
@@ -288,6 +289,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
                         help="Flag that indicates if deployment zone selection is included in the deployment plan. Applicable to self-managed environments only.")
     parser.add_argument("-cf", "--config_file", type=str,
                         help="Config file path. Contains configuration values to override the default ones.")
+    parser.add_argument("-p", "--allow_parallel_deployments", action='store_true',
+                        help="Skip LifeTime validation for active deployment plans.")
 
     args = parser.parse_args()
 
@@ -336,5 +339,8 @@ def main(artifact_dir: str, lt_http_proto: str, lt_url: str, lt_api_endpoint: st
     # Parse Include Deployment Zones flag
     include_deployment_zones = args.include_deployment_zones
 
+    # Parse Allow Parallel Deployments
+    allow_parallel_deployments = args.allow_parallel_deployments
+
     # Calls the main script
-    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, source_env_label, dest_env_label, include_test_apps, trigger_manifest, force_two_step_deployment, include_deployment_zones)
+    main(artifact_dir, lt_http_proto, lt_url, lt_api_endpoint, lt_version, lt_token, source_env_label, dest_env_label, include_test_apps, trigger_manifest, force_two_step_deployment, include_deployment_zones, allow_parallel_deployments)