Source Code and Solution Download (#76)

* added fetch app source code logic
* added solutions logic

Author: duarte-castano

CHANGELOG.md

@@ -7,6 +7,37 @@
 [//]: # (Features)
 [//]: # (BREAKING CHANGES)
 
+## May 16th, 2024
+
+### Download Application Source Code
+
+A new script was added to download platform-generated source code:
+
+* `fetch_apps_source_code.py`
+
+Use the following parameters to generate more human-readable outputs and facilitate the compilation of the source code:
+
+* --friendly_package_names: source code packages with user-friendly names.
+* --include_all_refs: adds to .csproj file all assemblies in the bin folder as references.
+* --remove_resources_files: removes references to embedded resources files from the.csproj file.
+
+### Solution Download and Deploy
+
+Added new functions to leverage the recently released/improved APIs to download and deploy outsystems packages:
+
+* `fetch_lifetime_solution_from_manifest.py` - downloads a solution file based on manifest data.
+* `deploy_package_to_target_env.py` - deploys an outsystems package (solution or application) to a target environment.
+* `deploy_package_to_target_env_with_osptool.py` - deploys an outsystems package (solution or application) using OSP Tool.
+
+### Improved OSPTool Operations
+
+OSP Tool command line calls now have live output callback and catalog mapping support.
+
+### Updated Package Dependencies
+
+* Updated python-dateutil dependency to version 2.9.0.post0
+* Updated python-dotenv dependency to version 1.0.1
+
 ## November 15th, 2023
 
 ### Config File Support

INSTALL.md

@@ -5,7 +5,7 @@ In order to be able to test locally, there's a few things you'll have to install
 ## Install Python
 
 * Go to <https://www.python.org/downloads/>
-* Install Python v3.7.x (the code was tested with v3.7.1)
+* Install Python v3.11.x (the code was tested with v3.11.3)
 
 ## Install Python dependencies
 

build-requirements.txt

@@ -1,6 +1,6 @@
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
 requests==2.31.0
 unittest-xml-reporting==3.2.0
 xunitparser==1.3.4
 toposort==1.10
-python-dotenv==1.0.0
+python-dotenv==1.0.1

outsystems/exceptions/invalid_os_package.py

@@ -0,0 +1,2 @@
+class InvalidOutSystemsPackage(Exception):
+    pass

outsystems/file_helpers/file.py

@@ -1,15 +1,6 @@
 # Python Modules
 import json
 import os
-import requests
-
-
-def download_oap(file_path: str, auth_token: str, oap_url: str):
-    response = requests.get(oap_url, headers={"Authorization": auth_token})
-    # Makes sure that, if a directory is in the filename, that directory exists
-    os.makedirs(os.path.dirname(file_path), exist_ok=True)
-    with open(file_path, "wb") as f:
-        f.write(response.content)
 
 
 def store_data(artifact_dir: str, filename: str, data: str):
@@ -43,3 +34,12 @@ def clear_cache(artifact_dir: str, filename: str):
         return
     filename = os.path.join(artifact_dir, filename)
     os.remove(filename)
+
+
+# Returns a human readable string representation of bytes
+def bytes_human_readable_size(bytes, units=[' bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']):
+    return str(bytes) + units[0] if bytes < 1024 else bytes_human_readable_size(bytes >> 10, units[1:])
+
+
+def is_valid_os_package(filename: str):
+    return filename.lower().split('.')[-1] in ("osp", "oap")

outsystems/lifetime/lifetime_applications.py

@@ -12,8 +12,9 @@
 from outsystems.exceptions.environment_not_found import EnvironmentNotFoundError
 from outsystems.exceptions.app_version_error import AppVersionsError
 # Functions
-from outsystems.file_helpers.file import store_data, load_data, clear_cache, download_oap
+from outsystems.file_helpers.file import store_data, load_data, clear_cache
 from outsystems.lifetime.lifetime_base import send_get_request, send_post_request
+from outsystems.lifetime.lifetime_downloads import download_package
 # Variables
 from outsystems.vars.file_vars import APPLICATION_FOLDER, APPLICATIONS_FILE, APPLICATION_FILE, APPLICATION_VERSIONS_FILE, APPLICATION_VERSION_FILE
 from outsystems.vars.lifetime_vars import APPLICATIONS_ENDPOINT, APPLICATION_VERSIONS_ENDPOINT, APPLICATIONS_SUCCESS_CODE, \
@@ -158,7 +159,8 @@ def get_running_app_version(artifact_dir: str, endpoint: str, auth_token: str, e
                 "ApplicationName": app_tuple[0],
                 "ApplicationKey": app_tuple[1],
                 "Version": app_version_data["Version"],
-                "VersionKey": status_in_env["BaseApplicationVersionKey"]
+                "VersionKey": status_in_env["BaseApplicationVersionKey"],
+                "IsModified": status_in_env["IsModified"]
             }
             # Since these 2 fields were only introduced in a minor of OS11, we check here if they exist
             # We can't just use the version
@@ -212,7 +214,7 @@ def export_app_oap(file_path: str, endpoint: str, auth_token: str, env_key: str,
         # Stores the result
         url_string = response["response"]
         url_string = url_string["url"]
-        download_oap(file_path, auth_token, url_string)
+        download_package(file_path, auth_token, url_string)
         return
     elif status_code == APPLICATION_VERSION_NO_PERMISSION_CODE:
         raise NotEnoughPermissionsError(

outsystems/lifetime/lifetime_base.py

@@ -9,6 +9,7 @@
 from outsystems.vars.lifetime_vars import LIFETIME_SSL_CERT_VERIFY
 # Functions
 from outsystems.vars.vars_base import get_configuration_value
+from outsystems.file_helpers.file import check_file
 
 
 # Method that builds the LifeTime endpoint based on the LT host
@@ -55,6 +56,29 @@ def send_post_request(lt_api: str, token: str, api_endpoint: str, payload: str):
     return response_obj
 
 
+# Sends a POST request to LT, with binary content.
+def send_binary_post_request(lt_api: str, token: str, api_endpoint: str, dest_env: str, lt_endpont: str, binary_file_path: str):
+    # Auth token + content type octet-stream
+    headers = {'content-type': 'application/octet-stream',
+               'authorization': 'Bearer ' + token}
+    # Format the request URL to include the api endpoint
+    request_string = "{}/{}/{}/{}".format(lt_api, api_endpoint, dest_env, lt_endpont)
+
+    if check_file("", binary_file_path):
+        with open(binary_file_path, 'rb') as f:
+            data = f.read()
+    response = requests.post(request_string, data=data, headers=headers, verify=get_configuration_value("LIFETIME_SSL_CERT_VERIFY", LIFETIME_SSL_CERT_VERIFY))
+    response_obj = {"http_status": response.status_code, "response": {}}
+    # Since LT API POST requests do not reply with native JSON, we have to make it ourselves
+    if len(response.text) > 0:
+        try:
+            response_obj["response"] = response.json()
+        except:
+            # Workaround for POST /deployments/ since the response is not JSON, just text
+            response_obj["response"] = json.loads('"{}"'.format(response.text))
+    return response_obj
+
+
 # Sends a DELETE request to LT
 def send_delete_request(lt_api: str, token: str, api_endpoint: str):
     # Auth token + content type json
@@ -71,3 +95,14 @@ def send_delete_request(lt_api: str, token: str, api_endpoint: str):
             raise InvalidJsonResponseError(
                 "DELETE {}: The JSON response could not be parsed. Response: {}".format(request_string, response.text))
     return response_obj
+
+
+# Sends a GET request to LT, with url_params
+def send_download_request(pkg_url: str, token: str):
+    # Auth token + content type json
+    headers = {'content-type': 'application/json',
+               'authorization': token}
+    # Format the request URL to include the api endpoint
+    response = requests.get(pkg_url, headers=headers, verify=get_configuration_value("LIFETIME_SSL_CERT_VERIFY", LIFETIME_SSL_CERT_VERIFY))
+    response_obj = {"http_status": response.status_code, "response": response.content}
+    return response_obj

outsystems/lifetime/lifetime_deployments.py

@@ -12,7 +12,7 @@
 from outsystems.exceptions.environment_not_found import EnvironmentNotFoundError
 from outsystems.exceptions.impossible_action_deployment import ImpossibleApplyActionDeploymentError
 # Functions
-from outsystems.lifetime.lifetime_base import send_get_request, send_post_request, send_delete_request
+from outsystems.lifetime.lifetime_base import send_get_request, send_post_request, send_delete_request, send_binary_post_request
 from outsystems.lifetime.lifetime_environments import get_environment_key
 from outsystems.file_helpers.file import store_data
 # Variables
@@ -24,7 +24,8 @@
     DEPLOYMENT_SUCCESS_CODE, DEPLOYMENT_INVALID_CODE, DEPLOYMENT_NO_PERMISSION_CODE, DEPLOYMENT_NO_ENVIRONMENT_CODE, DEPLOYMENT_FAILED_CODE, \
     DEPLOYMENT_DELETE_SUCCESS_CODE, DEPLOYMENT_DELETE_IMPOSSIBLE_CODE, DEPLOYMENT_DELETE_NO_PERMISSION_CODE, DEPLOYMENT_DELETE_NO_DEPLOYMENT_CODE, \
     DEPLOYMENT_DELETE_FAILED_CODE, DEPLOYMENT_ACTION_SUCCESS_CODE, DEPLOYMENT_ACTION_IMPOSSIBLE_CODE, DEPLOYMENT_ACTION_NO_PERMISSION_CODE, \
-    DEPLOYMENT_ACTION_NO_DEPLOYMENT_CODE, DEPLOYMENT_ACTION_FAILED_CODE, DEPLOYMENT_PLAN_V1_API_OPS, DEPLOYMENT_PLAN_V2_API_OPS
+    DEPLOYMENT_ACTION_NO_DEPLOYMENT_CODE, DEPLOYMENT_ACTION_FAILED_CODE, DEPLOYMENT_PLAN_V1_API_OPS, DEPLOYMENT_PLAN_V2_API_OPS, \
+    ENVIRONMENTS_ENDPOINT, DEPLOYMENT_ENDPOINT
 from outsystems.vars.file_vars import DEPLOYMENTS_FILE, DEPLOYMENT_FILE, DEPLOYMENT_FOLDER, DEPLOYMENT_STATUS_FILE
 from outsystems.vars.pipeline_vars import DEPLOYMENT_STATUS_LIST, DEPLOYMENT_SAVED_STATUS
 
@@ -193,6 +194,32 @@ def send_deployment(artifact_dir: str, endpoint: str, auth_token: str, lt_api_ve
             "There was an error. Response from server: {}".format(response))
 
 
+# Creates a deployment to a target environment.
+# The input is a binary file.
+def send_binary_deployment(artifact_dir: str, endpoint: str, auth_token: str, lt_api_version: int, dest_env: str, binary_path: str):
+    # Sends the request
+    response = send_binary_post_request(
+        endpoint, auth_token, ENVIRONMENTS_ENDPOINT, dest_env, DEPLOYMENT_ENDPOINT, binary_path)
+    status_code = int(response["http_status"])
+    if status_code == DEPLOYMENT_SUCCESS_CODE:
+        return response["response"]
+    elif status_code == DEPLOYMENT_INVALID_CODE:
+        raise InvalidParametersError("The request is invalid. Check the body of the request for errors. Details: {}.".format(
+            response["response"]))
+    elif status_code == DEPLOYMENT_NO_PERMISSION_CODE:
+        raise NotEnoughPermissionsError(
+            "You don't have enough permissions to create the deployment. Details: {}".format(response["response"]))
+    elif status_code == DEPLOYMENT_NO_ENVIRONMENT_CODE:
+        raise EnvironmentNotFoundError(
+            "Can't find the source or target environment. Details: {}.".format(response["response"]))
+    elif status_code == DEPLOYMENT_FAILED_CODE:
+        raise ServerError(
+            "Failed to create the deployment. Details: {}".format(response["response"]))
+    else:
+        raise NotImplementedError(
+            "There was an error. Response from server: {}".format(response))
+
+
 # Discards a deployment, if possible. Only deployments whose state is “saved” can be deleted.
 def delete_deployment(endpoint: str, auth_token: str, deployment_key: str):
     # Builds the API call

outsystems/lifetime/lifetime_downloads.py

@@ -0,0 +1,45 @@
+# Python Modules
+import os
+
+# Custom Modules
+# Exceptions
+from outsystems.exceptions.invalid_parameters import InvalidParametersError
+from outsystems.exceptions.environment_not_found import EnvironmentNotFoundError
+from outsystems.exceptions.not_enough_permissions import NotEnoughPermissionsError
+from outsystems.exceptions.server_error import ServerError
+# Functions
+from outsystems.lifetime.lifetime_base import send_download_request
+
+# Variables
+from outsystems.vars.lifetime_vars import DOWNLOAD_SUCCESS_CODE, DOWNLOAD_INVALID_KEY_CODE, \
+    DOWNLOAD_NO_PERMISSION_CODE, DOWNLOAD_NOT_FOUND, DOWNLOAD_FAILED_CODE
+
+
+# Downloads a binary file from a LifeTime download link
+def download_package(file_path: str, auth_token: str, pkg_url: str):
+    # Sends the request
+    response = send_download_request(pkg_url, auth_token)
+    status_code = int(response["http_status"])
+
+    if status_code == DOWNLOAD_SUCCESS_CODE:
+        # Remove the spaces in the filename
+        file_path = file_path.replace(" ", "_")
+        # Makes sure that, if a directory is in the filename, that directory exists
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, "wb") as f:
+            f.write(response["response"])
+    elif status_code == DOWNLOAD_INVALID_KEY_CODE:
+        raise InvalidParametersError("The required type <Type> is invalid for given keys (EnvironmentKey; ApplicationKey). Details: {}".format(
+            response["response"]))
+    elif status_code == DOWNLOAD_NO_PERMISSION_CODE:
+        raise NotEnoughPermissionsError("User doesn't have permissions for the given keys (EnvironmentKey; ApplicationKey). Details: {}".format(
+            response["response"]))
+    elif status_code == DOWNLOAD_NOT_FOUND:
+        raise EnvironmentNotFoundError("No environment or application found. Please check that the EnvironmentKey and ApplicationKey exist. Details: {}".format(
+            response["response"]))
+    elif status_code == DOWNLOAD_FAILED_CODE:
+        raise ServerError("Failed to start the operation to package. Details: {}".format(
+            response["response"]))
+    else:
+        raise NotImplementedError(
+            "There was an error. Response from server: {}".format(response))

outsystems/lifetime/lifetime_environments.py

@@ -9,16 +9,19 @@
 from outsystems.exceptions.app_does_not_exist import AppDoesNotExistError
 from outsystems.exceptions.server_error import ServerError
 # Functions
-from outsystems.lifetime.lifetime_base import send_get_request
+from outsystems.lifetime.lifetime_base import send_get_request, send_post_request
 from outsystems.lifetime.lifetime_applications import _get_application_info
 from outsystems.file_helpers.file import load_data, store_data, clear_cache
 # Variables
 from outsystems.vars.lifetime_vars import ENVIRONMENTS_ENDPOINT, ENVIRONMENT_APPLICATIONS_ENDPOINT, ENVIRONMENTS_SUCCESS_CODE, \
     ENVIRONMENTS_NOT_FOUND_CODE, ENVIRONMENTS_FAILED_CODE, ENVIRONMENT_APP_SUCCESS_CODE, ENVIRONMENT_APP_NOT_STATUS_CODE, \
     ENVIRONMENT_APP_NO_PERMISSION_CODE, ENVIRONMENT_APP_NOT_FOUND, ENVIRONMENT_APP_FAILED_CODE, ENVIRONMENT_DEPLOYMENT_ZONES_ENDPOINT, \
     ENVIRONMENT_ZONES_SUCCESS_CODE, ENVIRONMENT_ZONES_NOT_STATUS_CODE, ENVIRONMENT_ZONES_NO_PERMISSION_CODE, ENVIRONMENT_ZONES_NOT_FOUND, \
-    ENVIRONMENT_ZONES_FAILED_CODE
-from outsystems.vars.file_vars import ENVIRONMENTS_FILE, ENVIRONMENT_FOLDER, ENVIRONMENT_APPLICATION_FILE, ENVIRONMENT_DEPLOYMENT_ZONES_FILE
+    ENVIRONMENT_ZONES_FAILED_CODE, ENVIRONMENT_APPLICATIONS_SOURCECODE_ENDPOINT, ENVIRONMENT_SOURCECODE_PACKAGE_SUCCESS_CODE, \
+    ENVIRONMENT_SOURCECODE_LINK_SUCCESS_CODE, ENVIRONMENT_SOURCECODE_FAILED_CODE
+from outsystems.vars.file_vars import ENVIRONMENTS_FILE, ENVIRONMENT_FOLDER, ENVIRONMENT_APPLICATION_FILE, \
+    ENVIRONMENT_DEPLOYMENT_ZONES_FILE, ENVIRONMENT_SOURCECODE_FOLDER, ENVIRONMENT_SOURCECODE_STATUS_FILE, \
+    ENVIRONMENT_SOURCECODE_LINK_FILE
 
 
 # Lists all the environments in the infrastructure.
@@ -120,6 +123,91 @@ def get_environment_deployment_zones(artifact_dir: str, endpoint: str, auth_toke
             "There was an error. Response from server: {}".format(response))
 
 
+# Returns the package key to download the source code of the specified application in a given environment.
+def get_environment_app_source_code(artifact_dir: str, endpoint: str, auth_token: str, **kwargs):
+    # Tuple with (AppName, AppKey): app_tuple[0] = AppName; app_tuple[1] = AppKey
+    app_tuple = _get_application_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Tuple with (EnvName, EnvKey): env_tuple[0] = EnvName; env_tuple[1] = EnvKey
+    env_tuple = _get_environment_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Builds the query and arguments for the call to the API
+    query = "{}/{}/{}/{}/{}".format(ENVIRONMENTS_ENDPOINT, env_tuple[1],
+                                    ENVIRONMENT_APPLICATIONS_ENDPOINT, app_tuple[1],
+                                    ENVIRONMENT_APPLICATIONS_SOURCECODE_ENDPOINT)
+    # Sends the request
+    response = send_post_request(endpoint, auth_token, query, None)
+    status_code = int(response["http_status"])
+    if status_code == ENVIRONMENT_SOURCECODE_PACKAGE_SUCCESS_CODE:
+        return response["response"]
+    elif status_code == ENVIRONMENT_SOURCECODE_FAILED_CODE:
+        raise ServerError("Failed to access the source code of an application. Details: {}".format(
+            response["response"]))
+    else:
+        raise NotImplementedError(
+            "There was an error. Response from server: {}".format(response))
+
+
+# Returns current status of source code package of the specified application in a given environment.
+def get_environment_app_source_code_status(artifact_dir: str, endpoint: str, auth_token: str, **kwargs):
+    # Tuple with (AppName, AppKey): app_tuple[0] = AppName; app_tuple[1] = AppKey
+    app_tuple = _get_application_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Tuple with (EnvName, EnvKey): env_tuple[0] = EnvName; env_tuple[1] = EnvKey
+    env_tuple = _get_environment_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Builds the query and arguments for the call to the API
+    query = "{}/{}/{}/{}/{}/{}/status".format(ENVIRONMENTS_ENDPOINT, env_tuple[1],
+                                              ENVIRONMENT_APPLICATIONS_ENDPOINT, app_tuple[1],
+                                              ENVIRONMENT_APPLICATIONS_SOURCECODE_ENDPOINT, kwargs["pkg_key"])
+    # Sends the request
+    response = send_get_request(endpoint, auth_token, query, None)
+    status_code = int(response["http_status"])
+    if status_code == ENVIRONMENT_SOURCECODE_PACKAGE_SUCCESS_CODE:
+        # Stores the result
+        filename = "{}{}".format(
+            kwargs["pkg_key"], ENVIRONMENT_SOURCECODE_STATUS_FILE)
+        filename = os.path.join(ENVIRONMENT_SOURCECODE_FOLDER, filename)
+        store_data(artifact_dir, filename, response["response"])
+        return response["response"]
+    elif status_code == ENVIRONMENT_SOURCECODE_FAILED_CODE:
+        raise ServerError("Failed to access the source code package status of an application. Details: {}".format(
+            response["response"]))
+    else:
+        raise NotImplementedError(
+            "There was an error. Response from server: {}".format(response))
+
+
+# Returns download link of source code package of the specified application in a given environment.
+def get_environment_app_source_code_link(artifact_dir: str, endpoint: str, auth_token: str, **kwargs):
+    # Tuple with (AppName, AppKey): app_tuple[0] = AppName; app_tuple[1] = AppKey
+    app_tuple = _get_application_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Tuple with (EnvName, EnvKey): env_tuple[0] = EnvName; env_tuple[1] = EnvKey
+    env_tuple = _get_environment_info(
+        artifact_dir, endpoint, auth_token, **kwargs)
+    # Builds the query and arguments for the call to the API
+    query = "{}/{}/{}/{}/{}/{}/download".format(ENVIRONMENTS_ENDPOINT, env_tuple[1],
+                                                ENVIRONMENT_APPLICATIONS_ENDPOINT, app_tuple[1],
+                                                ENVIRONMENT_APPLICATIONS_SOURCECODE_ENDPOINT, kwargs["pkg_key"])
+    # Sends the request
+    response = send_get_request(endpoint, auth_token, query, None)
+    status_code = int(response["http_status"])
+    if status_code == ENVIRONMENT_SOURCECODE_LINK_SUCCESS_CODE:
+        # Stores the result
+        filename = "{}{}".format(
+            kwargs["pkg_key"], ENVIRONMENT_SOURCECODE_LINK_FILE)
+        filename = os.path.join(ENVIRONMENT_SOURCECODE_FOLDER, filename)
+        store_data(artifact_dir, filename, response["response"])
+        return response["response"]
+    elif status_code == ENVIRONMENT_SOURCECODE_FAILED_CODE:
+        raise ServerError("Failed to access the source code package link of an application. Details: {}".format(
+            response["response"]))
+    else:
+        raise NotImplementedError(
+            "There was an error. Response from server: {}".format(response))
+
+
 # ---------------------- PRIVATE METHODS ----------------------
 # Private method to get the App name or key into a tuple (name,key).
 def _get_environment_info(artifact_dir: str, api_url: str, auth_token: str, **kwargs):